From 054bf58a6e05cacd04035669307d1c2f07cde72b Mon Sep 17 00:00:00 2001 From: Yalz Date: Mon, 23 Sep 2024 10:53:25 +0200 Subject: [PATCH 01/17] ci: fix missing project version on docker images --- .github/workflows/pr-merged.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-merged.yml b/.github/workflows/pr-merged.yml index 22997e714..066a532d9 100644 --- a/.github/workflows/pr-merged.yml +++ b/.github/workflows/pr-merged.yml @@ -21,6 +21,8 @@ jobs: permissions: contents: read packages: write + outputs: + version: ${{ steps.determine-version.outputs.version }} steps: - uses: actions/checkout@v4 - name: Set up JDK 21 @@ -52,11 +54,14 @@ jobs: gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Determine version + id: determine-version + run: | + export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) + echo "::set-output name=version::$VERSION" - name: Analyse & publish package run: | mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar deploy -Dsonar.projectKey=Informatievlaanderen_VSDS-Linked-Data-Interactions -Pcoverage,release -Dparallel=all -DperCoreThreadCount=true - export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) - echo "version=$VERSION" >> $GITHUB_ENV env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} @@ -93,9 +98,9 @@ jobs: - name: Define docker variables run: | if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - echo "IMAGE_TAG=${{ env.version }}" >> $GITHUB_ENV + echo "IMAGE_TAG=${{ needs.build.outputs.version }}" >> $GITHUB_ENV echo "IMAGES=ldes/${{ env.LDIO_IMAGE_NAME }}" >> $GITHUB_ENV - if [[ "${{ env.version }}" != *"SNAPSHOT"* ]]; then + if [[ "${{ needs.build.outputs.version }}" != *"SNAPSHOT"* ]]; then echo "LATEST=latest" >> $GITHUB_ENV fi else @@ -154,9 +159,9 @@ jobs: - name: Define docker variables run: | if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - echo "IMAGE_TAG=${{ env.version }}" >> $GITHUB_ENV + echo "IMAGE_TAG=${{ needs.build.outputs.version }}" >> $GITHUB_ENV echo "IMAGES=ldes/${{ env.DISCOVERER_IMAGE_NAME }}" >> $GITHUB_ENV - if [[ "${{ env.version }}" != *"SNAPSHOT"* ]]; then + if [[ "${{ needs.build.outputs.version }}" != *"SNAPSHOT"* ]]; then echo "LATEST=latest" >> $GITHUB_ENV fi else From 710f287ad368e1695924def7f4879be400928555 Mon Sep 17 00:00:00 2001 From: Yalz Date: Mon, 30 Sep 2024 17:58:58 +0200 Subject: [PATCH 02/17] ci: prepare release 2.10.0 --- ldi-api/pom.xml | 2 +- ldi-core/change-detection-filter/pom.xml | 2 +- ldi-core/file-archiving/pom.xml | 2 +- ldi-core/geojson-to-wkt/pom.xml | 2 +- ldi-core/json-to-ld-adapter/pom.xml | 2 +- ldi-core/ldes-client/pom.xml | 2 +- ldi-core/ldes-client/starting-node/pom.xml | 2 +- ldi-core/ldes-client/tree-node-fetcher/pom.xml | 2 +- ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml | 2 +- ldi-core/ldes-client/tree-node-supplier/pom.xml | 2 +- ldi-core/ldi-common/pom.xml | 2 +- ldi-core/ldi-infra-sql/pom.xml | 2 +- ldi-core/ngsiv2-to-ld-adapter/pom.xml | 2 +- ldi-core/pom.xml | 2 +- ldi-core/rdf-adapter/pom.xml | 2 +- ldi-core/repository-sink/pom.xml | 2 +- ldi-core/request-executor/pom.xml | 2 +- ldi-core/rml-adapter/pom.xml | 2 +- ldi-core/sparql-construct/pom.xml | 2 +- ldi-core/version-materialiser/pom.xml | 2 +- ldi-core/version-object-creator/pom.xml | 2 +- ldi-extensions/ldes-discoverer/pom.xml | 2 +- ldi-extensions/pom.xml | 2 +- ldi-nifi/ldi-nifi-common/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml | 2 +- .../ldi-nifi-processors/create-version-object-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/pom.xml | 2 +- .../ldi-nifi-processors/rdf4j-repository-sink-processor/pom.xml | 2 +- ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml | 2 +- .../ldi-nifi-processors/sparql-interactions-processor/pom.xml | 2 +- .../version-materialisation-processor/pom.xml | 2 +- ldi-nifi/pom.xml | 2 +- ldi-orchestrator/ldio-application/pom.xml | 2 +- ldi-orchestrator/ldio-common/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml | 2 +- .../ldio-connectors/ldio-change-detection-filter/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml | 2 +- .../ldio-connectors/ldio-json-to-ld-adapter/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml | 2 +- .../ldio-connectors/ldio-ldes-client-connector/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml | 2 +- .../ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-repository-sink/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml | 2 +- .../ldio-connectors/ldio-version-materialiser/pom.xml | 2 +- .../ldio-connectors/ldio-version-object-creator/pom.xml | 2 +- ldi-orchestrator/ldio-connectors/pom.xml | 2 +- ldi-orchestrator/pom.xml | 2 +- pom.xml | 2 +- 68 files changed, 68 insertions(+), 68 deletions(-) diff --git a/ldi-api/pom.xml b/ldi-api/pom.xml index 2d6710452..fd9e8ee57 100644 --- a/ldi-api/pom.xml +++ b/ldi-api/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/change-detection-filter/pom.xml b/ldi-core/change-detection-filter/pom.xml index ae49788ea..75a39d366 100644 --- a/ldi-core/change-detection-filter/pom.xml +++ b/ldi-core/change-detection-filter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT change-detection-filter diff --git a/ldi-core/file-archiving/pom.xml b/ldi-core/file-archiving/pom.xml index 75db9bb1a..63eec409b 100644 --- a/ldi-core/file-archiving/pom.xml +++ b/ldi-core/file-archiving/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT file-archiving diff --git a/ldi-core/geojson-to-wkt/pom.xml b/ldi-core/geojson-to-wkt/pom.xml index 5879ee530..83638a118 100644 --- a/ldi-core/geojson-to-wkt/pom.xml +++ b/ldi-core/geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT geojson-to-wkt diff --git a/ldi-core/json-to-ld-adapter/pom.xml b/ldi-core/json-to-ld-adapter/pom.xml index 79f6d79c8..2360f796f 100644 --- a/ldi-core/json-to-ld-adapter/pom.xml +++ b/ldi-core/json-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT json-to-ld-adapter diff --git a/ldi-core/ldes-client/pom.xml b/ldi-core/ldes-client/pom.xml index c9d36e9fd..590948eb5 100644 --- a/ldi-core/ldes-client/pom.xml +++ b/ldi-core/ldes-client/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/starting-node/pom.xml b/ldi-core/ldes-client/starting-node/pom.xml index 47490dbd1..480ad07c1 100644 --- a/ldi-core/ldes-client/starting-node/pom.xml +++ b/ldi-core/ldes-client/starting-node/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 starting-node diff --git a/ldi-core/ldes-client/tree-node-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-fetcher/pom.xml index 7cb449c88..58e7ed304 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml index 2a632a159..c6d875229 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/tree-node-supplier/pom.xml b/ldi-core/ldes-client/tree-node-supplier/pom.xml index 2aa521779..aeae9236e 100644 --- a/ldi-core/ldes-client/tree-node-supplier/pom.xml +++ b/ldi-core/ldes-client/tree-node-supplier/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 tree-node-supplier diff --git a/ldi-core/ldi-common/pom.xml b/ldi-core/ldi-common/pom.xml index 0bfacef2e..7772a91fd 100644 --- a/ldi-core/ldi-common/pom.xml +++ b/ldi-core/ldi-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldi-common diff --git a/ldi-core/ldi-infra-sql/pom.xml b/ldi-core/ldi-infra-sql/pom.xml index 5c1054e61..927782a38 100644 --- a/ldi-core/ldi-infra-sql/pom.xml +++ b/ldi-core/ldi-infra-sql/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldi-infra-sql diff --git a/ldi-core/ngsiv2-to-ld-adapter/pom.xml b/ldi-core/ngsiv2-to-ld-adapter/pom.xml index 784d5f7b5..a5262e3ed 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-core/ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ngsiv2-to-ld-adapter diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index 8206e85f4..88d19139c 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rdf-adapter/pom.xml b/ldi-core/rdf-adapter/pom.xml index 6c09db7c6..976b7f131 100644 --- a/ldi-core/rdf-adapter/pom.xml +++ b/ldi-core/rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/repository-sink/pom.xml b/ldi-core/repository-sink/pom.xml index 3490640d1..afc9878e6 100644 --- a/ldi-core/repository-sink/pom.xml +++ b/ldi-core/repository-sink/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT diff --git a/ldi-core/request-executor/pom.xml b/ldi-core/request-executor/pom.xml index 041c01551..aa3893df0 100644 --- a/ldi-core/request-executor/pom.xml +++ b/ldi-core/request-executor/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 request-executor diff --git a/ldi-core/rml-adapter/pom.xml b/ldi-core/rml-adapter/pom.xml index 90b3beb91..1f56821bc 100644 --- a/ldi-core/rml-adapter/pom.xml +++ b/ldi-core/rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/sparql-construct/pom.xml b/ldi-core/sparql-construct/pom.xml index f865e5d88..809835f96 100644 --- a/ldi-core/sparql-construct/pom.xml +++ b/ldi-core/sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-materialiser/pom.xml b/ldi-core/version-materialiser/pom.xml index 1b710f5f5..fe54ef934 100644 --- a/ldi-core/version-materialiser/pom.xml +++ b/ldi-core/version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-object-creator/pom.xml b/ldi-core/version-object-creator/pom.xml index a795a36aa..d6a2f657a 100644 --- a/ldi-core/version-object-creator/pom.xml +++ b/ldi-core/version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-extensions/ldes-discoverer/pom.xml b/ldi-extensions/ldes-discoverer/pom.xml index 5cf5088e5..3578205e1 100644 --- a/ldi-extensions/ldes-discoverer/pom.xml +++ b/ldi-extensions/ldes-discoverer/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-extensions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldes-discoverer diff --git a/ldi-extensions/pom.xml b/ldi-extensions/pom.xml index 193b786d9..2122497eb 100644 --- a/ldi-extensions/pom.xml +++ b/ldi-extensions/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT be.vlaanderen.informatievlaanderen.ldes.ldi diff --git a/ldi-nifi/ldi-nifi-common/pom.xml b/ldi-nifi/ldi-nifi-common/pom.xml index d778fde63..6ffe4990d 100644 --- a/ldi-nifi/ldi-nifi-common/pom.xml +++ b/ldi-nifi/ldi-nifi-common/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml index 4d05866f9..bed35626e 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT archive-file-in diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml index aaad02a1e..777ee7918 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT archive-file-out diff --git a/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml index 873e93e49..54ada55b0 100644 --- a/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT change-detection-processor diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml index b0b3cb2d9..b5a15256b 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml index adbf9ea8a..44dc85b94 100644 --- a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT geojson-to-wkt-processor diff --git a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml index d409dce89..2be4682ef 100644 --- a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT json-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index 8025c2ddd..0634f8e20 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml index c23dbfd83..b5a98ada9 100644 --- a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldi-processors-bundle diff --git a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml index b4402565f..98b96413e 100644 --- a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml @@ -7,7 +7,7 @@ ldi-nifi-processors be.vlaanderen.informatievlaanderen.ldes.ldi.nifi - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ngsiv2-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/pom.xml b/ldi-nifi/ldi-nifi-processors/pom.xml index 468e41417..5f8eeb377 100644 --- a/ldi-nifi/ldi-nifi-processors/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-sink-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-sink-processor/pom.xml index 099f61408..fd545c2d5 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-sink-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-sink-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT rdf4j-repository-sink-processor diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml index a871816f4..cde26fc1b 100644 --- a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT rml-adapter-processor diff --git a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml index 2f78381ca..3f83be559 100644 --- a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml index fde47457f..bef45f11b 100644 --- a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/pom.xml b/ldi-nifi/pom.xml index 40af71b05..52a11dc13 100644 --- a/ldi-nifi/pom.xml +++ b/ldi-nifi/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 pom diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index e9820fae8..04defe782 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-common/pom.xml b/ldi-orchestrator/ldio-common/pom.xml index 84f6b5864..0aaf9551f 100644 --- a/ldi-orchestrator/ldio-common/pom.xml +++ b/ldi-orchestrator/ldio-common/pom.xml @@ -5,7 +5,7 @@ ldi-orchestrator be.vlaanderen.informatievlaanderen.ldes.ldi - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml index 306c4f362..a49dc1bd3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldio-amqp diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml index aa82b8e78..1381365a9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldio-archive-file-in diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml index 9d3ddf7ca..70b795905 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT diff --git a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml index a17ce01d7..a961894d1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldio-change-detection-filter diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml index 017c8f18f..0afbb2fbc 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml index eaccead17..4ecaa257c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldio-file-out diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml index d59dbcf55..464e32ff5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldio-geojson-to-wkt diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml index 4cdbf097d..181227884 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml index 50425e5c8..59d206f15 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldio-http-in-poller diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml index 88c1333c9..215dd94fe 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml @@ -4,7 +4,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml index 473893e60..338cc5652 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml index ff8f9cd5b..263c55621 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml index e2e1b5473..71756e5c2 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml index dc12c5ef9..7ea8ce07e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index 86909195e..2aa2a57f4 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml index a24f4b33c..082de591b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldio-ngsiv2-to-ld-adapter diff --git a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml index d88a97c6a..83ae1c32e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml @@ -4,7 +4,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml index d3f751c97..c674ff43f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-sink/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-repository-sink/pom.xml index ec182e849..644762296 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-sink/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-sink/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldio-repository-sink diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml index 37d9beb8d..5f1fde380 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldio-request-executor diff --git a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml index eff114a69..756344b75 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml index 30cfaac3e..6d532da31 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml index d8f0d3c4e..60ebfc1ba 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml index 4d55c97db..b3ed357e9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index 9746aa08c..bf6ff5b4d 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/pom.xml b/ldi-orchestrator/pom.xml index 0491fb114..f14deffa4 100644 --- a/ldi-orchestrator/pom.xml +++ b/ldi-orchestrator/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 979e6d3b2..51f82f89b 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions pom - 2.9.0-SNAPSHOT + 2.10.0-SNAPSHOT ldi-api From daa29a7e223c717b23a0a97e4a095b41d29746fc Mon Sep 17 00:00:00 2001 From: Jan Robert <15772440+Yalz@users.noreply.github.com> Date: Mon, 30 Sep 2024 18:22:04 +0200 Subject: [PATCH 03/17] feat: virtuoso out (#685) Co-authored-by: Yalz --- .github/ldio.Dockerfile | 1 + ldi-orchestrator/ldio-application/pom.xml | 10 ++++ .../ldio-connectors/ldio-virtuoso-out/pom.xml | 27 +++++++++ .../ldes/ldio/LdioVirtuosoOut.java | 60 +++++++++++++++++++ .../config/LdioVirtuosoOutAutoConfig.java | 35 +++++++++++ ldi-orchestrator/ldio-connectors/pom.xml | 1 + 6 files changed, 134 insertions(+) create mode 100644 ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/pom.xml create mode 100644 ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVirtuosoOut.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVirtuosoOutAutoConfig.java diff --git a/.github/ldio.Dockerfile b/.github/ldio.Dockerfile index 52c5ce510..1fb41b6ab 100644 --- a/.github/ldio.Dockerfile +++ b/.github/ldio.Dockerfile @@ -32,6 +32,7 @@ COPY ./ldi-orchestrator/ldio-connectors/ldio-console-out/target/ldio-console-out COPY ./ldi-orchestrator/ldio-connectors/ldio-http-out/target/ldio-http-out-jar-with-dependencies.jar ./lib/ COPY ./ldi-orchestrator/ldio-connectors/ldio-noop-out/target/ldio-noop-out-jar-with-dependencies.jar ./lib/ COPY ./ldi-orchestrator/ldio-connectors/ldio-repository-sink/target/ldio-repository-sink-jar-with-dependencies.jar ./lib/ +COPY ./ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/target/ldio-virtuoso-out-jar-with-dependencies.jar ./lib/ RUN mkdir "state" diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index 04defe782..97ab124a3 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -379,6 +379,16 @@ + + ldio-virtuoso-out + + + be.vlaanderen.informatievlaanderen.ldes.ldio + ldio-virtuoso-out + ${project.version} + + + \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/pom.xml new file mode 100644 index 000000000..2b2c27e7e --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/pom.xml @@ -0,0 +1,27 @@ + + + + ldio-connectors + be.vlaanderen.informatievlaanderen.ldes.ldio + 2.10.0-SNAPSHOT + + 4.0.0 + + ldio-virtuoso-out + + + + org.springframework + spring-web + + + be.vlaanderen.informatievlaanderen.ldes.ldio + ldio-request-executor + ${project.version} + compile + + + + \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVirtuosoOut.java b/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVirtuosoOut.java new file mode 100644 index 000000000..29b9f73a4 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVirtuosoOut.java @@ -0,0 +1,60 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.LdiRdfWriter; +import be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.LdiRdfWriterProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.PostRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeader; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeaders; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Response; +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOutput; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpHeaders; + +import java.io.ByteArrayOutputStream; +import java.util.List; + +public class LdioVirtuosoOut implements LdiOutput { + public static final String NAME = "Ldio:VirtuosoOut"; + private static final Logger log = LoggerFactory.getLogger(LdioVirtuosoOut.class); + + private final RequestExecutor requestExecutor; + private final String endpoint; + private final String graph; + private final LdiRdfWriter ldiRdfWriter; + + public LdioVirtuosoOut(RequestExecutor requestExecutor, String endpoint, String graph) { + this.requestExecutor = requestExecutor; + this.endpoint = endpoint; + this.graph = graph; + this.ldiRdfWriter = LdiRdfWriter.getRdfWriter(new LdiRdfWriterProperties().withLang(Lang.NQUADS)); + } + + @Override + public void accept(Model linkedDataModel) { + if (!linkedDataModel.isEmpty()) { + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + ldiRdfWriter.writeToOutputStream(linkedDataModel, output); + String modelAsNQuads = RDFWriter.source(linkedDataModel).lang(Lang.NQUADS).asString(); + String body = "INSERT INTO <%s> { %s }".formatted(graph, modelAsNQuads); + + final PostRequest request = new PostRequest(endpoint, new RequestHeaders(List.of( + new RequestHeader(HttpHeaders.CONTENT_TYPE, "application/sparql-update"), + new RequestHeader(HttpHeaders.ACCEPT, "application/json"))), body); + synchronized (requestExecutor) { + Response response = requestExecutor.execute(request); + if (response.isSuccess()) { + log.debug("{} {} {}", request.getMethod(), request.getUrl(), response.getHttpStatus()); + } else { + log.atError().log("Failed to post model. The request url was {}. " + + "The http response obtained from the server has code {} and body \"{}\".", + response.getRequestedUrl(), response.getHttpStatus(), response.getBodyAsString().orElse(null)); + } + } + } + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVirtuosoOutAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVirtuosoOutAutoConfig.java new file mode 100644 index 000000000..60e628e35 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVirtuosoOutAutoConfig.java @@ -0,0 +1,35 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVirtuosoOut; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioOutputConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVirtuosoOut.NAME; + +@Configuration +public class LdioVirtuosoOutAutoConfig { + + @SuppressWarnings("java:S6830") + @Bean(NAME) + public LdioOutputConfigurator ldiHttpOutConfigurator() { + return new LdioVirtuosoOutConfigurator(); + } + + public static class LdioVirtuosoOutConfigurator implements LdioOutputConfigurator { + + @Override + public LdiComponent configure(ComponentProperties config) { + final RequestExecutor requestExecutor = new LdioRequestExecutorSupplier().getRequestExecutor(config); + + String graph = config.getProperty("graph"); + String endpoint = config.getProperty("endpoint"); + + return new LdioVirtuosoOut(requestExecutor, endpoint, graph); + } + } +} diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index bf6ff5b4d..248ae8ae5 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -40,6 +40,7 @@ ldio-file-out ldio-repository-sink ldio-change-detection-filter + ldio-virtuoso-out From 0240955178aac5504e1abb4d947e3f5d30c010a9 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Thu, 3 Oct 2024 09:10:03 +0200 Subject: [PATCH 04/17] chore: ldes client status (#686) --- docs/_ldio/ldio-transformers/index.md | 6 +- docs/_ldio/pipeline-management/index.md | 98 +---------------- .../pipeline-management/ldes-client-status.md | 45 ++++++++ .../management-of-pipelines.md | 103 ++++++++++++++++++ docs/_ldio/pipeline-management/openapi.json | 81 ++++++++++++++ .../_ldio/pipeline-management/pipeline-api.md | 1 + .../pipeline-management/pipeline-status.md | 13 ++- .../pipeline-management/startup-config.md | 45 +++++++- .../status/ClientStatusController.java | 6 +- 9 files changed, 286 insertions(+), 112 deletions(-) create mode 100644 docs/_ldio/pipeline-management/ldes-client-status.md create mode 100644 docs/_ldio/pipeline-management/management-of-pipelines.md diff --git a/docs/_ldio/ldio-transformers/index.md b/docs/_ldio/ldio-transformers/index.md index c52a15ac8..406ca09cf 100644 --- a/docs/_ldio/ldio-transformers/index.md +++ b/docs/_ldio/ldio-transformers/index.md @@ -6,4 +6,8 @@ has_toc: true nav_order: 5 --- -# Linked Data Interactions Orchestrator Transformers \ No newline at end of file +# Linked Data Interactions Orchestrator Transformers + +The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. + +Each component can be wrapped in a desired implementation framework (LDI-orchestrator, NiFi, ...) to be used. diff --git a/docs/_ldio/pipeline-management/index.md b/docs/_ldio/pipeline-management/index.md index 6fdd07db8..00e94e685 100644 --- a/docs/_ldio/pipeline-management/index.md +++ b/docs/_ldio/pipeline-management/index.md @@ -5,100 +5,4 @@ has_children: true has_toc: true nav_order: 1 --- - -# Management of Pipelines - -Pipelines in LDIO can be created in YAML or JSON configuration (although all example configurations are made in YAML, -these can also be formatted in JSON). - -A default pipeline looks as follows: - -```yaml - name: my-first-pipeline - input: - name: fully-qualified name of LDI Input - config: - foo: bar - adapter: - name: fully-qualified name of LDI Adapter - config: - foo: bar - transformers: - - name: fully-qualified name of LDI Transformer - config: - foo: bar - outputs: - - name: fully-qualified name of LDI Transformer - config: - foo: bar -``` - -- Note that one orchestrator can have multiple pipelines -- Note that one pipeline can have multiple LDI Transformers and LDI Outputs - -## Anatomy of a pipeline - -Each pipeline is built up of the following components: - -* [LDIO Input](ldi-inputs): A component that will receive data (not necessarily LD) to then feed the LDIO pipeline. -* [LDIO Adapter](ldi-adapters): To be used in conjunction with the LDIO Input, the LDIO Adapter will transform the - provided content into and internal Linked Data model and sends it down the pipeline. -* [LDIO Transformer](ldi-transformers): A component that takes in a Linked Data model, transforms/modifies it and then - puts it back on the pipeline. -* [LDIO Output](ldi-outputs): A component that will take in Linked Data and will export it to external sources. - -````mermaid -stateDiagram-v2 - direction LR - - LDI_Input --> LDI_Transformer : LD - LDI_Transformer --> LDI_Output : LD - - state LDI_Input { - direction LR - [*] --> LDI_Adapter : Non LD - - state LDI_Adapter { - direction LR - [*] --> adapt - adapt --> [*] - } - - LDI_Adapter --> [*] : LD - } - - state LDI_Transformer { - direction LR - [*] --> transform - transform --> [*] - } - state LDI_Output { - direction LR - [*] --> [*] - } -```` - -## Persistence of Pipelines - -By default, all pipelines defined after startup (via management API) will be lost on restart. - -To prevent this behaviour, add the `orchestrator.directory` property as follows: - -```yaml -orchestrator: - directory: "{directory in application folder}" -``` - -If this directory does not exist, it will be created. - -> **_NOTE:_** An application config can be defined by creating an application YAML file in the LDIO directory -(in docker, this correlates to `/ldio/application.yml`). - - -## Pausing & Resuming LDIO - -Sometimes it might be preferred to pause an LDIO pipeline instead of deleting and recreating it. -The endpoints to manage pipelines can be found [here](pipeline-api.md) - -The exact behaviour of a paused pipeline depends on its input component and can be found in the [documentation of these components](docs/_ldio/ldio-inputs/index.md). -However, it will always complete its current run through the pipeline and then seize sending any output. \ No newline at end of file +# Pipeline Management \ No newline at end of file diff --git a/docs/_ldio/pipeline-management/ldes-client-status.md b/docs/_ldio/pipeline-management/ldes-client-status.md new file mode 100644 index 000000000..0659c9ed6 --- /dev/null +++ b/docs/_ldio/pipeline-management/ldes-client-status.md @@ -0,0 +1,45 @@ +--- +layout: default +parent: Pipeline Management +title: LDES Client Status +nav_order: 4 +--- + +# LDES Client Status + +Just like the LDIO pipelines have a status, so does the [`Ldio:LdesClient`](../ldio-inputs/ldio-ldes-client). The client +status can be fetched when a pipeline that has a running status, and of course when it contains an LDES client as input +component. + +## Overview Of The Status Flow + +```mermaid +graph LR +; + REPLICATING --> SYNCHRONISING; + REPLICATING --> COMPLETED; + SYNCHRONISING --> COMPLETED; + SYNCHRONISING --> ERROR; + REPLICATING --> ERROR; +``` + +The above diagram shows the flow between the different statuses of the client. + +## REPLICATING + +The startup status of the client. This status indicates that the LDES client have not yet fetched all the available +fragments of a view (or views if so configured) + +## SYNCHRONISING + +This status indicates that all the fragments of the configured view(s) have been fetched at least once, and there is at +least one fragment that does not have an immutable state yet. + +## ERROR + +This status indicates that an error has occurred somewhere while `REPLICATING` or `SYNCHRONISING` + +## COMPLETED + +This status indicates that all the fragments of the configured view has been fetched at least once and all those have an +immutable state, or in other words, the end of the LDES has been reached. diff --git a/docs/_ldio/pipeline-management/management-of-pipelines.md b/docs/_ldio/pipeline-management/management-of-pipelines.md new file mode 100644 index 000000000..dd2be92bb --- /dev/null +++ b/docs/_ldio/pipeline-management/management-of-pipelines.md @@ -0,0 +1,103 @@ +--- +layout: default +parent: Pipeline Management +title: Management of Pipelines +nav_order: 1 +--- + +# Management of Pipelines + +Pipelines in LDIO can be created in YAML or JSON configuration (although all example configurations are made in YAML, +these can also be formatted in JSON). + +A default pipeline looks as follows: + +```yaml + name: my-first-pipeline + input: + name: name of LDI Input + config: + foo: bar + adapter: + name: name of LDI Adapter + config: + foo: bar + transformers: + - name: name of LDI Transformer + config: + foo: bar + outputs: + - name: name of LDI Transformer + config: + foo: bar +``` + +- Note that one orchestrator can have multiple pipelines +- Note that one pipeline can have multiple LDI Transformers and LDI Outputs + +## Anatomy of a pipeline + +Each pipeline is built up of the following components: + +* [LDIO Input](ldi-inputs): A component that will receive data (not necessarily LD) to then feed the LDIO pipeline. +* [LDIO Adapter](ldi-adapters): To be used in conjunction with the LDIO Input, the LDIO Adapter will transform the + provided content into and internal Linked Data model and sends it down the pipeline. +* [LDIO Transformer](ldi-transformers): A component that takes in a Linked Data model, transforms/modifies it and then + puts it back on the pipeline. +* [LDIO Output](ldi-outputs): A component that will take in Linked Data and will export it to external sources. + +````mermaid +stateDiagram-v2 + direction LR + + LDI_Input --> LDI_Transformer : LD + LDI_Transformer --> LDI_Output : LD + + state LDI_Input { + direction LR + [*] --> LDI_Adapter : Non LD + + state LDI_Adapter { + direction LR + [*] --> adapt + adapt --> [*] + } + + LDI_Adapter --> [*] : LD + } + + state LDI_Transformer { + direction LR + [*] --> transform + transform --> [*] + } + state LDI_Output { + direction LR + [*] --> [*] + } +```` + +## Persistence of Pipelines + +By default, all pipelines defined after startup (via management API) will be lost on restart. + +To prevent this behaviour, add the `orchestrator.directory` property as follows: + +```yaml +orchestrator: + directory: "{directory in application folder}" +``` + +If this directory does not exist, it will be created. + +> **_NOTE:_** An application config can be defined by creating an application YAML file in the LDIO directory +(in docker, this correlates to `/ldio/application.yml`). + + +## Pausing & Resuming LDIO + +Sometimes it might be preferred to pause an LDIO pipeline instead of deleting and recreating it. +The endpoints to manage pipelines can be found [here](pipeline-api.md) + +The exact behaviour of a paused pipeline depends on its input component and can be found in the [documentation of these components](docs/_ldio/ldio-inputs/index.md). +However, it will always complete its current run through the pipeline and then seize sending any output. \ No newline at end of file diff --git a/docs/_ldio/pipeline-management/openapi.json b/docs/_ldio/pipeline-management/openapi.json index 214b71f57..b0f232034 100644 --- a/docs/_ldio/pipeline-management/openapi.json +++ b/docs/_ldio/pipeline-management/openapi.json @@ -219,6 +219,70 @@ } } }, + "/admin/api/v1/pipeline/ldes-client": { + "get": { + "tags": [ + "LDES Client Status" + ], + "summary": "Get a list of all LDES Client statuses pipelines.", + "operationId": "getStatusses", + "responses": { + "200": { + "description": "A list statuses of all active LDES Client pipelines.", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ClientStatusTo" + } + } + } + } + } + } + } + }, + "/admin/api/v1/pipeline/ldes-client/{pipeline}": { + "get": { + "tags": [ + "LDES Client Status" + ], + "summary": "Get the status of a requested LDES Client pipeline.", + "operationId": "getPipelineStatus_2", + "parameters": [ + { + "name": "pipeline", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "404": { + "description": "No LDES Client pipeline exists by that name" + }, + "200": { + "description": "Status of a requested pipeline", + "content": { + "text/plain": { + "schema": { + "type": "string", + "enum": [ + "REPLICATING", + "SYNCHRONISING", + "COMPLETED", + "ERROR" + ] + } + } + } + } + } + } + }, "/admin/api/v1/catalog": { "get": { "tags": [ @@ -372,6 +436,23 @@ } } }, + "ClientStatusTo": { + "type": "object", + "properties": { + "pipeline": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "REPLICATING", + "SYNCHRONISING", + "COMPLETED", + "ERROR" + ] + } + } + }, "LdioCatalog": { "type": "object", "properties": { diff --git a/docs/_ldio/pipeline-management/pipeline-api.md b/docs/_ldio/pipeline-management/pipeline-api.md index b76e170d1..a65615e0d 100644 --- a/docs/_ldio/pipeline-management/pipeline-api.md +++ b/docs/_ldio/pipeline-management/pipeline-api.md @@ -2,6 +2,7 @@ layout: default parent: Pipeline Management title: Pipeline Management API +nav_order: 2 --- diff --git a/docs/_ldio/pipeline-management/pipeline-status.md b/docs/_ldio/pipeline-management/pipeline-status.md index 35c0a0020..4f0a6affc 100644 --- a/docs/_ldio/pipeline-management/pipeline-status.md +++ b/docs/_ldio/pipeline-management/pipeline-status.md @@ -2,24 +2,25 @@ layout: default parent: Pipeline Management title: Pipeline Status +nav_order: 3 --- -# Status +# Pipeline Status An individual ldio-pipeline can be in one of several different statuses. These different statuses and their behaviour are dependent on the input component of the pipeline. -# Overview Of The Status Flow +## Overview Of The Status Flow - -
-graph LR; +```mermaid +graph LR +; INIT --> RUNNING; INIT --> STOPPED; RUNNING --> STOPPED; RUNNING --> HALTED; HALTED --> RUNNING; HALTED --> STOPPED; -
+``` The above diagram shows the flow between the different statuses of the pipeline. diff --git a/docs/_ldio/pipeline-management/startup-config.md b/docs/_ldio/pipeline-management/startup-config.md index 62c992a41..e7b99ce71 100644 --- a/docs/_ldio/pipeline-management/startup-config.md +++ b/docs/_ldio/pipeline-management/startup-config.md @@ -4,7 +4,7 @@ parent: Pipeline Management title: Startup Configuration --- -### Startup Configuration +## Startup Configuration On startup, pipelines can be defined by creating an application YAML file in the LDIO directory (in docker, this correlates to `/ldio/application.yml`) that looks as follows: @@ -14,19 +14,54 @@ orchestrator: pipelines: - name: my-first-pipeline input: - name: fully-qualified name of LDI Input + name: name of LDI Input config: foo: bar adapter: - name: fully-qualified name of LDI Adapter + name: name of LDI Adapter config: foo: bar transformers: - - name: fully-qualified name of LDI Transformer + - name: name of LDI Transformer config: foo: bar outputs: - - name: fully-qualified name of LDI Transformer + - name: name of LDI Output config: foo: bar ```` + +### Since the introduction of dynamic pipelines + +Since version 2.1.0, it is possible to manage pipelines on the fly. If pipelines must be instantiated on startup, those +pipelines can be added to a configured directory. + +First of all, to configure the directory, the `/ldio/application.yml` should look like this: + +````yaml +orchestrator: + directory: +```` + +The folder contains a yaml file for each pipeline. It is the preferred way to call the file the same way as the name of +the pipeline, in this case, that would be `my-first-pipeline.yml`: + +```yaml +name: my-first-pipeline +input: + name: name of LDI Input + config: + foo: bar + adapter: + name: name of LDI Adapter + config: + foo: bar +transformers: + - name: name of LDI Transformer + config: + foo: bar +outputs: + - name: name of LDI Output + config: + foo: bar +``` diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/status/ClientStatusController.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/status/ClientStatusController.java index 0269d9261..7a61a29b8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/status/ClientStatusController.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/status/ClientStatusController.java @@ -26,7 +26,7 @@ public ClientStatusController(ClientStatusService clientStatusService) { this.clientStatusService = clientStatusService; } - @GetMapping() + @GetMapping(produces = MediaType.APPLICATION_JSON_VALUE) @ApiResponse(responseCode = "200", description = "A list statuses of all active LDES Client pipelines.", content = { @Content(mediaType = MediaType.APPLICATION_JSON_VALUE, array = @ArraySchema(schema = @Schema(implementation = ClientStatusTo.class))), }) @@ -35,9 +35,9 @@ public List getStatusses() { return clientStatusService.getClientStatuses(); } - @GetMapping(path = "{pipeline}", produces = "application/json") + @GetMapping(path = "{pipeline}", produces = MediaType.TEXT_PLAIN_VALUE) @ApiResponse(responseCode = "200", description = "Status of a requested pipeline", content = { - @Content(mediaType = MediaType.APPLICATION_JSON_VALUE, schema = @Schema(implementation = ClientStatus.class)), + @Content(mediaType = MediaType.TEXT_PLAIN_VALUE, schema = @Schema(implementation = ClientStatus.class)), }) @ApiResponse(responseCode = "404", description = "No LDES Client pipeline exists by that name", content = { @Content(schema = @Schema()), From 735753d158a87b5a9d5a3c94e619d045a6b51972 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:06:01 +0200 Subject: [PATCH 05/17] fix: unexpected end of ldes behavior (#687) --- .../treenodesupplier/TreeNodeProcessor.java | 23 ++--- .../domain/valueobject/TreeNodeStatus.java | 5 +- .../ldio/pipeline/creation/LdioObserver.java | 8 ++ .../ldes/ldio/LdioLdesClient.java | 25 ++++- .../ldes/ldio/LdioLdesClientTest.java | 95 +++++++++++-------- 5 files changed, 101 insertions(+), 55 deletions(-) diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java index 6d4465123..96b3054cf 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java @@ -107,18 +107,19 @@ private void saveNewRelations(TreeNodeResponse treeNodeResponse) { private TreeNodeRecord getNextTreeNode() { TreeNodeRecord treeNodeRecord = treeNodeRecordRepository .getTreeNodeRecordWithStatusAndEarliestNextVisit(TreeNodeStatus.IMMUTABLE_WITH_UNPROCESSED_MEMBERS) - .orElseGet(() -> treeNodeRecordRepository.getTreeNodeRecordWithStatusAndEarliestNextVisit(TreeNodeStatus.NOT_VISITED) - .orElseGet(() -> treeNodeRecordRepository.getTreeNodeRecordWithStatusAndEarliestNextVisit(TreeNodeStatus.MUTABLE_AND_ACTIVE) - .orElseThrow(() -> { - clientStatusConsumer.accept(COMPLETED); - return new EndOfLdesException("No fragments to mutable or new fragments to process -> LDES ends."); - }))); - - if (Objects.requireNonNull(treeNodeRecord.getTreeNodeStatus()) == TreeNodeStatus.IMMUTABLE_WITH_UNPROCESSED_MEMBERS || - treeNodeRecord.getTreeNodeStatus() == TreeNodeStatus.IMMUTABLE_WITHOUT_UNPROCESSED_MEMBERS || - treeNodeRecord.getTreeNodeStatus() == TreeNodeStatus.NOT_VISITED) { + .or(() -> treeNodeRecordRepository.getTreeNodeRecordWithStatusAndEarliestNextVisit(TreeNodeStatus.NOT_VISITED)) + .or(() -> treeNodeRecordRepository.getTreeNodeRecordWithStatusAndEarliestNextVisit(TreeNodeStatus.MUTABLE_AND_ACTIVE)) + .orElseThrow(() -> { + clientStatusConsumer.accept(COMPLETED); + return new EndOfLdesException("No fragments to mutable or new fragments to process -> LDES ends."); + }); + + TreeNodeStatus treeNodeStatus = Objects.requireNonNull(treeNodeRecord.getTreeNodeStatus()); + if (treeNodeStatus == TreeNodeStatus.IMMUTABLE_WITH_UNPROCESSED_MEMBERS || + treeNodeStatus == TreeNodeStatus.IMMUTABLE_WITHOUT_UNPROCESSED_MEMBERS || + treeNodeStatus == TreeNodeStatus.NOT_VISITED) { clientStatusConsumer.accept(REPLICATING); - } else if (treeNodeRecord.getTreeNodeStatus() == TreeNodeStatus.MUTABLE_AND_ACTIVE) { + } else if (treeNodeStatus == TreeNodeStatus.MUTABLE_AND_ACTIVE) { clientStatusConsumer.accept(SYNCHRONISING); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/TreeNodeStatus.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/TreeNodeStatus.java index 8bfceea0a..ba13a8623 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/TreeNodeStatus.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/TreeNodeStatus.java @@ -4,5 +4,8 @@ * Representation how much of a TreeNode has been processed */ public enum TreeNodeStatus { - NOT_VISITED, MUTABLE_AND_ACTIVE, IMMUTABLE_WITH_UNPROCESSED_MEMBERS, IMMUTABLE_WITHOUT_UNPROCESSED_MEMBERS + NOT_VISITED, + MUTABLE_AND_ACTIVE, + IMMUTABLE_WITH_UNPROCESSED_MEMBERS, + IMMUTABLE_WITHOUT_UNPROCESSED_MEMBERS } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/creation/LdioObserver.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/creation/LdioObserver.java index 1dc4d88d2..57acf27a6 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/creation/LdioObserver.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/pipeline/creation/LdioObserver.java @@ -17,6 +17,7 @@ */ public class LdioObserver { private static final String LDIO_DATA_IN = "ldio_data_in"; + private static final String LDIO_DATA_OUT = "ldio_data_out"; private static final String LDIO_COMPONENT_NAME = "ldio_type"; private static final Logger log = LoggerFactory.getLogger(LdioObserver.class); private final String componentName; @@ -69,6 +70,13 @@ public void increment() { Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(); } + public boolean hasProcessedAllData() { + final double dataIn = Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).count(); + final double dataOut = Metrics.counter(LDIO_DATA_OUT, PIPELINE_NAME, pipelineName).count(); + log.atDebug().log("Received data: {} - Sent data: {}", dataIn, dataOut); + return dataOut >= dataIn; + } + /** * Registers a pipeline and the component and initializes the metrics * diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java index fc78f6826..2a88cdeb3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java @@ -13,18 +13,22 @@ import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; +import java.time.Duration; import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; import static java.util.concurrent.Executors.newSingleThreadExecutor; public class LdioLdesClient extends LdioInput { public static final String NAME = "Ldio:LdesClient"; + public static final String LDIO_SHUTDOWN_THREAD_NAME = "ldio-ldes-client-shutdown"; private final Logger log = LoggerFactory.getLogger(LdioLdesClient.class); private final MemberSupplier memberSupplier; private boolean threadRunning = true; + private final Supplier canGracefullyShutdownChecker; private boolean paused = false; private final boolean keepState; private final String pipelineName; @@ -37,10 +41,11 @@ public LdioLdesClient(ComponentExecutor componentExecutor, boolean keepState, ClientStatusConsumer clientStatusConsumer) { super(componentExecutor, null, ldioObserver, applicationEventPublisher); this.pipelineName = ldioObserver.getPipelineName(); + this.canGracefullyShutdownChecker = ldioObserver::hasProcessedAllData; this.memberSupplier = memberSupplier; - this.keepState = keepState; + this.keepState = keepState; this.clientStatusConsumer = clientStatusConsumer; - } + } @Override public void start() { @@ -77,7 +82,7 @@ private synchronized void checkPause() { while (paused) { try { this.wait(); - } catch (InterruptedException e) { + } catch (InterruptedException e) { log.error("Thread interrupted: {}", e.getMessage()); Thread.currentThread().interrupt(); } @@ -104,7 +109,17 @@ protected void pause() { } private void shutdownPipeline() { - log.info("SHUTTING DOWN pipeline {} because end of LDES has been reached", pipelineName); - applicationEventPublisher.publishEvent(new PipelineShutdownEvent(pipelineName)); + Thread.ofVirtual().name(LDIO_SHUTDOWN_THREAD_NAME).start(() -> { + while (Boolean.FALSE.equals(canGracefullyShutdownChecker.get())) { + try { + Thread.sleep(Duration.ofSeconds(30)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + log.info("SHUTTING DOWN pipeline {} because end of LDES has been reached", pipelineName); + updateStatus(PipelineStatusTrigger.HALT); + applicationEventPublisher.publishEvent(new PipelineShutdownEvent(pipelineName)); + }); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientTest.java index 1aae66552..d400a2f54 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientTest.java @@ -13,6 +13,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.context.ApplicationEventPublisher; @@ -25,60 +26,78 @@ @ExtendWith(MockitoExtension.class) class LdioLdesClientTest { - @Mock - private ComponentExecutor componentExecutor; + @Mock + private ComponentExecutor componentExecutor; - @Mock - private LdioObserver observer; + @Mock + private LdioObserver observer; - @Mock - private MemberSupplier supplier; + @Mock + private MemberSupplier supplier; - @Mock - private ApplicationEventPublisher eventPublisher; + @Mock + private ApplicationEventPublisher eventPublisher; - @Mock - private ClientStatusConsumer clientStatusConsumer; + @Mock + private ClientStatusConsumer clientStatusConsumer; - private LdioLdesClient client; - private final String pipelineName = "pipeline"; + private LdioLdesClient client; + private final String pipelineName = "pipeline"; - @BeforeEach - void setUp() { - when(observer.getPipelineName()).thenReturn(pipelineName); - client = new LdioLdesClient( - componentExecutor, - observer, - supplier, - eventPublisher, - false, - clientStatusConsumer); - } + @BeforeEach + void setUp() { + when(observer.getPipelineName()).thenReturn(pipelineName); + client = new LdioLdesClient( + componentExecutor, + observer, + supplier, + eventPublisher, + false, + clientStatusConsumer); + } - @AfterEach - void tearDown() { - client.shutdown(); - } + @AfterEach + void tearDown() { + client.shutdown(); + } - @Test - void when_EndOfLdesException_ShutdownPipeline() { - when(supplier.get()).thenThrow(EndOfLdesException.class); + @Test + void when_EndOfLdesException_And_AllDataProcessed_ShutdownPipeline() { + when(observer.hasProcessedAllData()).thenReturn(true); + when(supplier.get()).thenThrow(EndOfLdesException.class); - client.start(); + client.start(); - verify(eventPublisher).publishEvent(new PipelineShutdownEvent(pipelineName)); - } + InOrder inOrder = inOrder(eventPublisher); + inOrder.verify(eventPublisher).publishEvent(new PipelineStatusEvent(pipelineName, PipelineStatus.RUNNING, StatusChangeSource.MANUAL)); + inOrder.verify(eventPublisher).publishEvent(new PipelineStatusEvent(pipelineName, PipelineStatus.HALTED, StatusChangeSource.MANUAL)); + inOrder.verify(eventPublisher).publishEvent(new PipelineShutdownEvent(pipelineName)); + } - @Test - void when_RuntimeException_StopPipeline() { - doThrow(RuntimeException.class).when(supplier).init(); + @Test + void when_EndOfLdesException_ShutdownPipeline() { + when(observer.hasProcessedAllData()).thenReturn(false).thenReturn(true); + when(supplier.get()).thenThrow(EndOfLdesException.class); - client.start(); + client.start(); + verify(eventPublisher).publishEvent(new PipelineStatusEvent(pipelineName, PipelineStatus.RUNNING, StatusChangeSource.MANUAL)); + + await().atMost(Duration.ofSeconds(40)).untilAsserted(() -> { + verify(eventPublisher).publishEvent(new PipelineStatusEvent(pipelineName, PipelineStatus.HALTED, StatusChangeSource.MANUAL)); + verify(eventPublisher).publishEvent(new PipelineShutdownEvent(pipelineName)); + }); + } + + @Test + void when_RuntimeException_StopPipeline() { + doThrow(RuntimeException.class).when(supplier).init(); + + client.start(); await().atMost(Duration.ofSeconds(20)).untilAsserted(() -> { verify(eventPublisher).publishEvent(new PipelineStatusEvent(pipelineName, PipelineStatus.RUNNING, StatusChangeSource.MANUAL)); verify(eventPublisher).publishEvent(new PipelineStatusEvent(pipelineName, PipelineStatus.HALTED, StatusChangeSource.MANUAL)); }); - } + } } \ No newline at end of file From 9ee9e0bf9e1af647be0bde0686210ea1038cbc1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:26:54 +0200 Subject: [PATCH 06/17] build(deps): bump org.apache.maven.plugins:maven-source-plugin from 3.2.1 to 3.3.1 (#574) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 51f82f89b..8695746d5 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ 21 3.5.0 - 3.2.1 + 3.3.1 2.5.2 1.6.13 3.1.2 From 90165caa4ace13cbd730680cf55c11f6393474fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:00:11 +0200 Subject: [PATCH 07/17] chore(deps): bump org.testcontainers:postgresql from 1.19.0 to 1.20.2 (#690) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8695746d5..a606e2ea5 100644 --- a/pom.xml +++ b/pom.xml @@ -86,7 +86,7 @@ 5.5.0 5.5.0 1.19.6 - 1.19.0 + 1.20.2 2.0.7 1.4.11 From 92017b71a4b7d9ce8a7f2fb6663c2232b3c3d29e Mon Sep 17 00:00:00 2001 From: Jonas Bulcke Date: Fri, 4 Oct 2024 15:16:56 +0200 Subject: [PATCH 08/17] Revert "build(deps): bump org.apache.maven.plugins:maven-source-plugin from 3.2.1 to 3.3.1 (#574)" This reverts commit 9ee9e0bf9e1af647be0bde0686210ea1038cbc1b. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a606e2ea5..57c3d154d 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ 21 3.5.0 - 3.3.1 + 3.2.1 2.5.2 1.6.13 3.1.2 From 2b6321022b7bb83c0b166d70cb0640349c772009 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:46:07 +0200 Subject: [PATCH 09/17] chore(deps): bump org.jacoco:jacoco-maven-plugin from 0.8.10 to 0.8.12 (#694) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 57c3d154d..9d3a1bb9e 100644 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,7 @@ 2.0.7 1.4.11 - 0.8.10 + 0.8.12 jacoco reuseReports ${project.basedir}/target/site/jacoco/jacoco.xml From 9dc984cda0f8f2388f7f6390a16c8b364d902c32 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:46:39 +0000 Subject: [PATCH 10/17] chore(deps): bump org.apache.maven.plugins:maven-jar-plugin from 3.3.0 to 3.4.2 (#695) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9d3a1bb9e..08ce598b1 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ 1.6.13 3.1.2 3.0.0-M7 - 3.3.0 + 3.4.2 3.0.1 3.6.0 3.1.0 From 5a5fd3e1037d4d7f760ffbf8b2c46ea9b287ea67 Mon Sep 17 00:00:00 2001 From: Jan Robert <15772440+Yalz@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:22:08 +0200 Subject: [PATCH 11/17] Update deploy-documentation.yml --- .github/workflows/deploy-documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-documentation.yml b/.github/workflows/deploy-documentation.yml index fa54df29b..64a9d1ce7 100644 --- a/.github/workflows/deploy-documentation.yml +++ b/.github/workflows/deploy-documentation.yml @@ -58,6 +58,6 @@ jobs: env: PAT: ${{secrets.DEPLOY_DOCS_PAT}} - run: | - gh-pages-multi deploy --title ${{env.title}} -t ${{steps.version.outputs.version}} -s docs/_site + gh-pages-multi deploy --title "${{env.title}}" -t ${{steps.version.outputs.version}} -s docs/_site env: NODE_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} From cb4684ca541fb4f0c14f8a60e5e5f851631d55e4 Mon Sep 17 00:00:00 2001 From: Jan Robert <15772440+Yalz@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:33:43 +0200 Subject: [PATCH 12/17] ci; Update deploy-documentation.yml --- .github/workflows/deploy-documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-documentation.yml b/.github/workflows/deploy-documentation.yml index 64a9d1ce7..a580ff8fb 100644 --- a/.github/workflows/deploy-documentation.yml +++ b/.github/workflows/deploy-documentation.yml @@ -50,7 +50,7 @@ jobs: with: node-version: 16 registry-url: https://npm.pkg.github.com/ - - run: npm i -g @koumoul/gh-pages-multi + - run: npm i -g @yalz/gh-pages-multi - run: | git config --global user.email "vsds@noreply.com" git config --global user.name "VSDS CI" From 38100f1120e5fcb2273691c78405b83ebccfeb1e Mon Sep 17 00:00:00 2001 From: Jan Robert <15772440+Yalz@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:26:12 +0200 Subject: [PATCH 13/17] ci: Update deploy-documentation.yml --- .github/workflows/deploy-documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-documentation.yml b/.github/workflows/deploy-documentation.yml index a580ff8fb..ee3f261e3 100644 --- a/.github/workflows/deploy-documentation.yml +++ b/.github/workflows/deploy-documentation.yml @@ -6,7 +6,7 @@ name: Build Docs on: push: branches: - - master + - main - develop workflow_dispatch: From f84a3fd4a6a3e9f6d2e6e6eb0a885893ed60a542 Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Thu, 17 Oct 2024 13:09:29 +0200 Subject: [PATCH 14/17] feat: extract props instead of using config (#699) --- docs/_ldio/ldio-inputs/ldio-ldes-client.md | 10 - .../event-stream-properties-fetcher/pom.xml | 16 ++ .../EventStreamPropertiesFetcher.java | 40 ++++ .../StartingNodeSpecificationFactory.java | 22 +++ .../valueobjects/EventStreamProperties.java | 39 ++++ .../valueobjects/PropertiesRequest.java | 24 +++ .../StartingNodeSpecification.java | 5 + .../valueobjects/TreeNodeSpecification.java | 36 ++++ .../valueobjects/ViewSpecification.java | 47 +++++ .../EventStreamPropertiesFetcherTest.java | 84 ++++++++ .../StartingNodeSpecificationFactoryTest.java | 20 ++ .../TreeNodeSpecificationTest.java | 21 ++ .../valueobjects/ViewSpecificationTest.java | 28 +++ .../src/test/resources/models/eventstream.ttl | 59 ++++++ .../src/test/resources/models/treenode.ttl | 13 ++ .../src/test/resources/models/view.ttl | 43 +++++ ldi-core/ldes-client/pom.xml | 1 + .../services/MemberSupplierWrapper.java | 15 ++ .../services/MemberSupplierWrappers.java | 24 +++ .../membersuppliers/MemberSupplierImpl.java | 2 +- .../services/MemberSupplierWrapperTest.java | 55 ++++++ .../services/MemberSupplierWrappersTest.java | 53 ++++++ .../ldes-client-processor/pom.xml | 4 + .../domain/valueobjects/LdesProperties.java | 25 --- .../ldi/processors/LdesClientProcessor.java | 77 ++------ .../config/LdesProcessorProperties.java | 28 --- .../ExactlyOnceMemberSupplierWrapper.java | 40 ++++ .../LatestStateMemberSupplierWrapper.java | 43 +++++ .../MemberSupplierWrappersBuilder.java | 31 +++ ...sionMaterialisedMemberSupplierWrapper.java | 36 ++++ .../ldi/services/LdesPropertiesExtractor.java | 72 ------- .../LdesPropertyNotFoundException.java | 8 - .../processors/LdesClientProcessorTest.java | 11 +- .../ExactlyOnceMemberSupplierWrapperTest.java | 45 +++++ .../LatestStateMemberSupplierWrapperTest.java | 47 +++++ ...MaterialisedMemberSupplierWrapperTest.java | 35 ++++ .../testutils/TestProcessContext.java | 38 ++++ .../200-response-with-indirect-url.json | 9 +- .../LdioLdesClientConnectorAutoConfig.java | 19 +- .../ldio-connectors/ldio-ldes-client/pom.xml | 5 +- .../ldes/ldio/LdioLdesClientProperties.java | 100 +++++++++- .../ldes/ldio/LdioLdesClientPropertyKeys.java | 19 ++ .../ldio/config/LdioLdesClientAutoConfig.java | 43 ----- .../config/LdioLdesClientConfigurator.java | 52 +++++ .../ldio/config/MemberSupplierFactory.java | 180 ++++-------------- .../ExactlyOnceMemberSupplierWrapper.java | 33 ++++ .../LatestStateMemberSupplierWrapper.java | 36 ++++ .../MemberSupplierWrappersBuilder.java | 30 +++ ...sionMaterialisedMemberSupplierWrapper.java | 35 ++++ .../status/ClientStatusConverter.java | 42 ++++ .../ldio/LdioLdesClientPropertiesTest.java | 24 +++ .../ExactlyOnceMemberSupplierWrapperTest.java | 47 +++++ .../LatestStateMemberSupplierWrapperTest.java | 64 +++++++ .../ldio/config/LdioLdesClientITSteps.java | 8 +- .../config/MemberSupplierFactoryTest.java | 49 ++++- ...MaterialisedMemberSupplierWrapperTest.java | 49 +++++ .../src/test/resources/__files/items-ldes.ttl | 11 ++ .../resources/features/ldes-client-in.feature | 2 - .../src/test/resources/mappings/items.json | 15 ++ pom.xml | 5 + 60 files changed, 1649 insertions(+), 425 deletions(-) create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcher.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/services/StartingNodeSpecificationFactory.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/EventStreamProperties.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/PropertiesRequest.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/StartingNodeSpecification.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/TreeNodeSpecification.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/ViewSpecification.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcherTest.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/services/StartingNodeSpecificationFactoryTest.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/valueobjects/TreeNodeSpecificationTest.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/valueobjects/ViewSpecificationTest.java create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/eventstream.ttl create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/treenode.ttl create mode 100644 ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/view.ttl create mode 100644 ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrapper.java create mode 100644 ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrappers.java create mode 100644 ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrapperTest.java create mode 100644 ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrappersTest.java delete mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/domain/valueobjects/LdesProperties.java create mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapper.java create mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapper.java create mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/MemberSupplierWrappersBuilder.java create mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/VersionMaterialisedMemberSupplierWrapper.java delete mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/LdesPropertiesExtractor.java delete mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/LdesPropertyNotFoundException.java create mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapperTest.java create mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapperTest.java create mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/VersionMaterialisedMemberSupplierWrapperTest.java create mode 100644 ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/testutils/TestProcessContext.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientPropertyKeys.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConfigurator.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/ExactlyOnceMemberSupplierWrapper.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/LatestStateMemberSupplierWrapper.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/MemberSupplierWrappersBuilder.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/VersionMaterialisedMemberSupplierWrapper.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/status/ClientStatusConverter.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientPropertiesTest.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/ExactlyOnceMemberSupplierWrapperTest.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LatestStateMemberSupplierWrapperTest.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/VersionMaterialisedMemberSupplierWrapperTest.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-ldes.ttl create mode 100644 ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items.json diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client.md b/docs/_ldio/ldio-inputs/ldio-ldes-client.md index eec191826..1283d05e7 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client.md @@ -32,10 +32,6 @@ within a fragment. When the fragment is marked as immutable, and no members can be added anymore, the LDES Client will stop keeping track of members processed within that fragment. -Members within a fragment can be processed in order of time based on a timestamp. The path to this timestamp needs to be -configured. -If the patch is missing, members will be processed in random order. - ### Filtering #### Exactly-once-filter @@ -96,7 +92,6 @@ CPU ([source](https://www.sqlite.org/faq.html#q19)). | _source-format_ | The 'Content-Type' that should be requested to the server | No | text/turtle | application/n-quads | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | | _state_ | 'memory', 'sqlite' or 'postgres' to indicate how the state should be persisted | No | memory | sqlite | 'memory', 'sqlite' or 'postgres' | | _keep-state_ | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No | false | false | true or false | -| _timestamp-path_ | The property-path used to determine the timestamp on which the members will be ordered, and used for the `latest-state-filter` when enabled | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | A property path | | _enable-exactly-once_ | Indicates whether a member must be sent exactly once or at least once | No | true | true | true or false | {: .note } @@ -113,13 +108,8 @@ api | Property | Description | Required | Default | Example | Supported values | |:--------------------------------------|:--------------------------------------------------------------------------------------|:---------|:-------------------------------------|:-------------------------------------|:-----------------| | _materialisation.enabled_ | Indicates if the client should return state-objects (true) or version-objects (false) | No | false | true | true or false | -| _materialisation.version-of-property_ | Property that points to the versionOfPath | No | http://purl.org/dc/terms/isVersionOf | http://purl.org/dc/terms/isVersionOf | true or false | | _materialisation.enable-latest-state_ | Indicates whether all state or only the latest state must be sent | No | true | false | true or false | -{: .note } -Don't forgot to provide a timestamp-path in the general properties, as this property is not required, but necessary for -this filter to work properly! - {% include ldio-core/http-requester.md %} ### SQLite properties diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml b/ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml new file mode 100644 index 000000000..3969e2ed5 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml @@ -0,0 +1,16 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.ldes.client + ldes-client + 2.10.0-SNAPSHOT + + + event-stream-properties-fetcher + + + + \ No newline at end of file diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcher.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcher.java new file mode 100644 index 000000000..367707fda --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcher.java @@ -0,0 +1,40 @@ +package ldes.client.eventstreamproperties; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import ldes.client.eventstreamproperties.services.StartingNodeSpecificationFactory; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.eventstreamproperties.valueobjects.PropertiesRequest; +import ldes.client.eventstreamproperties.valueobjects.StartingNodeSpecification; +import org.apache.jena.riot.RDFParser; + +import java.io.ByteArrayInputStream; + +public class EventStreamPropertiesFetcher { + private final RequestExecutor requestExecutor; + + public EventStreamPropertiesFetcher(RequestExecutor requestExecutor) { + this.requestExecutor = requestExecutor; + } + + public EventStreamProperties fetchEventStreamProperties(PropertiesRequest request) { + final EventStreamProperties eventStreamProperties = executePropertiesRequest(request); + + if(eventStreamProperties.containsRequiredProperties()) { + return eventStreamProperties; + } + + return executePropertiesRequest(request.withUrl(eventStreamProperties.getUri())); + + } + + private EventStreamProperties executePropertiesRequest(PropertiesRequest request) { + return requestExecutor.execute(request.createRequest()).getBody() + .map(ByteArrayInputStream::new) + .map(body -> RDFParser.source(body).lang(request.lang()).toModel()) + .map(StartingNodeSpecificationFactory::fromModel) + .map(StartingNodeSpecification::extractEventStreamProperties) + .orElseThrow(); + } + + +} diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/services/StartingNodeSpecificationFactory.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/services/StartingNodeSpecificationFactory.java new file mode 100644 index 000000000..f7f430551 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/services/StartingNodeSpecificationFactory.java @@ -0,0 +1,22 @@ +package ldes.client.eventstreamproperties.services; + +import ldes.client.eventstreamproperties.valueobjects.StartingNodeSpecification; +import ldes.client.eventstreamproperties.valueobjects.TreeNodeSpecification; +import ldes.client.eventstreamproperties.valueobjects.ViewSpecification; +import org.apache.jena.rdf.model.Model; + +public class StartingNodeSpecificationFactory { + private StartingNodeSpecificationFactory() { + } + + public static StartingNodeSpecification fromModel(Model model) { + if (TreeNodeSpecification.isTreeNode(model)) { + return new TreeNodeSpecification(model); + } + if (ViewSpecification.isViewSpecification(model)) { + return new ViewSpecification(model); + } + throw new IllegalStateException("The provided starting node must contain either a dcterms:isPartOf property or the ldes:versionOfPath and ldes:timestampPath properties"); + } + +} diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/EventStreamProperties.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/EventStreamProperties.java new file mode 100644 index 000000000..ca12dc767 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/EventStreamProperties.java @@ -0,0 +1,39 @@ +package ldes.client.eventstreamproperties.valueobjects; + +public final class EventStreamProperties { + private final String uri; + private final String versionOfPath; + private final String timestampPath; + private final String shaclShapeUri; + + public EventStreamProperties(String uri) { + this(uri, null, null, null); + } + + public EventStreamProperties(String uri, String versionOfPath, String timestampPath, String shaclShapeUri) { + this.uri = uri; + this.versionOfPath = versionOfPath; + this.timestampPath = timestampPath; + this.shaclShapeUri = shaclShapeUri; + } + + public String getUri() { + return uri; + } + + public String getVersionOfPath() { + return versionOfPath; + } + + public String getTimestampPath() { + return timestampPath; + } + + public String getShaclShapeUri() { + return shaclShapeUri; + } + + public boolean containsRequiredProperties() { + return timestampPath != null && versionOfPath != null; + } +} diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/PropertiesRequest.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/PropertiesRequest.java new file mode 100644 index 000000000..c04cfc406 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/PropertiesRequest.java @@ -0,0 +1,24 @@ +package ldes.client.eventstreamproperties.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.GetRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Request; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeader; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeaders; +import org.apache.http.HttpHeaders; +import org.apache.jena.riot.Lang; + +import java.util.List; + +public record PropertiesRequest(String url, Lang lang) { + + public Request createRequest() { + RequestHeaders requestHeaders = new RequestHeaders(List.of( + new RequestHeader(HttpHeaders.ACCEPT, lang.getHeaderString()) + )); + return new GetRequest(url, requestHeaders); + } + + public PropertiesRequest withUrl(String url) { + return new PropertiesRequest(url, lang); + } +} diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/StartingNodeSpecification.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/StartingNodeSpecification.java new file mode 100644 index 000000000..a1d7f8537 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/StartingNodeSpecification.java @@ -0,0 +1,5 @@ +package ldes.client.eventstreamproperties.valueobjects; + +public interface StartingNodeSpecification { + EventStreamProperties extractEventStreamProperties(); +} diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/TreeNodeSpecification.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/TreeNodeSpecification.java new file mode 100644 index 000000000..00d9c80e0 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/TreeNodeSpecification.java @@ -0,0 +1,36 @@ +package ldes.client.eventstreamproperties.valueobjects; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.RDFNode; + +import java.util.Optional; + +import static org.apache.jena.rdf.model.ResourceFactory.createProperty; + +public class TreeNodeSpecification implements StartingNodeSpecification { + public static final String DC_TERMS = "http://purl.org/dc/terms/"; + public static final Property IS_PART_OF = createProperty(DC_TERMS, "isPartOf"); + private final Model model; + + public TreeNodeSpecification(Model model) { + this.model = model; + } + + @Override + public EventStreamProperties extractEventStreamProperties() { + return extractTreeNode(model).filter(RDFNode::isURIResource) + .map(RDFNode::asResource) + .map(node -> new EventStreamProperties(node.getURI())) + .orElseThrow(); + } + + public static boolean isTreeNode(Model model) { + return extractTreeNode(model).isPresent(); + } + + private static Optional extractTreeNode(Model model) { + return model.listObjectsOfProperty(IS_PART_OF).nextOptional(); + } + +} diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/ViewSpecification.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/ViewSpecification.java new file mode 100644 index 000000000..6f75ae1e8 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/ViewSpecification.java @@ -0,0 +1,47 @@ +package ldes.client.eventstreamproperties.valueobjects; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.Resource; + +import java.util.Optional; + +import static org.apache.jena.rdf.model.ResourceFactory.createProperty; + +public class ViewSpecification implements StartingNodeSpecification { + public static final String LDES = "https://w3id.org/ldes#"; + public static final Property LDES_EVENT_STREAM = createProperty(LDES, "EventStream"); + public static final String RDF_SYNTAX = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; + public static final Property RDF_SYNTAX_TYPE = createProperty(RDF_SYNTAX, "type"); + public static final Property TREE_SHAPE = createProperty("https://w3id.org/tree#", "shape"); + public static final Property LDES_VERSION_OF_PATH = createProperty(LDES, "versionOfPath"); + public static final Property LDES_TIMESTAMP_PATH = createProperty(LDES, "timestampPath"); + + private final Model model; + + public ViewSpecification(Model model) { + this.model = model; + } + + @Override + public EventStreamProperties extractEventStreamProperties() { + final Resource subject = extractEventStream(model).orElseThrow(); + final String shapeUri = Optional.ofNullable(subject.getPropertyResourceValue(TREE_SHAPE)) + .map(Resource::getURI) + .orElse(""); + return new EventStreamProperties( + subject.getURI(), + subject.getPropertyResourceValue(LDES_VERSION_OF_PATH).getURI(), + subject.getPropertyResourceValue(LDES_TIMESTAMP_PATH).getURI(), + shapeUri + ); + } + + public static boolean isViewSpecification(Model model) { + return extractEventStream(model).isPresent(); + } + + private static Optional extractEventStream(Model model) { + return model.listSubjectsWithProperty(RDF_SYNTAX_TYPE, LDES_EVENT_STREAM).nextOptional(); + } +} diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcherTest.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcherTest.java new file mode 100644 index 000000000..55f2bd276 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcherTest.java @@ -0,0 +1,84 @@ +package ldes.client.eventstreamproperties; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.eventstreamproperties.valueobjects.PropertiesRequest; +import org.apache.jena.riot.Lang; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.assertj.core.api.Assertions.assertThat; + +@WireMockTest(httpPort = EventStreamPropertiesFetcherTest.WIREMOCK_PORT) +class EventStreamPropertiesFetcherTest { + public static final int WIREMOCK_PORT = 12121; + private static final EventStreamProperties eventStreamProperties = new EventStreamProperties( + "http://localhost:12121/observations", + "http://purl.org/dc/terms/isVersionOf", + "http://www.w3.org/ns/prov#generatedAtTime", + "" + ); + private EventStreamPropertiesFetcher fetcher; + + @BeforeEach + void setUp() { + RequestExecutor requestExecutor = new RequestExecutorFactory().createNoAuthExecutor(); + fetcher = new EventStreamPropertiesFetcher(requestExecutor); + } + + @Test + void given_EventStream_when_FetchProperties_then_ReturnValidProperties() throws IOException, URISyntaxException { + URL resource = getClass().getClassLoader().getResource("models/eventstream.ttl"); + final byte[] responseBytes = Files.readAllBytes(Path.of(Objects.requireNonNull(resource).toURI())); + stubFor(get("/observations").willReturn(ok().withBody(responseBytes))); + + final EventStreamProperties properties = fetcher.fetchEventStreamProperties(new PropertiesRequest("http://localhost:12121/observations", Lang.TTL)); + + verify(getRequestedFor(urlEqualTo("/observations"))); + assertThat(properties) + .usingRecursiveComparison() + .isEqualTo(eventStreamProperties); + } + + @Test + void given_View_when_FetchProperties_then_ReturnValidProperties() throws IOException, URISyntaxException { + URL resource = getClass().getClassLoader().getResource("models/view.ttl"); + final byte[] responseBytes = Files.readAllBytes(Path.of(Objects.requireNonNull(resource).toURI())); + stubFor(get("/observations/by-page").willReturn(ok().withBody(responseBytes))); + + final EventStreamProperties properties = fetcher.fetchEventStreamProperties(new PropertiesRequest("http://localhost:12121/observations/by-page", Lang.TTL)); + + verify(getRequestedFor(urlEqualTo("/observations/by-page"))); + assertThat(properties) + .usingRecursiveComparison() + .isEqualTo(eventStreamProperties); + } + + @Test + void given_Fragment_when_FetchProperties_then_ReturnValidProperties() throws IOException, URISyntaxException { + URL treeNodeResource = getClass().getClassLoader().getResource("models/treenode.ttl"); + final byte[] treeNodeBody = Files.readAllBytes(Path.of(Objects.requireNonNull(treeNodeResource).toURI())); + stubFor(get("/observations/by-page?pageNumber=1").willReturn(ok().withBody(treeNodeBody))); + URL eventSourceResource = getClass().getClassLoader().getResource("models/eventstream.ttl"); + final byte[] eventStreamBody = Files.readAllBytes(Path.of(Objects.requireNonNull(eventSourceResource).toURI())); + stubFor(get("/observations").willReturn(ok().withBody(eventStreamBody))); + + final EventStreamProperties properties = fetcher.fetchEventStreamProperties(new PropertiesRequest("http://localhost:12121/observations/by-page?pageNumber=1", Lang.TTL)); + + verify(getRequestedFor(urlEqualTo("/observations/by-page?pageNumber=1"))); + verify(getRequestedFor(urlEqualTo("/observations"))); + assertThat(properties) + .usingRecursiveComparison() + .isEqualTo(eventStreamProperties); + } +} \ No newline at end of file diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/services/StartingNodeSpecificationFactoryTest.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/services/StartingNodeSpecificationFactoryTest.java new file mode 100644 index 000000000..8fd0a692c --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/services/StartingNodeSpecificationFactoryTest.java @@ -0,0 +1,20 @@ +package ldes.client.eventstreamproperties.services; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParser; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class StartingNodeSpecificationFactoryTest { + @Test + void given_invalidModel_when_FromModel_then_ThrowException() { + String invalid = " ."; + Model model = RDFParser.fromString(invalid).lang(Lang.NQ).toModel(); + + assertThatThrownBy(() -> StartingNodeSpecificationFactory.fromModel(model)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("The provided starting node must contain either a dcterms:isPartOf property or the ldes:versionOfPath and ldes:timestampPath properties"); + } +} \ No newline at end of file diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/valueobjects/TreeNodeSpecificationTest.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/valueobjects/TreeNodeSpecificationTest.java new file mode 100644 index 000000000..aa2365a42 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/valueobjects/TreeNodeSpecificationTest.java @@ -0,0 +1,21 @@ +package ldes.client.eventstreamproperties.valueobjects; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.RDFParser; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class TreeNodeSpecificationTest { + @Test + void test_ExtractEventStreamProperties() { + final EventStreamProperties expectedESProperties = new EventStreamProperties("http://localhost:12121/observations"); + final Model model = RDFParser.source("models/treenode.ttl").toModel(); + + final EventStreamProperties eventStreamProperties = new TreeNodeSpecification(model).extractEventStreamProperties(); + + assertThat(eventStreamProperties) + .usingRecursiveComparison() + .isEqualTo(expectedESProperties); + } +} \ No newline at end of file diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/valueobjects/ViewSpecificationTest.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/valueobjects/ViewSpecificationTest.java new file mode 100644 index 000000000..f680e28fb --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/valueobjects/ViewSpecificationTest.java @@ -0,0 +1,28 @@ +package ldes.client.eventstreamproperties.valueobjects; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.RDFParser; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; + +class ViewSpecificationTest { + @ParameterizedTest + @ValueSource(strings = {"models/view.ttl", "models/eventstream.ttl"}) + void test_ExtractEventStreamProperties(String fileUri) { + final EventStreamProperties expectedESProperties = new EventStreamProperties( + "http://localhost:12121/observations", + "http://purl.org/dc/terms/isVersionOf", + "http://www.w3.org/ns/prov#generatedAtTime", + "" + ); + final Model model = RDFParser.source(fileUri).toModel(); + + final EventStreamProperties eventStreamProperties = new ViewSpecification(model).extractEventStreamProperties(); + + assertThat(eventStreamProperties) + .usingRecursiveComparison() + .isEqualTo(expectedESProperties); + } +} \ No newline at end of file diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/eventstream.ttl b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/eventstream.ttl new file mode 100644 index 000000000..1485bf810 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/eventstream.ttl @@ -0,0 +1,59 @@ +@prefix by-location: . +@prefix by-page: . +@prefix dc: . +@prefix dcat: . +@prefix ldes: . +@prefix observations: . +@prefix prov: . +@prefix rdf: . +@prefix rdfs: . +@prefix shacl: . +@prefix skos: . +@prefix terms: . +@prefix tree: . +@prefix vsds-verkeersmetingen: . + +observations:by-location + rdf:type tree:Node , rdfs:Resource ; + tree:viewDescription by-location:description . + + + rdf:type dc:Standard . + +by-location:description + rdf:type dcat:DataService , tree:ViewDescription ; + ldes:retentionPolicy [ rdf:type ldes:DurationAgoPolicy ; + tree:value "P2Y"^^ + ] ; + tree:fragmentationStrategy ( [ rdf:type tree:GeospatialFragmentation ; + tree:fragmentationPath "http://www.opengis.net/ont/geosparql#asWKT" ; + tree:maxZoom "14" + ] + ) ; + tree:pageSize "250"^^ . + + + +by-page:description rdf:type dcat:DataService , tree:ViewDescription ; + ldes:retentionPolicy [ rdf:type ldes:DurationAgoPolicy ; + tree:value "P2Y"^^ + ] ; + tree:fragmentationStrategy () ; + tree:pageSize "250"^^ . + + + + rdf:type dc:Standard . + + + rdf:type ldes:EventStream ; + ldes:timestampPath prov:generatedAtTime ; + ldes:versionOfPath dc:isVersionOf ; + tree:shape [ rdf:type shacl:NodeShape ; + shacl:targetClass vsds-verkeersmetingen:Verkeerstelling + ] ; + tree:view observations:by-page , observations:by-location . + + +observations:by-page rdf:type tree:Node , rdfs:Resource ; + tree:viewDescription by-page:description . diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/treenode.ttl b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/treenode.ttl new file mode 100644 index 000000000..1c8983b76 --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/treenode.ttl @@ -0,0 +1,13 @@ +@prefix ldes: . +@prefix prov: . +@prefix rdf: . +@prefix terms: . +@prefix tree: . + + + + rdf:type tree:Node ; + terms:isPartOf ; + tree:relation [ rdf:type tree:Relation ; + tree:node + ] . diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/view.ttl b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/view.ttl new file mode 100644 index 000000000..2ec41535c --- /dev/null +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/resources/models/view.ttl @@ -0,0 +1,43 @@ +@prefix GDI-Vlaanderen-Trefwoorden: . +@prefix access-right: . +@prefix dcat: . +@prefix ldes: . +@prefix metadata-dcat: . +@prefix observations: . +@prefix prov: . +@prefix rdf: . +@prefix rdfs: . +@prefix shacl: . +@prefix terms: . +@prefix tree: . +@prefix vsds-verkeersmetingen: . + + + rdf:type dcat:DataService ; + metadata-dcat:statuut GDI-Vlaanderen-Trefwoorden:VLOPENDATASERVICE ; + terms:accessRights access-right:PUBLIC ; + terms:description "Default paginated view on the traffic counts in Flanders"@en , "Default gepagineerde view van de verkeerstellingen in Vlaanderen"@nl ; + terms:identifier "http://localhost:12121/observations/by-page"^^rdfs:Literal ; + terms:title "Traffic Count in Flanders by page"@en , "Verkeerstellingen in Vlaanderen per pagina"@nl ; + dcat:contactPoint ; + dcat:endpointDescription ; + dcat:endpointURL observations:by-page ; + dcat:servesDataset . + + + rdf:type ldes:EventStream ; + ldes:timestampPath prov:generatedAtTime ; + ldes:versionOfPath terms:isVersionOf ; + tree:view observations:by-page . + +observations:by-page rdf:type rdfs:Resource , tree:Node ; + tree:relation [ rdf:type tree:Relation ; + tree:node + ] . + +[ rdf:type shacl:NodeShape ; + shacl:targetClass vsds-verkeersmetingen:Verkeerstelling +] . + + + rdf:type rdfs:Resource . \ No newline at end of file diff --git a/ldi-core/ldes-client/pom.xml b/ldi-core/ldes-client/pom.xml index 590948eb5..538931924 100644 --- a/ldi-core/ldes-client/pom.xml +++ b/ldi-core/ldes-client/pom.xml @@ -18,6 +18,7 @@ tree-node-fetcher tree-node-supplier tree-node-relations-fetcher + event-stream-properties-fetcher diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrapper.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrapper.java new file mode 100644 index 000000000..5b61c2c20 --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrapper.java @@ -0,0 +1,15 @@ +package ldes.client.treenodesupplier.domain.services; + +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; + +public abstract class MemberSupplierWrapper { + public final MemberSupplier wrapMemberSupplier(MemberSupplier memberSupplier) { + return shouldBeWrapped() + ? createWrappedMemberSupplier(memberSupplier) + : memberSupplier; + } + + protected abstract boolean shouldBeWrapped(); + + protected abstract MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSupplier); +} diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrappers.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrappers.java new file mode 100644 index 000000000..778bc56ff --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrappers.java @@ -0,0 +1,24 @@ +package ldes.client.treenodesupplier.domain.services; + +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; + +import java.util.List; + +public class MemberSupplierWrappers { + private final List wrappers; + + public MemberSupplierWrappers(List wrappers) { + this.wrappers = wrappers; + } + + public MemberSupplier wrapMemberSupplier(MemberSupplier memberSupplier) { + for (MemberSupplierWrapper wrapper : wrappers) { + memberSupplier = wrapper.wrapMemberSupplier(memberSupplier); + } + return memberSupplier; + } + + public interface Builder { + MemberSupplierWrappers build(); + } +} diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierImpl.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierImpl.java index 84f3f4575..a369f1acf 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierImpl.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/membersuppliers/MemberSupplierImpl.java @@ -29,7 +29,7 @@ public SuppliedMember get() { @Override public void destroyState() { - if (!keepState) { + if (!keepState && treeNodeProcessor != null) { treeNodeProcessor.destroyState(); } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrapperTest.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrapperTest.java new file mode 100644 index 000000000..4ba404342 --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrapperTest.java @@ -0,0 +1,55 @@ +package ldes.client.treenodesupplier.domain.services; + +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +@ExtendWith(MockitoExtension.class) +class MemberSupplierWrapperTest { + private MemberSupplierWrapper wrapper; + + + @Test + void given_ShouldBeWrapped_when_Wrap_then_ReturnWrappedMemberSupplier() { + final MemberSupplier base = mock(); + wrapper = new TestMemberSupplierWrapper(true, mock()); + + final var result = wrapper.wrapMemberSupplier(base); + + assertThat(result).isNotSameAs(base); + } + + @Test + void given_ShouldNotBeWrapped_when_Wrap_then_ReturnBaseMemberSupplier() { + final MemberSupplier base = mock(); + wrapper = new TestMemberSupplierWrapper(false, mock()); + + final var result = wrapper.wrapMemberSupplier(base); + + assertThat(result).isSameAs(base); + } + + private static class TestMemberSupplierWrapper extends MemberSupplierWrapper { + private final boolean shouldBeWrapped; + private final MemberSupplier nextMemberSupplier; + + private TestMemberSupplierWrapper(boolean shouldBeWrapped, MemberSupplier nextMemberSupplier) { + this.shouldBeWrapped = shouldBeWrapped; + this.nextMemberSupplier = nextMemberSupplier; + } + + @Override + protected boolean shouldBeWrapped() { + return shouldBeWrapped; + } + + @Override + protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSupplier) { + return nextMemberSupplier; + } + } +} \ No newline at end of file diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrappersTest.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrappersTest.java new file mode 100644 index 000000000..34905c03a --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/services/MemberSupplierWrappersTest.java @@ -0,0 +1,53 @@ +package ldes.client.treenodesupplier.domain.services; + +import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplierImpl; +import ldes.client.treenodesupplier.membersuppliers.VersionMaterialisedMemberSupplier; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class MemberSupplierWrappersTest { + private final MemberSupplier baseMemberSupplier = new MemberSupplierImpl(mock(), false); + private final MemberSupplier firstWrapped = new FilteredMemberSupplier(baseMemberSupplier, mock()); + private final MemberSupplier secondWrapped = new VersionMaterialisedMemberSupplier(firstWrapped, mock()); + @Mock + private MemberSupplierWrapper mockedMemberSupplierWrapper; + + private MemberSupplierWrappers wrappers; + + @BeforeEach + void setUp() { + wrappers = new MemberSupplierWrappers(List.of( + mockedMemberSupplierWrapper, + mockedMemberSupplierWrapper, + mockedMemberSupplierWrapper + )); + } + + @Test + void name() { + when(mockedMemberSupplierWrapper.wrapMemberSupplier(any())) + .thenReturn(firstWrapped) + .thenReturn(firstWrapped) + .thenReturn(secondWrapped); + + final MemberSupplier result = wrappers.wrapMemberSupplier(baseMemberSupplier); + + assertThat(result).isSameAs(secondWrapped); + verify(mockedMemberSupplierWrapper).wrapMemberSupplier(baseMemberSupplier); + verify(mockedMemberSupplierWrapper, times(2)).wrapMemberSupplier(firstWrapped); + verify(mockedMemberSupplierWrapper, never()).wrapMemberSupplier(secondWrapped); + } + +} \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index 0634f8e20..edf8abf1c 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -22,6 +22,10 @@ ldi-common ${project.version} + + be.vlaanderen.informatievlaanderen.ldes.client + event-stream-properties-fetcher + com.github.tomakehurst wiremock-jre8 diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/domain/valueobjects/LdesProperties.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/domain/valueobjects/LdesProperties.java deleted file mode 100644 index d41206849..000000000 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/domain/valueobjects/LdesProperties.java +++ /dev/null @@ -1,25 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.domain.valueobjects; - -public class LdesProperties { - private final String timestampPath; - private final String versionOfPath; - private final String shape; - - public LdesProperties(String timestampPath, String versionOfPath, String shape) { - this.timestampPath = timestampPath; - this.versionOfPath = versionOfPath; - this.shape = shape; - } - - public String getTimestampPath() { - return timestampPath; - } - - public String getVersionOfPath() { - return versionOfPath; - } - - public String getShape() { - return shape; - } -} diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java index ffe7745e2..d2ab73f7f 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java @@ -1,26 +1,22 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; -import be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser; -import be.vlaanderen.informatievlaanderen.ldes.ldi.domain.valueobjects.LdesProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers.MemberSupplierWrappersBuilder; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.retry.RetryConfig; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorDecorator; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.services.LdesPropertiesExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; import io.github.resilience4j.retry.Retry; +import ldes.client.eventstreamproperties.EventStreamPropertiesFetcher; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.eventstreamproperties.valueobjects.PropertiesRequest; import ldes.client.treenodesupplier.TreeNodeProcessor; import ldes.client.treenodesupplier.domain.valueobject.*; -import ldes.client.treenodesupplier.filters.ExactlyOnceFilter; -import ldes.client.treenodesupplier.filters.LatestStateFilter; -import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; import ldes.client.treenodesupplier.membersuppliers.MemberSupplierImpl; -import ldes.client.treenodesupplier.membersuppliers.VersionMaterialisedMemberSupplier; import org.apache.http.Header; import org.apache.http.message.BasicHeader; import org.apache.jena.rdf.model.Model; @@ -60,11 +56,9 @@ public class LdesClientProcessor extends AbstractProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(LdesClientProcessor.class); private MemberSupplier memberSupplier; - private LdesProperties ldesProperties; + private EventStreamProperties eventStreamProperties; private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); - private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory(); private boolean hasLdesEnded; - private boolean keepState; @Override public Set getRelationships() { @@ -78,10 +72,8 @@ public final List getSupportedPropertyDescriptors() { DATA_DESTINATION_FORMAT, STATE_PERSISTENCE_STRATEGY, KEEP_STATE, - TIMESTAMP_PATH, USE_EXACTLY_ONCE_FILTER, USE_VERSION_MATERIALISATION, - VERSION_OF_PROPERTY, USE_LATEST_STATE_FILTER, AUTHORIZATION_STRATEGY, API_KEY_PROPERTY, @@ -109,45 +101,22 @@ public void onScheduled(final ProcessContext context) { Lang dataSourceFormat = LdesProcessorProperties.getDataSourceFormat(context); final RequestExecutor requestExecutor = getRequestExecutorWithPossibleRetry(context); LdesMetaData ldesMetaData = new LdesMetaData(dataSourceUrls, dataSourceFormat); - StatePersistence statePersistence = statePersistenceFactory.getStatePersistence(context); - String timestampPath = LdesProcessorProperties.getTimestampPath(context); - TimestampExtractor timestampExtractor = timestampPath.isBlank() ? new TimestampFromCurrentTimeExtractor() : - new TimestampFromPathExtractor(createProperty(timestampPath)); + StatePersistence statePersistence = new StatePersistenceFactory().getStatePersistence(context); + eventStreamProperties = fetchEventStreamProperties(ldesMetaData, requestExecutor); + TimestampExtractor timestampExtractor = new TimestampFromPathExtractor(createProperty(eventStreamProperties.getTimestampPath())); TreeNodeProcessor treeNodeProcessor = new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor, timestampExtractor, clientStatusConsumer()); - keepState = stateKept(context); - final MemberSupplierImpl baseMemberSupplier = new MemberSupplierImpl(treeNodeProcessor, keepState); - - if (useVersionMaterialisation(context)) { - final var versionOfProperty = createProperty(getVersionOfProperty(context)); - final var versionMaterialiser = new VersionMaterialiser(versionOfProperty, restrictToMembers(context)); - MemberSupplier decoratedMemberSupplier = useLatestStateFilter(context) - ? new FilteredMemberSupplier(baseMemberSupplier, getLatestStateFilter(context, statePersistence)) - : baseMemberSupplier; - memberSupplier = new VersionMaterialisedMemberSupplier( - decoratedMemberSupplier, - versionMaterialiser - ); - } else if (useExactlyOnceFilter(context)) { - memberSupplier = new FilteredMemberSupplier( - baseMemberSupplier, - new ExactlyOnceFilter(statePersistence.getMemberIdRepository(), keepState) - ); - } else { - memberSupplier = baseMemberSupplier; - } - + final MemberSupplier baseMemberSupplier = new MemberSupplierImpl(treeNodeProcessor, stateKept(context)); + memberSupplier = new MemberSupplierWrappersBuilder() + .withContext(context) + .withEventStreamProperties(eventStreamProperties) + .build() + .wrapMemberSupplier(baseMemberSupplier); memberSupplier.init(); - determineLdesProperties(ldesMetaData, requestExecutor, context); - LOGGER.info("LDES Client processor {} configured to follow (sub)streams {} (expected LDES source format: {})", context.getName(), dataSourceUrls, dataSourceFormat); } - private LatestStateFilter getLatestStateFilter(ProcessContext context, StatePersistence statePersistence) { - return new LatestStateFilter(statePersistence.getMemberVersionRepository(), keepState, getTimestampPath(context), getVersionOfProperty(context)); - } - private RequestExecutor getRequestExecutorWithPossibleRetry(final ProcessContext context) { return RequestExecutorDecorator.decorate(getRequestExecutor(context)).with(getRetry(context)).get(); } @@ -175,13 +144,9 @@ private RequestExecutor getRequestExecutor(final ProcessContext context) { }; } - private void determineLdesProperties(LdesMetaData ldesMetaData, RequestExecutor requestExecutor, - ProcessContext context) { - boolean timestampPath = streamTimestampPathProperty(context); - boolean versionOfPath = streamVersionOfProperty(context); - boolean shape = streamShapeProperty(context); - ldesProperties = new LdesPropertiesExtractor(requestExecutor).getLdesProperties(ldesMetaData, timestampPath, - versionOfPath, shape); + private EventStreamProperties fetchEventStreamProperties(LdesMetaData ldesMetaData, RequestExecutor requestExecutor) { + PropertiesRequest request = new PropertiesRequest(ldesMetaData.getStartingNodeUrl(), ldesMetaData.getLang()); + return new EventStreamPropertiesFetcher(requestExecutor).fetchEventStreamProperties(request); } @Override @@ -205,13 +170,13 @@ private void processNextMember(ProcessContext context, ProcessSession session) { FlowFile flowFile = session.create(); if (streamTimestampPathProperty(context)) { - session.putAttribute(flowFile, "ldes.timestamppath", ldesProperties.getTimestampPath()); + session.putAttribute(flowFile, "ldes.timestamppath", eventStreamProperties.getTimestampPath()); } if (streamVersionOfProperty(context)) { - session.putAttribute(flowFile, "ldes.isversionofpath", ldesProperties.getVersionOfPath()); + session.putAttribute(flowFile, "ldes.isversionofpath", eventStreamProperties.getVersionOfPath()); } if (streamShapeProperty(context)) { - session.putAttribute(flowFile, "ldes.shacleshapes", ldesProperties.getShape()); + session.putAttribute(flowFile, "ldes.shacleshapes", eventStreamProperties.getShaclShapeUri()); } Lang dataDestinationFormat = LdesProcessorProperties.getDataDestinationFormat(context); FlowManager.sendRDFToRelation(session, flowFile, @@ -221,7 +186,7 @@ private void processNextMember(ProcessContext context, ProcessSession session) { @OnRemoved public void onRemoved() { - if (!keepState && memberSupplier != null) { + if (memberSupplier != null) { memberSupplier.destroyState(); } } diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java index ee6a2bd8f..32a86e299 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java @@ -5,7 +5,6 @@ import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFLanguages; import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.Validator; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.util.StandardValidators; @@ -47,15 +46,6 @@ private LdesProcessorProperties() { .defaultValue(DEFAULT_DATA_SOURCE_FORMAT.getHeaderString()) .build(); - public static final PropertyDescriptor TIMESTAMP_PATH = new PropertyDescriptor.Builder() - .name("TIMESTAMP_PATH") - .displayName("Timestamp path") - .description("Property path determining the timestamp used to order the members within a fragment") - .required(false) - .addValidator(Validator.VALID) - .defaultValue("http://www.w3.org/ns/prov#generatedAtTime") - .build(); - public static final PropertyDescriptor STREAM_TIMESTAMP_PATH_PROPERTY = new PropertyDescriptor.Builder() .name("STREAM_TIMESTAMP_PATH_PROPERTY") .displayName("Stream timestamp path property") @@ -190,16 +180,6 @@ private LdesProcessorProperties() { .addValidator(StandardValidators.BOOLEAN_VALIDATOR) .build(); - public static final PropertyDescriptor VERSION_OF_PROPERTY = new PropertyDescriptor.Builder() - .name("VERSION_OF_PROPERTY") - .displayName("Version of property") - .description("Property that points to the versionOfPath") - .required(true) - .defaultValue("http://purl.org/dc/terms/isVersionOf") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .addValidator(StandardValidators.URI_VALIDATOR) - .build(); - public static final PropertyDescriptor USE_EXACTLY_ONCE_FILTER = new PropertyDescriptor.Builder() .name("USE_EXACTLY_ONCE_FILTER") .displayName("Use exactly once filter") @@ -238,10 +218,6 @@ public static Lang getDataDestinationFormat(final ProcessContext context) { return RDFLanguages.nameToLang(context.getProperty(DATA_DESTINATION_FORMAT).getValue()); } - public static String getTimestampPath(final ProcessContext context) { - return context.getProperty(TIMESTAMP_PATH).getValue(); - } - public static boolean streamTimestampPathProperty(final ProcessContext context) { return TRUE.equals(context.getProperty(STREAM_TIMESTAMP_PATH_PROPERTY).asBoolean()); } @@ -318,10 +294,6 @@ public static boolean restrictToMembers(final ProcessContext context) { return TRUE.equals(context.getProperty(RESTRICT_TO_MEMBERS).asBoolean()); } - public static String getVersionOfProperty(final ProcessContext context) { - return context.getProperty(VERSION_OF_PROPERTY).getValue(); - } - private static boolean isValidUrl(String url) { try { new URI(url); diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapper.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapper.java new file mode 100644 index 000000000..7a6e8599f --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapper.java @@ -0,0 +1,40 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.StatePersistenceFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties; +import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; +import ldes.client.treenodesupplier.filters.ExactlyOnceFilter; +import ldes.client.treenodesupplier.filters.MemberFilter; +import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import ldes.client.treenodesupplier.repository.MemberIdRepository; +import org.apache.nifi.processor.ProcessContext; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties.useExactlyOnceFilter; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties.useVersionMaterialisation; + +public class ExactlyOnceMemberSupplierWrapper extends MemberSupplierWrapper { + private final ProcessContext context; + + public ExactlyOnceMemberSupplierWrapper(ProcessContext context) { + this.context = context; + } + + @Override + protected boolean shouldBeWrapped() { + return !useVersionMaterialisation(context) && useExactlyOnceFilter(context); + } + + @Override + protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSupplier) { + return new FilteredMemberSupplier(memberSupplier, createMemberFilter()); + } + + private MemberFilter createMemberFilter() { + final MemberIdRepository memberIdRepository = new StatePersistenceFactory() + .getStatePersistence(context) + .getMemberIdRepository(); + final boolean keepState = PersistenceProperties.stateKept(context); + return new ExactlyOnceFilter(memberIdRepository, keepState); + } +} diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapper.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapper.java new file mode 100644 index 000000000..dbbeeb675 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapper.java @@ -0,0 +1,43 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.StatePersistenceFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; +import ldes.client.treenodesupplier.filters.LatestStateFilter; +import ldes.client.treenodesupplier.filters.MemberFilter; +import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import ldes.client.treenodesupplier.repository.MemberVersionRepository; +import org.apache.nifi.processor.ProcessContext; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties.useLatestStateFilter; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties.useVersionMaterialisation; + +public class LatestStateMemberSupplierWrapper extends MemberSupplierWrapper { + private final ProcessContext context; + private final EventStreamProperties eventStreamProperties; + + public LatestStateMemberSupplierWrapper(ProcessContext context, EventStreamProperties eventStreamProperties) { + this.context = context; + this.eventStreamProperties = eventStreamProperties; + } + + @Override + protected boolean shouldBeWrapped() { + return useVersionMaterialisation(context) && useLatestStateFilter(context); + } + + @Override + protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSupplier) { + return new FilteredMemberSupplier(memberSupplier, createMemberFilter()); + } + + private MemberFilter createMemberFilter() { + final MemberVersionRepository memberVersionRepository = new StatePersistenceFactory() + .getStatePersistence(context) + .getMemberVersionRepository(); + final boolean keepState = PersistenceProperties.stateKept(context); + return new LatestStateFilter(memberVersionRepository, keepState, eventStreamProperties.getTimestampPath(), eventStreamProperties.getVersionOfPath()); + } +} diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/MemberSupplierWrappersBuilder.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/MemberSupplierWrappersBuilder.java new file mode 100644 index 000000000..3b440717a --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/MemberSupplierWrappersBuilder.java @@ -0,0 +1,31 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers; + +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.treenodesupplier.domain.services.MemberSupplierWrappers; +import org.apache.nifi.processor.ProcessContext; + +import java.util.List; + +public class MemberSupplierWrappersBuilder implements MemberSupplierWrappers.Builder { + private ProcessContext context; + private EventStreamProperties eventStreamProperties; + + public MemberSupplierWrappersBuilder withContext(ProcessContext context) { + this.context = context; + return this; + } + + public MemberSupplierWrappersBuilder withEventStreamProperties(EventStreamProperties eventStreamProperties) { + this.eventStreamProperties = eventStreamProperties; + return this; + } + + @Override + public MemberSupplierWrappers build() { + return new MemberSupplierWrappers(List.of( + new ExactlyOnceMemberSupplierWrapper(context), + new LatestStateMemberSupplierWrapper(context, eventStreamProperties), + new VersionMaterialisedMemberSupplierWrapper(context, eventStreamProperties.getVersionOfPath()) + )); + } +} diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/VersionMaterialisedMemberSupplierWrapper.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/VersionMaterialisedMemberSupplierWrapper.java new file mode 100644 index 000000000..98e3fcb74 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/VersionMaterialisedMemberSupplierWrapper.java @@ -0,0 +1,36 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties; +import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.VersionMaterialisedMemberSupplier; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.ResourceFactory; +import org.apache.nifi.processor.ProcessContext; + +public class VersionMaterialisedMemberSupplierWrapper extends MemberSupplierWrapper { + private final ProcessContext context; + private final String versionOfPath; + + public VersionMaterialisedMemberSupplierWrapper(ProcessContext context, String versionOfPath) { + this.context = context; + this.versionOfPath = versionOfPath; + } + + @Override + protected boolean shouldBeWrapped() { + return LdesProcessorProperties.useVersionMaterialisation(context); + } + + @Override + protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSupplier) { + return new VersionMaterialisedMemberSupplier(memberSupplier, createVersionMaterialiser()); + } + + private VersionMaterialiser createVersionMaterialiser() { + final Property versionOfPredicate = ResourceFactory.createProperty(versionOfPath); + final boolean restrictToMembers = LdesProcessorProperties.restrictToMembers(context); + return new VersionMaterialiser(versionOfPredicate, restrictToMembers); + } +} diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/LdesPropertiesExtractor.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/LdesPropertiesExtractor.java deleted file mode 100644 index 1af8e216d..000000000 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/LdesPropertiesExtractor.java +++ /dev/null @@ -1,72 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.services; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.domain.valueobjects.LdesProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Response; -import ldes.client.startingtreenode.RedirectRequestExecutor; -import ldes.client.startingtreenode.domain.valueobjects.RedirectHistory; -import ldes.client.startingtreenode.domain.valueobjects.StartingNodeRequest; -import ldes.client.treenodesupplier.domain.valueobject.LdesMetaData; -import org.apache.jena.rdf.model.*; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFParser; - -import java.io.ByteArrayInputStream; -import java.util.Optional; - -import static org.apache.jena.rdf.model.ResourceFactory.createProperty; - -public class LdesPropertiesExtractor { - - private static final String LDES = "https://w3id.org/ldes#"; - private static final Property LDES_VERSION_OF = createProperty(LDES, "versionOfPath"); - private static final Property LDES_TIMESTAMP_PATH = createProperty(LDES, "timestampPath"); - private static final Property TREE_SHAPE = createProperty("https://w3id.org/tree#", "shape"); - private final RequestExecutor requestExecutor; - - public LdesPropertiesExtractor(RequestExecutor requestExecutor) { - this.requestExecutor = requestExecutor; - } - - private Optional getPropertyValue(Model model, Property property) { - return model.listStatements(null, property, (Resource) null) - .nextOptional() - .map(Statement::getObject) - .map(RDFNode::asResource) - .map(Resource::toString); - } - - public LdesProperties getLdesProperties(LdesMetaData ldesMetaData, boolean needTimestampPath, - boolean needVersionOfPath, - boolean needShape) { - - Model model = getModelFromStartingTreeNode(ldesMetaData.getStartingNodeUrl(), ldesMetaData.getLang()); - - String timestampPath = getResource(needTimestampPath, model, LDES_TIMESTAMP_PATH); - String versionOfPath = getResource(needVersionOfPath, model, LDES_VERSION_OF); - String shape = getResource(needShape, model, TREE_SHAPE); - return new LdesProperties(timestampPath, versionOfPath, shape); - } - - private Model getModelFromStartingTreeNode(String url, Lang lang) { - RedirectRequestExecutor redirectRequestExecutor = new RedirectRequestExecutor(requestExecutor); - Response response = redirectRequestExecutor - .execute(new StartingNodeRequest(url, lang, new RedirectHistory())); - - return RDFParser - .source(response.getBody().map(ByteArrayInputStream::new).orElseThrow()) - .lang(lang) - .build() - .toModel(); - } - - private String getResource(boolean resourceNeeded, Model model, Property property) { - return resourceNeeded - ? getResource(model, property).orElseThrow(() -> new LdesPropertyNotFoundException(property.toString())) - : null; - } - - public Optional getResource(Model model, Property property) { - return getPropertyValue(model, property); - } -} diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/LdesPropertyNotFoundException.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/LdesPropertyNotFoundException.java deleted file mode 100644 index 746ad57a5..000000000 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/LdesPropertyNotFoundException.java +++ /dev/null @@ -1,8 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.services; - -public class LdesPropertyNotFoundException extends RuntimeException { - - public LdesPropertyNotFoundException(String message) { - super(message); - } -} diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java index f6ad97a24..6719fb79a 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java @@ -67,8 +67,7 @@ void tearDown() { @ArgumentsSource(MatchNumberOfFlowFilesArgumentsProvider.class) void shouldMatchNumberOfFlowFiles(String dataSourceUrl, int numberOfRuns) { testRunner.setProperty("DATA_SOURCE_URLS", dataSourceUrl); - testRunner.setProperty(STATE_PERSISTENCE_STRATEGY, - "SQLITE"); + testRunner.setProperty(STATE_PERSISTENCE_STRATEGY, "SQLITE"); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); @@ -186,7 +185,6 @@ void shouldSupportVersionMaterialisation(Map statePe testRunner.setProperty("USE_VERSION_MATERIALISATION", Boolean.TRUE.toString()); testRunner.setProperty("USE_LATEST_STATE_FILTER", Boolean.FALSE.toString()); testRunner.setProperty("RESTRICT_TO_MEMBERS", Boolean.FALSE.toString()); - testRunner.setProperty("VERSION_OF_PROPERTY", VERSION_OF); testRunner.run(); @@ -205,8 +203,7 @@ void shouldSupportVersionMaterialisation(Map statePe @Test void when_runningLdesClientWithStreamPropertiesFlags_expectsLdesPropertiesInFlowFile() { - testRunner.setProperty("DATA_SOURCE_URLS", - "http://localhost:10101/exampleData?scenario=gml-data"); + testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/exampleData?scenario=gml-data"); testRunner.setProperty("STREAM_SHAPE_PROPERTY", Boolean.TRUE.toString()); testRunner.run(); @@ -250,7 +247,6 @@ void shouldSupportOnlyOnceFilter(Map statePersistenc testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); testRunner.setProperty("USE_EXACTLY_ONCE_FILTER", Boolean.TRUE.toString()); testRunner.setProperty("RESTRICT_TO_MEMBERS", Boolean.FALSE.toString()); - testRunner.setProperty("VERSION_OF_PROPERTY", VERSION_OF); testRunner.run(4); @@ -275,7 +271,6 @@ void shouldSupportDisableOfOnlyOnceFilter(Map stateP testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); testRunner.setProperty("USE_EXACTLY_ONCE_FILTER", Boolean.FALSE.toString()); testRunner.setProperty("RESTRICT_TO_MEMBERS", Boolean.FALSE.toString()); - testRunner.setProperty("VERSION_OF_PROPERTY", VERSION_OF); testRunner.run(4); @@ -303,7 +298,6 @@ void shouldSupportVersionMaterialisationWithLatestStateFilter(Map properties; + + + public TestProcessContext(boolean useExactlyOnceFilter, boolean useVersionMaterialisation, boolean useLatestStateFilter) { + properties = Map.of( + LdesProcessorProperties.USE_EXACTLY_ONCE_FILTER, String.valueOf(useExactlyOnceFilter), + LdesProcessorProperties.USE_VERSION_MATERIALISATION, String.valueOf(useVersionMaterialisation), + LdesProcessorProperties.USE_LATEST_STATE_FILTER, String.valueOf(useLatestStateFilter), + PersistenceProperties.KEEP_STATE, String.valueOf(false), + PersistenceProperties.STATE_PERSISTENCE_STRATEGY, "MEMORY" + ); + } + + public TestProcessContext(boolean useExactlyOnce) { + this(useExactlyOnce, false, false); + } + + public TestProcessContext(boolean useVersionMaterialisation, boolean useLatestStateFilter) { + this(false, useVersionMaterialisation, useLatestStateFilter); + } + + @Override + public PropertyValue getProperty(PropertyDescriptor descriptor) { + return new MockPropertyValue(properties.get(descriptor)); + } +} diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/resources/mappings/200-response-with-indirect-url.json b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/resources/mappings/200-response-with-indirect-url.json index 73e9c0ad2..53cd63763 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/resources/mappings/200-response-with-indirect-url.json +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/resources/mappings/200-response-with-indirect-url.json @@ -5,8 +5,7 @@ }, "response": { "status": 200, - "jsonBody": - [ + "jsonBody": [ { "@context": { "tree": "https://w3id.org/tree#", @@ -48,6 +47,12 @@ }, "@id": "http://localhost:10101/feed", "@type": "ldes:EventStream", + "ldes:versionOfPath": { + "@id": "dc:isVersionOf" + }, + "ldes:timestampPath": { + "@id": "dc:modified" + }, "tree:view": { "@id": "http://localhost:10101/feed?page=2023", "ldes:retentionPolicy": { diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index 668619b83..e2cfd6150 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -10,21 +10,21 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientConnectorApi; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.event.LdesClientConnectorApiCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.management.status.ClientStatusConsumer; import be.vlaanderen.informatievlaanderen.ldes.ldio.management.status.ClientStatusService; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioObserver; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; +import ldes.client.eventstreamproperties.EventStreamPropertiesFetcher; import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.persistence.PersistenceProperties.KEEP_STATE; - @SuppressWarnings("java:S6830") @Configuration @@ -60,6 +60,7 @@ public LdioClientConnectorConfigurator(ApplicationEventPublisher eventPublisher, @Override public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, ApplicationEventPublisher applicationEventPublisher, ComponentProperties properties) { final String pipelineName = properties.getPipelineName(); + final LdioLdesClientProperties ldioLdesClientProperties = LdioLdesClientProperties.fromComponentProperties(properties); final var connectorTransferUrl = properties.getProperty(CONNECTOR_TRANSFER_URL); final var transferService = new MemoryTransferService(baseRequestExecutor, connectorTransferUrl); final var memoryTokenServiceLifecycle = new MemoryTokenServiceLifecycle(); @@ -68,10 +69,16 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, Appli final var urlProxy = getEdcUrlProxy(properties); final var edcRequestExecutor = requestExecutorFactory.createEdcExecutor(baseRequestExecutor, tokenService, urlProxy); + final var eventStreamPropertiesFetcher = new EventStreamPropertiesFetcher(edcRequestExecutor); final var clientStatusConsumer = new ClientStatusConsumer(pipelineName, clientStatusService); - final MemberSupplier memberSupplier = new MemberSupplierFactory(properties, edcRequestExecutor, - clientStatusConsumer).getMemberSupplier(); - final boolean keepState = properties.getOptionalBoolean(KEEP_STATE).orElse(false); + final MemberSupplier memberSupplier = new MemberSupplierFactory( + ldioLdesClientProperties, + eventStreamPropertiesFetcher, + edcRequestExecutor, + clientStatusConsumer + ).getMemberSupplier(); + + final boolean keepState = ldioLdesClientProperties.isKeepStateEnabled(); final LdioObserver ldioObserver = LdioObserver.register(NAME, pipelineName, observationRegistry); final var ldesClient = new LdioLdesClient(executor, ldioObserver, memberSupplier, applicationEventPublisher, keepState, clientStatusConsumer); diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index 2aa2a57f4..9d51c4456 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -66,7 +66,10 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-infra-sql - ${project.version} + + + be.vlaanderen.informatievlaanderen.ldes.client + event-stream-properties-fetcher org.springdoc diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java index 98093452d..22686eb3c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientProperties.java @@ -1,20 +1,100 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.ConfigPropertyMissingException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.InvalidConfigException; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFLanguages; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Optional; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientPropertyKeys.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.persistence.PersistenceProperties.KEEP_STATE; + public class LdioLdesClientProperties { + private static final Logger log = LoggerFactory.getLogger(LdioLdesClientProperties.class); + public static final boolean DEFAULT_KEEP_STATE = false; + public static final boolean DEFAULT_USE_LATEST_STATE_FILTER = true; + public static final boolean DEFAULT_EXACTLY_ONCE_ENABLED = true; + private final ComponentProperties properties; - private LdioLdesClientProperties() { + private LdioLdesClientProperties(ComponentProperties properties) { + this.properties = properties; } - // general properties - public static final String URLS = "urls"; - public static final String SOURCE_FORMAT = "source-format"; + public List getUrls() { + final List urls = properties.getPropertyList(URLS); + if (urls.isEmpty()) { + throw new ConfigPropertyMissingException(properties.getPipelineName(), properties.getComponentName(), "urls"); + } + return urls; + } - public static final String TIMESTAMP_PATH_PROP = "timestamp-path"; - public static final String USE_EXACTLY_ONCE_FILTER = "enable-exactly-once"; + public String getFirstUrl() { + return getUrls().getFirst(); + } - // version materialisation properties - public static final String USE_VERSION_MATERIALISATION = "materialisation.enabled"; - public static final String VERSION_OF_PROPERTY = "materialisation.version-of-property"; - public static final String USE_LATEST_STATE_FILTER = "materialisation.enable-latest-state"; + public Lang getSourceFormat() { + return properties.getOptionalProperty(SOURCE_FORMAT) + .map(RDFLanguages::nameToLang) + .orElse(DEFAULT_SOURCE_FORMAT); + } + public boolean isExactlyOnceEnabled() { + return properties.getOptionalBoolean(USE_EXACTLY_ONCE_FILTER) + .or(() -> getOptionalVersionMaterialisationBoolean().map(isEnabled -> !isEnabled)) + .orElse(DEFAULT_EXACTLY_ONCE_ENABLED); + } + + public boolean isVersionMaterialisationEnabled() { + return getOptionalVersionMaterialisationBoolean().orElse(false); + } + + private Optional getOptionalVersionMaterialisationBoolean() { + return properties.getOptionalBoolean(USE_VERSION_MATERIALISATION); + } + + public boolean isKeepStateEnabled() { + return properties.getOptionalBoolean(KEEP_STATE).orElse(DEFAULT_KEEP_STATE); + } + + public boolean isLatestStateEnabled() { + return properties.getOptionalBoolean(USE_LATEST_STATE_FILTER).orElse(DEFAULT_USE_LATEST_STATE_FILTER); + } + + public ComponentProperties getProperties() { + return properties; + } + + public static LdioLdesClientProperties fromComponentProperties(ComponentProperties properties) { + final LdioLdesClientProperties clientProps = new LdioLdesClientProperties(properties); + warnWhenVersionMaterialisationIsNotEnabled(clientProps); + checkIfBothVersionMaterialisationAndExactlyOnceAreExplicitlyEnabled(clientProps); + warnIfExactlyOnceFilterMustBeDisabled(clientProps); + return clientProps; + } + + private static void warnWhenVersionMaterialisationIsNotEnabled(LdioLdesClientProperties clientProps) { + if(clientProps.getOptionalVersionMaterialisationBoolean().isEmpty()) { + log.atWarn().log(""" + Version-materialization in the LDES Client hasn’t been turned on. Please note that in the future, \ + this will be the default output of the LDES Client \ + and having version-objects as output will have to be configured explicitly."""); + } + } + + private static void checkIfBothVersionMaterialisationAndExactlyOnceAreExplicitlyEnabled(LdioLdesClientProperties clientProps) { + if(clientProps.isVersionMaterialisationEnabled() && clientProps.isExactlyOnceEnabled()) { + throw new InvalidConfigException("The exactly once filter can not be enabled with version materialisation."); + } + } + + private static void warnIfExactlyOnceFilterMustBeDisabled(LdioLdesClientProperties clientProps) { + if(clientProps.isExactlyOnceEnabled()) { + log.warn("The exactly once filter can not be used while version materialisation is active, disabling filter"); + } + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientPropertyKeys.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientPropertyKeys.java new file mode 100644 index 000000000..d5f9b72f1 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientPropertyKeys.java @@ -0,0 +1,19 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + +import org.apache.jena.riot.Lang; + +public class LdioLdesClientPropertyKeys { + private LdioLdesClientPropertyKeys() { + } + + // general properties + public static final String URLS = "urls"; + public static final String SOURCE_FORMAT = "source-format"; + public static final Lang DEFAULT_SOURCE_FORMAT = Lang.TURTLE; + + public static final String USE_EXACTLY_ONCE_FILTER = "enable-exactly-once"; + + // version materialisation properties + public static final String USE_VERSION_MATERIALISATION = "materialisation.enabled"; + public static final String USE_LATEST_STATE_FILTER = "materialisation.enable-latest-state"; +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java index 6e0189ef5..5e6e93a95 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java @@ -1,24 +1,12 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient; -import be.vlaanderen.informatievlaanderen.ldes.ldio.management.status.ClientStatusConsumer; import be.vlaanderen.informatievlaanderen.ldes.ldio.management.status.ClientStatusService; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioInputConfigurator; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioObserver; -import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; import io.micrometer.observation.ObservationRegistry; -import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.persistence.PersistenceProperties.KEEP_STATE; - @Configuration public class LdioLdesClientAutoConfig { @SuppressWarnings("java:S6830") @@ -27,35 +15,4 @@ public LdioInputConfigurator ldioConfigurator(ClientStatusService clientStatusSe return new LdioLdesClientConfigurator(clientStatusService, observationRegistry); } - public static class LdioLdesClientConfigurator implements LdioInputConfigurator { - private final ClientStatusService clientStatusService; - - private final ObservationRegistry observationRegistry; - - public LdioLdesClientConfigurator(ClientStatusService clientStatusService, ObservationRegistry observationRegistry) { - this.clientStatusService = clientStatusService; - this.observationRegistry = observationRegistry; - } - - @Override - public LdioInput configure(LdiAdapter adapter, ComponentExecutor componentExecutor, - ApplicationEventPublisher applicationEventPublisher, - ComponentProperties properties) { - String pipelineName = properties.getPipelineName(); - final var requestExecutorFactory = new RequestExecutorFactory(false); - final var requestExecutor = new LdioRequestExecutorSupplier(requestExecutorFactory).getRequestExecutor(properties); - final var clientStatusConsumer = new ClientStatusConsumer(pipelineName, clientStatusService); - final MemberSupplier memberSupplier = new MemberSupplierFactory(properties, requestExecutor, clientStatusConsumer).getMemberSupplier(); - final boolean keepState = properties.getOptionalBoolean(KEEP_STATE).orElse(false); - final LdioObserver ldioObserver = LdioObserver.register(LdioLdesClient.NAME, pipelineName, observationRegistry); - final var ldesClient = new LdioLdesClient(componentExecutor, ldioObserver, memberSupplier, applicationEventPublisher, keepState, clientStatusConsumer); - ldesClient.start(); - return ldesClient; - } - - @Override - public boolean isAdapterRequired() { - return false; - } - } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConfigurator.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConfigurator.java new file mode 100644 index 000000000..351306773 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConfigurator.java @@ -0,0 +1,52 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.management.status.ClientStatusConsumer; +import be.vlaanderen.informatievlaanderen.ldes.ldio.management.status.ClientStatusService; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioInputConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioObserver; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; +import io.micrometer.observation.ObservationRegistry; +import ldes.client.eventstreamproperties.EventStreamPropertiesFetcher; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import org.springframework.context.ApplicationEventPublisher; + +public class LdioLdesClientConfigurator implements LdioInputConfigurator { + private final ClientStatusService clientStatusService; + private final ObservationRegistry observationRegistry; + private final LdioRequestExecutorSupplier requestExecutorSupplier; + + public LdioLdesClientConfigurator(ClientStatusService clientStatusService, ObservationRegistry observationRegistry) { + this.clientStatusService = clientStatusService; + this.observationRegistry = observationRegistry; + requestExecutorSupplier = new LdioRequestExecutorSupplier(new RequestExecutorFactory(false)); + } + + @Override + public LdioInput configure(LdiAdapter adapter, ComponentExecutor componentExecutor, + ApplicationEventPublisher applicationEventPublisher, + ComponentProperties properties) { + final String pipelineName = properties.getPipelineName(); + final LdioLdesClientProperties ldioLdesClientProperties = LdioLdesClientProperties.fromComponentProperties(properties); + final var requestExecutor = requestExecutorSupplier.getRequestExecutor(properties); + final EventStreamPropertiesFetcher eventStreamPropertiesFetcher = new EventStreamPropertiesFetcher(requestExecutor); + final var clientStatusConsumer = new ClientStatusConsumer(pipelineName, clientStatusService); + final MemberSupplier memberSupplier = new MemberSupplierFactory(ldioLdesClientProperties, eventStreamPropertiesFetcher, requestExecutor, clientStatusConsumer).getMemberSupplier(); + final boolean keepState = ldioLdesClientProperties.isKeepStateEnabled(); + final LdioObserver ldioObserver = LdioObserver.register(LdioLdesClient.NAME, pipelineName, observationRegistry); + final var ldesClient = new LdioLdesClient(componentExecutor, ldioObserver, memberSupplier, applicationEventPublisher, keepState, clientStatusConsumer); + ldesClient.start(); + return ldesClient; + } + + @Override + public boolean isAdapterRequired() { + return false; + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java index 0783a68f4..f1de9ca78 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java @@ -1,156 +1,60 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; -import be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.ConfigPropertyMissingException; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.InvalidConfigException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers.MemberSupplierWrappersBuilder; +import be.vlaanderen.informatievlaanderen.ldes.ldio.management.status.ClientStatusConsumer; +import ldes.client.eventstreamproperties.EventStreamPropertiesFetcher; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.eventstreamproperties.valueobjects.PropertiesRequest; import ldes.client.treenodesupplier.TreeNodeProcessor; -import ldes.client.treenodesupplier.domain.valueobject.ClientStatus; import ldes.client.treenodesupplier.domain.valueobject.LdesMetaData; import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; -import ldes.client.treenodesupplier.filters.ExactlyOnceFilter; -import ldes.client.treenodesupplier.filters.LatestStateFilter; -import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; import ldes.client.treenodesupplier.membersuppliers.MemberSupplierImpl; -import ldes.client.treenodesupplier.membersuppliers.VersionMaterialisedMemberSupplier; -import org.apache.jena.rdf.model.Property; -import org.apache.jena.rdf.model.ResourceFactory; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFLanguages; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; -import java.util.Optional; -import java.util.function.Consumer; - -import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.*; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.persistence.PersistenceProperties.KEEP_STATE; import static org.apache.jena.rdf.model.ResourceFactory.createProperty; public class MemberSupplierFactory { - - private static final String DEFAULT_VERSION_OF_KEY = "http://purl.org/dc/terms/isVersionOf"; - private static final String DEFAULT_TIMESTAMP_KEY = "http://www.w3.org/ns/prov#generatedAtTime"; - - private final Logger log = LoggerFactory.getLogger(MemberSupplierFactory.class); - - private final ComponentProperties properties; - private final RequestExecutor requestExecutor; - private final Consumer clientStatusConsumer; - - public MemberSupplierFactory(ComponentProperties properties, RequestExecutor requestExecutor, - Consumer clientStatusConsumer) { - this.properties = properties; - this.requestExecutor = requestExecutor; - this.clientStatusConsumer = clientStatusConsumer; - } - - public MemberSupplier getMemberSupplier() { - log.info("Starting LdesClientRunner run setup"); - log.info("LdesClientRunner setup finished"); - MemberSupplier baseMemberSupplier = - new MemberSupplierImpl(getTreeNodeProcessor(), getKeepState()); - if (useExactlyOnceFilter()) { - return new FilteredMemberSupplier(baseMemberSupplier, getExactlyOnceFilter()); - } else if (useVersionMaterialisation()) { - MemberSupplier decoratedMemberSupplier = useLatestStateFilter() - ? new FilteredMemberSupplier(baseMemberSupplier, getLatestStateFilter()) - : baseMemberSupplier; - return new VersionMaterialisedMemberSupplier(decoratedMemberSupplier, createVersionMaterialiser()); - } else { - return baseMemberSupplier; - } - } - - private LatestStateFilter getLatestStateFilter() { - String timestampPath = properties.getOptionalProperty(TIMESTAMP_PATH_PROP).orElse(DEFAULT_TIMESTAMP_KEY); - String versionOfPath = properties.getOptionalProperty(VERSION_OF_PROPERTY).orElse(DEFAULT_VERSION_OF_KEY); - return new LatestStateFilter(getStatePersistence().getMemberVersionRepository(), getKeepState(), timestampPath, versionOfPath); - } - - private ExactlyOnceFilter getExactlyOnceFilter() { - return new ExactlyOnceFilter(getStatePersistence().getMemberIdRepository(), getKeepState()); - } - - private TreeNodeProcessor getTreeNodeProcessor() { - List targetUrls = properties.getPropertyList(URLS); - - if (targetUrls.isEmpty()) { - throw new ConfigPropertyMissingException(properties.getPipelineName(), properties.getComponentName(), "urls"); - } - - Lang sourceFormat = getSourceFormat(); - LdesMetaData ldesMetaData = new LdesMetaData(targetUrls, sourceFormat); - TimestampExtractor timestampExtractor = properties.getOptionalProperty(TIMESTAMP_PATH_PROP) - .map(timestampPath -> (TimestampExtractor) new TimestampFromPathExtractor(createProperty(timestampPath))) - .orElseGet(TimestampFromCurrentTimeExtractor::new); - - return new TreeNodeProcessor(ldesMetaData, getStatePersistence(), requestExecutor, timestampExtractor, clientStatusConsumer); - } - - private StatePersistence getStatePersistence() { - return new StatePersistenceFactory().getStatePersistence(properties); - } - - private Boolean getKeepState() { - return properties.getOptionalBoolean(KEEP_STATE).orElse(false); - } - - private boolean useVersionMaterialisation() { - return properties - .getOptionalBoolean(USE_VERSION_MATERIALISATION) - .orElseGet(() -> { - log.warn("Version-materialization in the LDES Client hasn’t been turned on. " + - "Please note that in the future, this will be the default output of the LDES Client " + - "and having version-objects as output will have to be configured explicitly."); - return false; - }); - } - - private boolean useExactlyOnceFilter() { - Optional exactlyOneFilterProperty = properties.getOptionalBoolean(USE_EXACTLY_ONCE_FILTER); - if (exactlyOneFilterProperty.isPresent()) { - // use filter is explicitly set - boolean useFilter = exactlyOneFilterProperty.get(); - if (useVersionMaterialisation() && useFilter) { - throw new InvalidConfigException("The exactly once filter can not be enabled with version materialisation."); - } else { - return useFilter; - } - } else { - // use filter is not explicitly set - if (useVersionMaterialisation()) { - log.warn("The exactly once filter can not be used while version materialisation is active, disabling filter"); - return false; - } else { - return true; - } - } - } - - private Lang getSourceFormat() { - return properties.getOptionalProperty(SOURCE_FORMAT) - .map(RDFLanguages::nameToLang) - .orElse(Lang.TURTLE); - } - - private VersionMaterialiser createVersionMaterialiser() { - final Property versionOfProperty = properties - .getOptionalProperty(VERSION_OF_PROPERTY) - .map(ResourceFactory::createProperty) - .orElseGet(() -> createProperty(DEFAULT_VERSION_OF_KEY)); - return new VersionMaterialiser(versionOfProperty, false); - } - - private boolean useLatestStateFilter() { - return properties.getOptionalBoolean(USE_LATEST_STATE_FILTER).orElse(true); - } - + private static final Logger log = LoggerFactory.getLogger(MemberSupplierFactory.class); + private final LdioLdesClientProperties clientProperties; + private final RequestExecutor requestExecutor; + private final ClientStatusConsumer clientStatusConsumer; + private final EventStreamPropertiesFetcher eventStreamPropertiesFetcher; + + public MemberSupplierFactory(LdioLdesClientProperties clientProperties, + EventStreamPropertiesFetcher eventStreamPropertiesFetcher, + RequestExecutor requestExecutor, + ClientStatusConsumer clientStatusConsumer ) { + this.clientProperties = clientProperties; + this.requestExecutor = requestExecutor; + this.clientStatusConsumer = clientStatusConsumer; + this.eventStreamPropertiesFetcher = eventStreamPropertiesFetcher; + } + + public MemberSupplier getMemberSupplier() { + log.info("Starting LdesClientRunner run setup"); + final EventStreamProperties eventStreamProperties = eventStreamPropertiesFetcher.fetchEventStreamProperties(new PropertiesRequest(clientProperties.getFirstUrl(), clientProperties.getSourceFormat())); + MemberSupplier baseMemberSupplier = new MemberSupplierImpl(getTreeNodeProcessor(eventStreamProperties), clientProperties.isKeepStateEnabled()); + baseMemberSupplier = new MemberSupplierWrappersBuilder() + .withEventStreamProperties(eventStreamProperties) + .withLdioLdesClientProperties(clientProperties) + .build() + .wrapMemberSupplier(baseMemberSupplier); + + log.info("LdesClientRunner setup finished"); + return baseMemberSupplier; + } + + private TreeNodeProcessor getTreeNodeProcessor(EventStreamProperties eventStreamProperties) { + final StatePersistence statePersistence = new StatePersistenceFactory().getStatePersistence(clientProperties.getProperties()); + LdesMetaData ldesMetaData = new LdesMetaData(clientProperties.getUrls(), clientProperties.getSourceFormat()); + TimestampExtractor timestampExtractor = new TimestampFromPathExtractor(createProperty(eventStreamProperties.getTimestampPath())); + return new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor, timestampExtractor, clientStatusConsumer); + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/ExactlyOnceMemberSupplierWrapper.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/ExactlyOnceMemberSupplierWrapper.java new file mode 100644 index 000000000..530795812 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/ExactlyOnceMemberSupplierWrapper.java @@ -0,0 +1,33 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.StatePersistenceFactory; +import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; +import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; +import ldes.client.treenodesupplier.filters.ExactlyOnceFilter; +import ldes.client.treenodesupplier.filters.MemberFilter; +import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; + +public class ExactlyOnceMemberSupplierWrapper extends MemberSupplierWrapper { + private final LdioLdesClientProperties properties; + + public ExactlyOnceMemberSupplierWrapper(LdioLdesClientProperties properties) { + this.properties = properties; + } + + @Override + public boolean shouldBeWrapped() { + return properties.isExactlyOnceEnabled(); + } + + @Override + protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSupplier) { + return new FilteredMemberSupplier(memberSupplier, createExactlyOnceFilter()); + } + + private MemberFilter createExactlyOnceFilter() { + final StatePersistence statePersistence = new StatePersistenceFactory().getStatePersistence(properties.getProperties()); + return new ExactlyOnceFilter(statePersistence.getMemberIdRepository(), properties.isKeepStateEnabled()); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/LatestStateMemberSupplierWrapper.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/LatestStateMemberSupplierWrapper.java new file mode 100644 index 000000000..19ab7864e --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/LatestStateMemberSupplierWrapper.java @@ -0,0 +1,36 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.StatePersistenceFactory; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; +import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; +import ldes.client.treenodesupplier.filters.LatestStateFilter; +import ldes.client.treenodesupplier.filters.MemberFilter; +import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; + +public class LatestStateMemberSupplierWrapper extends MemberSupplierWrapper { + private final EventStreamProperties eventStreamProperties; + private final LdioLdesClientProperties ldioLdesClientProperties; + + public LatestStateMemberSupplierWrapper(EventStreamProperties eventStreamProperties, LdioLdesClientProperties ldioLdesClientProperties) { + this.eventStreamProperties = eventStreamProperties; + this.ldioLdesClientProperties = ldioLdesClientProperties; + } + + @Override + public boolean shouldBeWrapped() { + return ldioLdesClientProperties.isVersionMaterialisationEnabled() && ldioLdesClientProperties.isLatestStateEnabled(); + } + + @Override + protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSupplier) { + return new FilteredMemberSupplier(memberSupplier, createLatestStateFilter()); + } + + private MemberFilter createLatestStateFilter() { + final StatePersistence statePersistence = new StatePersistenceFactory().getStatePersistence(ldioLdesClientProperties.getProperties()); + return new LatestStateFilter(statePersistence.getMemberVersionRepository(), ldioLdesClientProperties.isKeepStateEnabled(), eventStreamProperties.getTimestampPath(), eventStreamProperties.getVersionOfPath()); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/MemberSupplierWrappersBuilder.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/MemberSupplierWrappersBuilder.java new file mode 100644 index 000000000..dbbe7b486 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/MemberSupplierWrappersBuilder.java @@ -0,0 +1,30 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.treenodesupplier.domain.services.MemberSupplierWrappers; + +import java.util.List; + +public class MemberSupplierWrappersBuilder implements MemberSupplierWrappers.Builder { + private EventStreamProperties eventStreamProperties; + private LdioLdesClientProperties ldioLdesClientProperties; + + public MemberSupplierWrappersBuilder withEventStreamProperties(EventStreamProperties eventStreamProperties) { + this.eventStreamProperties = eventStreamProperties; + return this; + } + + public MemberSupplierWrappersBuilder withLdioLdesClientProperties(LdioLdesClientProperties ldioLdesClientProperties) { + this.ldioLdesClientProperties = ldioLdesClientProperties; + return this; + } + + public MemberSupplierWrappers build() { + return new MemberSupplierWrappers(List.of( + new ExactlyOnceMemberSupplierWrapper(ldioLdesClientProperties), + new LatestStateMemberSupplierWrapper(eventStreamProperties, ldioLdesClientProperties), + new VersionMaterialisedMemberSupplierWrapper(eventStreamProperties, ldioLdesClientProperties) + )); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/VersionMaterialisedMemberSupplierWrapper.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/VersionMaterialisedMemberSupplierWrapper.java new file mode 100644 index 000000000..eff99ebe7 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/VersionMaterialisedMemberSupplierWrapper.java @@ -0,0 +1,35 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.VersionMaterialisedMemberSupplier; +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.ResourceFactory; + +public class VersionMaterialisedMemberSupplierWrapper extends MemberSupplierWrapper { + private final EventStreamProperties eventStreamProperties; + private final LdioLdesClientProperties clientProperties; + + public VersionMaterialisedMemberSupplierWrapper(EventStreamProperties eventStreamProperties, LdioLdesClientProperties clientProperties) { + this.eventStreamProperties = eventStreamProperties; + this.clientProperties = clientProperties; + } + + @Override + public boolean shouldBeWrapped() { + return clientProperties.isVersionMaterialisationEnabled(); + } + + @Override + protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSupplier) { + return new VersionMaterialisedMemberSupplier(memberSupplier, createVersionMaterialiser()); + } + + private VersionMaterialiser createVersionMaterialiser() { + final Property versionOfPath = ResourceFactory.createProperty(eventStreamProperties.getVersionOfPath()); + return new VersionMaterialiser(versionOfPath, false); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/status/ClientStatusConverter.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/status/ClientStatusConverter.java new file mode 100644 index 000000000..305d75331 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/management/status/ClientStatusConverter.java @@ -0,0 +1,42 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.management.status; + +import ldes.client.treenodesupplier.domain.valueobject.ClientStatus; +import org.springframework.http.HttpInputMessage; +import org.springframework.http.HttpOutputMessage; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.http.converter.HttpMessageNotWritableException; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +@Component +public class ClientStatusConverter implements HttpMessageConverter { + @Override + public boolean canRead(Class clazz, MediaType mediaType) { + return clazz.equals(ClientStatus.class); + } + + @Override + public boolean canWrite(Class clazz, MediaType mediaType) { + return clazz.equals(ClientStatus.class); + } + + @Override + public List getSupportedMediaTypes() { + return List.of(MediaType.ALL); + } + + @Override + public ClientStatus read(Class clazz, HttpInputMessage inputMessage) throws IOException, HttpMessageNotReadableException { + return ClientStatus.valueOf(inputMessage.getBody().toString().toUpperCase()); + } + + @Override + public void write(ClientStatus clientStatus, MediaType contentType, HttpOutputMessage outputMessage) throws IOException, HttpMessageNotWritableException { + outputMessage.getBody().write(clientStatus.toString().getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientPropertiesTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientPropertiesTest.java new file mode 100644 index 000000000..cb800abd3 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientPropertiesTest.java @@ -0,0 +1,24 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.InvalidConfigException; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class LdioLdesClientPropertiesTest { + + @Test + void given_ExactlyOnceAndVersionMaterialisationAreBothExplicitlyEnabled_when_parseConfig_then_ThrowException() { + final ComponentProperties properties = new ComponentProperties("pipeline", "cname", Map.of( + LdioLdesClientPropertyKeys.USE_EXACTLY_ONCE_FILTER, String.valueOf(true), + LdioLdesClientPropertyKeys.USE_VERSION_MATERIALISATION, String.valueOf(true) + )); + + assertThatThrownBy(() -> LdioLdesClientProperties.fromComponentProperties(properties)) + .isInstanceOf(InvalidConfigException.class) + .hasMessage("Invalid config: \"The exactly once filter can not be enabled with version materialisation.\" ."); + } +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/ExactlyOnceMemberSupplierWrapperTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/ExactlyOnceMemberSupplierWrapperTest.java new file mode 100644 index 000000000..70b88a4c8 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/ExactlyOnceMemberSupplierWrapperTest.java @@ -0,0 +1,47 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers.ExactlyOnceMemberSupplierWrapper; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ExactlyOnceMemberSupplierWrapperTest { + @Mock + private ComponentProperties componentProperties; + @Mock + private LdioLdesClientProperties ldioLdesClientProperties; + @Mock + private MemberSupplier baseSupplier; + @InjectMocks + private ExactlyOnceMemberSupplierWrapper exactlyOnceMemberSupplierWrapper; + + @Test + void given_ExactlyOnceEnabled_when_wrap_then_ReturnFilteredMemberSupplier() { + when(ldioLdesClientProperties.isExactlyOnceEnabled()).thenReturn(true); + when(ldioLdesClientProperties.getProperties()).thenReturn(componentProperties); + + final MemberSupplier memberSupplier = exactlyOnceMemberSupplierWrapper.wrapMemberSupplier(baseSupplier); + + assertThat(memberSupplier).isInstanceOf(FilteredMemberSupplier.class); + } + + @Test + void given_ExactlyOnceDisabled_when_wrap_then_ReturnBaseMemberSupplier() { + when(ldioLdesClientProperties.isExactlyOnceEnabled()).thenReturn(false); + + final MemberSupplier memberSupplier = exactlyOnceMemberSupplierWrapper.wrapMemberSupplier(baseSupplier); + + assertThat(memberSupplier).isSameAs(baseSupplier); + } + +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LatestStateMemberSupplierWrapperTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LatestStateMemberSupplierWrapperTest.java new file mode 100644 index 000000000..141614c34 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LatestStateMemberSupplierWrapperTest.java @@ -0,0 +1,64 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers.LatestStateMemberSupplierWrapper; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class LatestStateMemberSupplierWrapperTest { + @Mock + private ComponentProperties componentProperties; + @Mock + private LdioLdesClientProperties ldioLdesClientProperties; + @Mock + private MemberSupplier baseSupplier; + private LatestStateMemberSupplierWrapper latestStateMemberSupplierWrapper; + + @BeforeEach + void setUp() { + final EventStreamProperties eventStreamProperties = new EventStreamProperties("test", "test", "test", "test"); + latestStateMemberSupplierWrapper = new LatestStateMemberSupplierWrapper(eventStreamProperties, ldioLdesClientProperties); + } + + @Test + void given_LatestStateEnabled_when_wrap_then_ReturnFilteredMemberSupplier() { + when(ldioLdesClientProperties.isLatestStateEnabled()).thenReturn(true); + when(ldioLdesClientProperties.isVersionMaterialisationEnabled()).thenReturn(true); + when(ldioLdesClientProperties.getProperties()).thenReturn(componentProperties); + + final MemberSupplier memberSupplier = latestStateMemberSupplierWrapper.wrapMemberSupplier(baseSupplier); + + assertThat(memberSupplier).isInstanceOf(FilteredMemberSupplier.class); + } + + @Test + void given_VersionMaterialisationDisabled_when_wrap_then_ReturnBaseSupplier() { + when(ldioLdesClientProperties.isVersionMaterialisationEnabled()).thenReturn(false); + final MemberSupplier memberSupplier = latestStateMemberSupplierWrapper.wrapMemberSupplier(baseSupplier); + + assertThat(memberSupplier).isSameAs(baseSupplier); + } + + @Test + void given_LatestStateDisabled_when_wrap_then_ReturnBaseSupplier() { + when(ldioLdesClientProperties.isVersionMaterialisationEnabled()).thenReturn(true); + when(ldioLdesClientProperties.isLatestStateEnabled()).thenReturn(false); + final MemberSupplier memberSupplier = latestStateMemberSupplierWrapper.wrapMemberSupplier(baseSupplier); + + assertThat(memberSupplier).isSameAs(baseSupplier); + } + + + +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java index 23d46a6a1..c10495b4e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientITSteps.java @@ -1,7 +1,7 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientPropertyKeys; import be.vlaanderen.informatievlaanderen.ldes.ldio.management.status.ClientStatusService; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; import com.github.tomakehurst.wiremock.WireMockServer; @@ -22,13 +22,13 @@ import java.util.concurrent.atomic.AtomicInteger; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClient.NAME; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.URLS; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientPropertyKeys.URLS; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; import static org.awaitility.Awaitility.await; import static org.mockito.Mockito.*; public class LdioLdesClientITSteps extends LdesClientInIT { - private final static WireMockServer wireMockServer = new WireMockServer(options().port(10101)); + private static final WireMockServer wireMockServer = new WireMockServer(options().port(10101)); private final String pipelineName = "pipelineName"; private final ApplicationEventPublisher applicationEventPublisher = applicationEventPublisher(); private final Map componentPropsMap = new HashMap<>(); @@ -52,7 +52,7 @@ public void iWantToFollowTheFollowingLDES(List urls) { @And("I configure this to be of RDF format {string}") public void iConfigureThisToBeOfRDFFormat(String contentType) { - componentPropsMap.put(LdioLdesClientProperties.SOURCE_FORMAT, contentType); + componentPropsMap.put(LdioLdesClientPropertyKeys.SOURCE_FORMAT, contentType); } @When("^I start an ldes-ldio-in component") diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactoryTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactoryTest.java index 6b8968578..366f2a990 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactoryTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactoryTest.java @@ -1,8 +1,11 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.management.status.ClientStatusConsumer; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.ConfigPropertyMissingException; +import ldes.client.eventstreamproperties.EventStreamPropertiesFetcher; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; import ldes.client.treenodesupplier.filters.LatestStateFilter; import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; @@ -10,32 +13,45 @@ import ldes.client.treenodesupplier.membersuppliers.VersionMaterialisedMemberSupplier; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import java.util.HashMap; import java.util.Map; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientPropertyKeys.*; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.mock; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +@ExtendWith(MockitoExtension.class) class MemberSupplierFactoryTest { private Map defaultInputConfig; - private final ClientStatusConsumer statusConsumer = mock(ClientStatusConsumer.class); + @Mock + private ClientStatusConsumer statusConsumer; + @Mock + private EventStreamPropertiesFetcher fetcher; + private LdioLdesClientProperties ldioLdesClientProperties; + private EventStreamProperties eventStreamProperties; @BeforeEach void setUp() { defaultInputConfig = new HashMap<>(); defaultInputConfig.put(URLS, "http://example.org"); + eventStreamProperties = new EventStreamProperties("http://localhost:8080/collection", "versionOf", "timestamp", "shaclUri"); } @Test void when_VersionMaterialisationIsEnabled_then_VersionMaterialisedMemberSupplierIsReturned() { defaultInputConfig.put(USE_VERSION_MATERIALISATION, "true"); final var componentProperties = new ComponentProperties("pipelineName", "cName", defaultInputConfig); + ldioLdesClientProperties = LdioLdesClientProperties.fromComponentProperties(componentProperties); + when(fetcher.fetchEventStreamProperties(any())).thenReturn(eventStreamProperties); - MemberSupplier memberSupplier = new MemberSupplierFactory(componentProperties, null, statusConsumer).getMemberSupplier(); + MemberSupplier memberSupplier = new MemberSupplierFactory(ldioLdesClientProperties, fetcher, null, statusConsumer).getMemberSupplier(); assertThat(memberSupplier).isInstanceOf(VersionMaterialisedMemberSupplier.class); } @@ -44,16 +60,21 @@ void when_VersionMaterialisationIsEnabled_then_VersionMaterialisedMemberSupplier void when_VersionMaterialisationAndOnlyOnceFilterAreNotEnabled_then_MemberSupplierImplIsReturned() { defaultInputConfig.put(USE_EXACTLY_ONCE_FILTER, "false"); final var componentProperties = new ComponentProperties("pipelineName", "cName", defaultInputConfig); + ldioLdesClientProperties = LdioLdesClientProperties.fromComponentProperties(componentProperties); + when(fetcher.fetchEventStreamProperties(any())).thenReturn(eventStreamProperties); - MemberSupplier memberSupplier = new MemberSupplierFactory(componentProperties, null, statusConsumer).getMemberSupplier(); + MemberSupplier memberSupplier = new MemberSupplierFactory(ldioLdesClientProperties, fetcher, null, statusConsumer).getMemberSupplier(); assertThat(memberSupplier).isInstanceOf(MemberSupplierImpl.class); } + @Test void when_VersionMaterialisationIsNotEnabled_then_OnlyOnceMemberSupplierIsReturned() { final var componentProperties = new ComponentProperties("pipelineName", "cName", defaultInputConfig); + ldioLdesClientProperties = LdioLdesClientProperties.fromComponentProperties(componentProperties); + when(fetcher.fetchEventStreamProperties(any())).thenReturn(eventStreamProperties); - MemberSupplier memberSupplier = new MemberSupplierFactory(componentProperties, null, statusConsumer).getMemberSupplier(); + MemberSupplier memberSupplier = new MemberSupplierFactory(ldioLdesClientProperties, fetcher, null, statusConsumer).getMemberSupplier(); assertThat(memberSupplier).isInstanceOf(FilteredMemberSupplier.class); } @@ -63,8 +84,10 @@ void when_LatestStateFilterIsEnabled_then_returnVersionMaterialisedMemberSupplie defaultInputConfig.put(USE_VERSION_MATERIALISATION, "true"); defaultInputConfig.put(USE_LATEST_STATE_FILTER, "true"); final var componentProperties = new ComponentProperties("pipelineName", "cName", defaultInputConfig); + ldioLdesClientProperties = LdioLdesClientProperties.fromComponentProperties(componentProperties); + when(fetcher.fetchEventStreamProperties(any())).thenReturn(eventStreamProperties); - MemberSupplier memberSupplier = new MemberSupplierFactory(componentProperties, null, statusConsumer).getMemberSupplier(); + MemberSupplier memberSupplier = new MemberSupplierFactory(ldioLdesClientProperties, fetcher, null, statusConsumer).getMemberSupplier(); assertThat(memberSupplier) .isInstanceOf(VersionMaterialisedMemberSupplier.class) @@ -77,8 +100,10 @@ void when_LatestStateFilterIsDisabled_then_returnVersionMaterialisedMemberSuppli defaultInputConfig.put(USE_VERSION_MATERIALISATION, "true"); defaultInputConfig.put(USE_LATEST_STATE_FILTER, "false"); final var componentProperties = new ComponentProperties("pipelineName", "cName", defaultInputConfig); + ldioLdesClientProperties = LdioLdesClientProperties.fromComponentProperties(componentProperties); + when(fetcher.fetchEventStreamProperties(any())).thenReturn(eventStreamProperties); - MemberSupplier memberSupplier = new MemberSupplierFactory(componentProperties, null, statusConsumer).getMemberSupplier(); + MemberSupplier memberSupplier = new MemberSupplierFactory(ldioLdesClientProperties, fetcher, null, statusConsumer).getMemberSupplier(); assertThat(memberSupplier) .isInstanceOf(VersionMaterialisedMemberSupplier.class) @@ -91,8 +116,10 @@ void when_LatestStateFilterIsEnabledWithoutMaterialisation_then_returnMemberSupp defaultInputConfig.put(USE_EXACTLY_ONCE_FILTER, "false"); defaultInputConfig.put(USE_LATEST_STATE_FILTER, "true"); final var componentProperties = new ComponentProperties("pipelineName", "cName", defaultInputConfig); + ldioLdesClientProperties = LdioLdesClientProperties.fromComponentProperties(componentProperties); + when(fetcher.fetchEventStreamProperties(any())).thenReturn(eventStreamProperties); - MemberSupplier memberSupplier = new MemberSupplierFactory(componentProperties, null, statusConsumer).getMemberSupplier(); + MemberSupplier memberSupplier = new MemberSupplierFactory(ldioLdesClientProperties, fetcher, null, statusConsumer).getMemberSupplier(); assertThat(memberSupplier).isInstanceOf(MemberSupplierImpl.class); } @@ -101,7 +128,9 @@ void when_LatestStateFilterIsEnabledWithoutMaterialisation_then_returnMemberSupp void when_NoUrlsAreConfigured_then_ThrowException() { final String expectedErrorMessage = "Pipeline \"pipelineName\": \"cName\" : Missing value for property \"urls\" ."; final var componentProperties = new ComponentProperties("pipelineName", "cName", Map.of("url", "http://localhost:8080/ldes")); - final MemberSupplierFactory memberSupplierFactory = new MemberSupplierFactory(componentProperties, null, statusConsumer); + ldioLdesClientProperties = LdioLdesClientProperties.fromComponentProperties(componentProperties); + + MemberSupplierFactory memberSupplierFactory = new MemberSupplierFactory(ldioLdesClientProperties, fetcher, null, statusConsumer); assertThatThrownBy(memberSupplierFactory::getMemberSupplier) .isInstanceOf(ConfigPropertyMissingException.class) .hasMessage(expectedErrorMessage); diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/VersionMaterialisedMemberSupplierWrapperTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/VersionMaterialisedMemberSupplierWrapperTest.java new file mode 100644 index 000000000..4513b5f12 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/VersionMaterialisedMemberSupplierWrapperTest.java @@ -0,0 +1,49 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers.VersionMaterialisedMemberSupplierWrapper; +import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; +import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; +import ldes.client.treenodesupplier.membersuppliers.VersionMaterialisedMemberSupplier; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class VersionMaterialisedMemberSupplierWrapperTest { + @Mock + private LdioLdesClientProperties ldioLdesClientProperties; + @Mock + private MemberSupplier baseSupplier; + private VersionMaterialisedMemberSupplierWrapper versionMaterialisedMemberSupplierWrapper; + + @BeforeEach + void setUp() { + final EventStreamProperties eventStreamProperties = new EventStreamProperties("test", "test", "test", "test"); + versionMaterialisedMemberSupplierWrapper = new VersionMaterialisedMemberSupplierWrapper(eventStreamProperties, ldioLdesClientProperties); + } + + @Test + void given_VersionMaterialisationEnabled_when_wrap_then_ReturnVersionMaterialisedMemberSupplier() { + when(ldioLdesClientProperties.isVersionMaterialisationEnabled()).thenReturn(true); + + final MemberSupplier memberSupplier = versionMaterialisedMemberSupplierWrapper.wrapMemberSupplier(baseSupplier); + + assertThat(memberSupplier).isInstanceOf(VersionMaterialisedMemberSupplier.class); + } + + @Test + void given_VersionMaterialisationDisabled_when_wrap_then_ReturnBaseMemberSupplier() { + when(ldioLdesClientProperties.isVersionMaterialisationEnabled()).thenReturn(false); + + final MemberSupplier memberSupplier = versionMaterialisedMemberSupplierWrapper.wrapMemberSupplier(baseSupplier); + + assertThat(memberSupplier).isSameAs(baseSupplier); + } + +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-ldes.ttl b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-ldes.ttl new file mode 100644 index 000000000..da6967501 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/__files/items-ldes.ttl @@ -0,0 +1,11 @@ +@prefix tree: . +@prefix rdf: . +@prefix ldes: . +@prefix schema: . +@prefix terms: . +@prefix prov: . + + + rdf:type ldes:EventStream ; + ldes:timestampPath prov:generatedAtTime ; + ldes:versionOfPath terms:isVersionOf . diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature index 6a3617e39..edf8e9617 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/features/ldes-client-in.feature @@ -32,8 +32,6 @@ Feature: LdesClientIntegrationTest And I want to add the following properties | materialisation.enabled | true | | materialisation.enable-latest-state | | - | materialisation.version-of-property | http://purl.org/dc/terms/isVersionOf | - | timestamp-path | http://www.w3.org/ns/prov#generatedAtTime | And I configure this to be of RDF format "application/ld+json" When I start an ldes-ldio-in component Then All members from the stream are passed to the pipeline diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items.json b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items.json new file mode 100644 index 000000000..8f339b456 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/resources/mappings/items.json @@ -0,0 +1,15 @@ +{ + "request": { + "method": "GET", + "url": "/items" + }, + "response": { + "status": 200, + "bodyFileName": "items-ldes.ttl", + "headers": { + "Content-Type": "text/turtle;charset=UTF-8", + "Cache-Control": "public, max-age=604800, immutable" + } + } +} + diff --git a/pom.xml b/pom.xml index 08ce598b1..b91ffc557 100644 --- a/pom.xml +++ b/pom.xml @@ -219,6 +219,11 @@ ldi-infra-sql ${project.version} + + be.vlaanderen.informatievlaanderen.ldes.client + event-stream-properties-fetcher + ${project.version} + From f3fe3592c478e5aeb86519ed7b68236c707c984b Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:51:20 +0200 Subject: [PATCH 15/17] fix: missing redirect handling (#700) --- .../EventStreamPropertiesFetcher.java | 27 +++++++++++++---- .../valueobjects/PropertiesRequest.java | 5 ++++ .../EventStreamPropertiesFetcherTest.java | 29 ++++++++++++++++++- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcher.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcher.java index 367707fda..9a59c5baa 100644 --- a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcher.java +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcher.java @@ -1,6 +1,7 @@ package ldes.client.eventstreamproperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Response; import ldes.client.eventstreamproperties.services.StartingNodeSpecificationFactory; import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; import ldes.client.eventstreamproperties.valueobjects.PropertiesRequest; @@ -28,12 +29,26 @@ public EventStreamProperties fetchEventStreamProperties(PropertiesRequest reques } private EventStreamProperties executePropertiesRequest(PropertiesRequest request) { - return requestExecutor.execute(request.createRequest()).getBody() - .map(ByteArrayInputStream::new) - .map(body -> RDFParser.source(body).lang(request.lang()).toModel()) - .map(StartingNodeSpecificationFactory::fromModel) - .map(StartingNodeSpecification::extractEventStreamProperties) - .orElseThrow(); + final Response response = requestExecutor.execute(request.createRequest()); + + if(response.isOk()) { + return response.getBody() + .map(ByteArrayInputStream::new) + .map(body -> RDFParser.source(body).lang(request.lang()).toModel()) + .map(StartingNodeSpecificationFactory::fromModel) + .map(StartingNodeSpecification::extractEventStreamProperties) + .orElseThrow(); + } + + if(response.isRedirect()) { + return response.getRedirectLocation() + .map(request::withUrl) + .map(this::executePropertiesRequest) + .orElseThrow(() -> new IllegalStateException("No Location Header in redirect.")); + } + + throw new UnsupportedOperationException( + "Cannot handle response " + response.getHttpStatus() + " of EventStreamPropertiesRequest " + request); } diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/PropertiesRequest.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/PropertiesRequest.java index c04cfc406..2fb5482e9 100644 --- a/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/PropertiesRequest.java +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/main/java/ldes/client/eventstreamproperties/valueobjects/PropertiesRequest.java @@ -21,4 +21,9 @@ public Request createRequest() { public PropertiesRequest withUrl(String url) { return new PropertiesRequest(url, lang); } + + @Override + public String toString() { + return "PropertiesRequest{url='%s', lang=%s}".formatted(url, lang); + } } diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcherTest.java b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcherTest.java index 55f2bd276..cabf05349 100644 --- a/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcherTest.java +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/src/test/java/ldes/client/eventstreamproperties/EventStreamPropertiesFetcherTest.java @@ -18,6 +18,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.*; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @WireMockTest(httpPort = EventStreamPropertiesFetcherTest.WIREMOCK_PORT) class EventStreamPropertiesFetcherTest { @@ -32,7 +33,7 @@ class EventStreamPropertiesFetcherTest { @BeforeEach void setUp() { - RequestExecutor requestExecutor = new RequestExecutorFactory().createNoAuthExecutor(); + RequestExecutor requestExecutor = new RequestExecutorFactory(false).createNoAuthExecutor(); fetcher = new EventStreamPropertiesFetcher(requestExecutor); } @@ -50,6 +51,32 @@ void given_EventStream_when_FetchProperties_then_ReturnValidProperties() throws .isEqualTo(eventStreamProperties); } + @Test + void given_EventStreamWitRedirects_when_FetchProperties_then_ReturnValidProperties() throws IOException, URISyntaxException { + URL resource = getClass().getClassLoader().getResource("models/eventstream.ttl"); + final byte[] responseBytes = Files.readAllBytes(Path.of(Objects.requireNonNull(resource).toURI())); + stubFor(get("/observations").willReturn(temporaryRedirect("http://localhost:12121/observations-redirected"))); + stubFor(get("/observations-redirected").willReturn(ok().withBody(responseBytes))); + + final EventStreamProperties properties = fetcher.fetchEventStreamProperties(new PropertiesRequest("http://localhost:12121/observations", Lang.TTL)); + + verify(getRequestedFor(urlEqualTo("/observations"))); + assertThat(properties) + .usingRecursiveComparison() + .isEqualTo(eventStreamProperties); + } + + @Test + void given_NoEventStream_when_FetchProperties_then_ReturnValidProperties() { + stubFor(get("/observations").willReturn(notFound())); + PropertiesRequest request = new PropertiesRequest("http://localhost:12121/observations", Lang.TTL); + + assertThatThrownBy(() -> fetcher.fetchEventStreamProperties(request)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Cannot handle response 404 of EventStreamPropertiesRequest %s", request); + verify(getRequestedFor(urlEqualTo("/observations"))); + } + @Test void given_View_when_FetchProperties_then_ReturnValidProperties() throws IOException, URISyntaxException { URL resource = getClass().getClassLoader().getResource("models/view.ttl"); From be6d1225ffc8d1594b3fba6491a56471a51ca8ab Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:30:32 +0200 Subject: [PATCH 16/17] fix: deadlock in the ldes client connector (#702) --- .../ldes/ldio/LdioLdesClientConnectorApi.java | 11 ++--------- .../config/LdioLdesClientConnectorAutoConfig.java | 3 +-- .../ldes/ldio/LdioLdesClientConnectorTest.java | 3 +-- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java index b89c83611..ec5971543 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorApi.java @@ -3,17 +3,13 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TokenService; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.edc.services.TransferService; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.status.PipelineStatusTrigger.START; - public class LdioLdesClientConnectorApi { private final TransferService transferService; private final TokenService tokenService; - private final LdioLdesClient ldesClient; - public LdioLdesClientConnectorApi(TransferService transferService, TokenService tokenService, LdioLdesClient ldesClient) { + public LdioLdesClientConnectorApi(TransferService transferService, TokenService tokenService) { this.transferService = transferService; this.tokenService = tokenService; - this.ldesClient = ldesClient; } public void handleToken(String token) { @@ -21,10 +17,7 @@ public void handleToken(String token) { } public String handleTransfer(String transfer) { - String response = transferService.startTransfer(transfer).getBodyAsString() - .orElse(""); - ldesClient.updateStatus(START); - return response; + return transferService.startTransfer(transfer).getBodyAsString().orElse(""); } public void shutdown() { diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index e2cfd6150..c7723b3cb 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -71,6 +71,7 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, Appli urlProxy); final var eventStreamPropertiesFetcher = new EventStreamPropertiesFetcher(edcRequestExecutor); final var clientStatusConsumer = new ClientStatusConsumer(pipelineName, clientStatusService); + eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(pipelineName, new LdioLdesClientConnectorApi(transferService, tokenService))); final MemberSupplier memberSupplier = new MemberSupplierFactory( ldioLdesClientProperties, eventStreamPropertiesFetcher, @@ -82,8 +83,6 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, Appli final LdioObserver ldioObserver = LdioObserver.register(NAME, pipelineName, observationRegistry); final var ldesClient = new LdioLdesClient(executor, ldioObserver, memberSupplier, applicationEventPublisher, keepState, clientStatusConsumer); - eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(pipelineName, new LdioLdesClientConnectorApi(transferService, tokenService, ldesClient))); - ldesClient.start(); return ldesClient; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java index 34662b7eb..3f4c7337b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClientConnectorTest.java @@ -39,9 +39,8 @@ class LdioLdesClientConnectorTest { void setup() { transferService = mock(TransferService.class); tokenService = mock(TokenService.class); - final LdioLdesClient ldesClient = mock(LdioLdesClient.class); - eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(endpoint, new LdioLdesClientConnectorApi(transferService, tokenService, ldesClient))); + eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(endpoint, new LdioLdesClientConnectorApi(transferService, tokenService))); } @Test From bf361a9abcddcd3ecdda4f86a85f1cce90a22e4c Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:04:50 +0100 Subject: [PATCH 17/17] feat: http-sparql-out (#705) --- .github/ldio.Dockerfile | 3 +- .../ldio-outputs/ldio-http-sparql-out.md | 40 ++++ .../ldio-skolemisation-transformer.md | 75 +++++++ ldi-core/http-sparql-out/pom.xml | 26 +++ .../ldes/ldi/HttpSparqlOut.java | 53 +++++ .../WriteActionFailedException.java | 7 + .../ldi/factory/DeleteFunctionBuilder.java | 52 +++++ .../ldi/factory/InsertFunctionBuilder.java | 25 +++ .../ldi/skolemisation/EmptySkolemizer.java | 10 + .../ldes/ldi/skolemisation/Skolemizer.java | 8 + .../ldi/skolemisation/SkolemizerImpl.java | 17 ++ .../ldes/ldi/valueobjects/DeleteFunction.java | 32 +++ .../ldes/ldi/valueobjects/InsertFunction.java | 19 ++ .../ldes/ldi/valueobjects/SparqlQuery.java | 27 +++ .../ldes/ldi/HttpSparqlOutTest.java | 80 +++++++ .../factory/DeleteFunctionBuilderTest.java | 114 ++++++++++ .../factory/InsertFunctionBuilderTest.java | 30 +++ .../skolemisation/EmptySkolemizerTest.java | 19 ++ .../ldi/skolemisation/SkolemizerImplTest.java | 34 +++ .../ldi/valueobjects/DeleteFunctionTest.java | 53 +++++ .../ldi/valueobjects/InsertFunctionTest.java | 39 ++++ .../ldi/valueobjects/SparqlQueryTest.java | 49 +++++ .../src/test/resources/jane-doe.nt | 3 + ldi-core/pom.xml | 2 + ldi-core/skolemisation-transformer/pom.xml | 14 ++ .../ldes/ldi/SkolemisationTransformer.java | 19 ++ .../ldes/ldi/services/StatementBuilder.java | 37 ++++ .../ldi/valueobjects/SkolemizedModel.java | 41 ++++ .../ldi/SkolemisationTransformerTest.java | 31 +++ .../ldi/util/SkolemizationConditions.java | 25 +++ .../ldi/valueobjects/SkolemizedModelTest.java | 34 +++ .../src/test/resources/activity.nq | 11 + .../src/test/resources/mob-hind-model.ttl | 24 ++ .../src/test/resources/skolemized-product.nq | 13 ++ ldi-nifi/ldi-nifi-common/pom.xml | 5 + .../config/RequestExecutorProperties.java | 155 +++++++++++++ .../services/RequestExecutorSupplier.java | 45 ++++ .../http-sparql-out-processor/pom.xml | 33 +++ .../processors/HttpSparqlOutProcessor.java | 116 ++++++++++ .../HttpSparqlOutProcessorProperties.java | 100 +++++++++ .../org.apache.nifi.processor.Processor | 1 + .../HttpSparqlOutProcessorTest.java | 207 ++++++++++++++++++ .../src/test/resources/mob-hind-model.ttl | 24 ++ .../ldi/processors/LdesClientProcessor.java | 40 +--- .../config/LdesProcessorProperties.java | 139 ------------ .../config/LdesProcessorPropertiesTest.java | 14 +- .../ldi-processors-bundle/pom.xml | 12 + ldi-nifi/ldi-nifi-processors/pom.xml | 2 + .../pom.xml | 28 +++ .../SkolemisationTransformerProcessor.java | 62 ++++++ .../SkolemisationTransformerProperties.java | 23 ++ .../org.apache.nifi.processor.Processor | 1 + ...SkolemisationTransformerProcessorTest.java | 104 +++++++++ .../src/test/resources/mob-hind-model.ttl | 24 ++ ldi-orchestrator/ldio-application/pom.xml | 14 +- .../pom.xml | 22 +- .../ldes/ldio/LdioHttpSparqlOut.java | 27 +++ .../config/LdioHttpSparqlOutAutoConfig.java | 68 ++++++ .../config/LdioHttpSparqlOutProperties.java | 52 +++++ .../ldes/ldio/LdioHttpSparqlOutTest.java | 145 ++++++++++++ .../LdioHttpSparqlOutPropertiesTest.java | 117 ++++++++++ .../src/test/resources/mob-hind-model.ttl | 24 ++ .../ldio-skolemisation-transformer/pom.xml | 58 +++++ .../config/LdioSkolemisationTransformer.java | 19 ++ ...dioSkolemisationTransformerAutoConfig.java | 27 +++ .../LdioSkolemisationTransformerTest.java | 28 +++ ...kolemisationTransformerAutoConfigTest.java | 42 ++++ .../ldes/ldio/LdioVirtuosoOut.java | 60 ----- .../config/LdioVirtuosoOutAutoConfig.java | 35 --- ldi-orchestrator/ldio-connectors/pom.xml | 3 +- 70 files changed, 2653 insertions(+), 289 deletions(-) create mode 100644 docs/_ldio/ldio-outputs/ldio-http-sparql-out.md create mode 100644 docs/_ldio/ldio-transformers/ldio-skolemisation-transformer.md create mode 100644 ldi-core/http-sparql-out/pom.xml create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HttpSparqlOut.java create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/exceptions/WriteActionFailedException.java create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/DeleteFunctionBuilder.java create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/InsertFunctionBuilder.java create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/EmptySkolemizer.java create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/Skolemizer.java create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/SkolemizerImpl.java create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/DeleteFunction.java create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/InsertFunction.java create mode 100644 ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SparqlQuery.java create mode 100644 ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HttpSparqlOutTest.java create mode 100644 ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/DeleteFunctionBuilderTest.java create mode 100644 ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/InsertFunctionBuilderTest.java create mode 100644 ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/EmptySkolemizerTest.java create mode 100644 ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/SkolemizerImplTest.java create mode 100644 ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/DeleteFunctionTest.java create mode 100644 ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/InsertFunctionTest.java create mode 100644 ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SparqlQueryTest.java create mode 100644 ldi-core/http-sparql-out/src/test/resources/jane-doe.nt create mode 100644 ldi-core/skolemisation-transformer/pom.xml create mode 100644 ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SkolemisationTransformer.java create mode 100644 ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/StatementBuilder.java create mode 100644 ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SkolemizedModel.java create mode 100644 ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SkolemisationTransformerTest.java create mode 100644 ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/util/SkolemizationConditions.java create mode 100644 ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SkolemizedModelTest.java create mode 100644 ldi-core/skolemisation-transformer/src/test/resources/activity.nq create mode 100644 ldi-core/skolemisation-transformer/src/test/resources/mob-hind-model.ttl create mode 100644 ldi-core/skolemisation-transformer/src/test/resources/skolemized-product.nq create mode 100644 ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/RequestExecutorProperties.java create mode 100644 ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/services/RequestExecutorSupplier.java create mode 100644 ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/pom.xml create mode 100644 ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HttpSparqlOutProcessor.java create mode 100644 ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/HttpSparqlOutProcessorProperties.java create mode 100644 ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor create mode 100644 ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HttpSparqlOutProcessorTest.java create mode 100644 ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/test/resources/mob-hind-model.ttl create mode 100644 ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/pom.xml create mode 100644 ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/SkolemisationTransformerProcessor.java create mode 100644 ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/SkolemisationTransformerProperties.java create mode 100644 ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor create mode 100644 ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/SkolemisationTransformerProcessorTest.java create mode 100644 ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/test/resources/mob-hind-model.ttl rename ldi-orchestrator/ldio-connectors/{ldio-virtuoso-out => ldio-http-sparql-out}/pom.xml (54%) create mode 100644 ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpSparqlOut.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutAutoConfig.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutProperties.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpSparqlOutTest.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutPropertiesTest.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/resources/mob-hind-model.ttl create mode 100644 ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/pom.xml create mode 100644 ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSkolemisationTransformer.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioSkolemisationTransformerAutoConfig.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSkolemisationTransformerTest.java create mode 100644 ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioSkolemisationTransformerAutoConfigTest.java delete mode 100644 ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVirtuosoOut.java delete mode 100644 ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVirtuosoOutAutoConfig.java diff --git a/.github/ldio.Dockerfile b/.github/ldio.Dockerfile index 1fb41b6ab..294bd8082 100644 --- a/.github/ldio.Dockerfile +++ b/.github/ldio.Dockerfile @@ -27,12 +27,13 @@ COPY ./ldi-orchestrator/ldio-connectors/ldio-version-object-creator/target/ldio- COPY ./ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/target/ldio-geojson-to-wkt-jar-with-dependencies.jar ./lib/ COPY ./ldi-orchestrator/ldio-connectors/ldio-http-enricher/target/ldio-http-enricher-jar-with-dependencies.jar ./lib/ COPY ./ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/target/ldio-change-detection-filter-jar-with-dependencies.jar ./lib/ +COPY ./ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/target/ldio-skolemisation-transformer-jar-with-dependencies.jar ./lib/ COPY ./ldi-orchestrator/ldio-connectors/ldio-console-out/target/ldio-console-out-jar-with-dependencies.jar ./lib/ COPY ./ldi-orchestrator/ldio-connectors/ldio-http-out/target/ldio-http-out-jar-with-dependencies.jar ./lib/ COPY ./ldi-orchestrator/ldio-connectors/ldio-noop-out/target/ldio-noop-out-jar-with-dependencies.jar ./lib/ COPY ./ldi-orchestrator/ldio-connectors/ldio-repository-sink/target/ldio-repository-sink-jar-with-dependencies.jar ./lib/ -COPY ./ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/target/ldio-virtuoso-out-jar-with-dependencies.jar ./lib/ +COPY ./ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/target/ldio-http-sparql-out-jar-with-dependencies.jar ./lib/ RUN mkdir "state" diff --git a/docs/_ldio/ldio-outputs/ldio-http-sparql-out.md b/docs/_ldio/ldio-outputs/ldio-http-sparql-out.md new file mode 100644 index 000000000..99caad09d --- /dev/null +++ b/docs/_ldio/ldio-outputs/ldio-http-sparql-out.md @@ -0,0 +1,40 @@ +--- +layout: default +parent: LDIO Outputs +title: HTTP Sparql Out +--- + +# HTTP Sparql Out + +***Ldio:HttpSparqlOut*** + +The HTTP SPARQL Out component can be used to write data to a SPARQL host, with Virtuoso as the most common known one. + +## Config + +| Property | Description | Required | Default | Example | Supported values | +|:-----------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------|:---------|:--------|:-----------------------------------------|:-----------------| +| _endpoint_ | The url of the sparql host | Yes | N/A | http://localhost:8890/sparql | URL | +| _graph_ | The graph whereto must be written | No | N/A | http://example.graph.com | String | +| _skolemisation.skolemDomain_ | If the skolem domain is set, skolemisation will be triggered before the triples are written to the sparql host | No | N/A | http://example.org | Any valid IRI | +| _replacement.enabled_ | Whether the old nodes must be replaced by the new ones | No | true | false | Boolean value | +| _replacement.depth_ | How deep the default delete query must delete nested nodes from the existing subject, will be ignored if `replacement.deleteFunction`is set | No | 10 | 15 | Integer | +| _replacement.deleteFunction_ | If this property is set, then the default delete function will be overridden with this delete function | No | N/A | `DELETE { ?s ?p ?o} WHERE { ?s ?p ?o }` | String | + +{% include ldio-core/http-requester.md %} + +### Replacement + +Replacement includes that all old nodes from certain subjects must be deleted before the new nodes with the same subject +can be inserted. \ +By default, a delete query is constructed by the service that delete all nodes, including nested nodes to a level, +specified by the `replacement.depth` property, deep. If for some reason, the constructed delete query is not sufficient, +or the query is too complex, a custom delete query can be configured. This query will override the default query created +by the service, which also mean the `replacement.depth` property will be ignored. + +### Skolemisation + +Not all sparql hosts can deal that well with blank nodes, therefore, those nodes can first be skolemised. However, to +skolemise nodes, a skolem domain is required, which can be set by the `skolemisation.skolemDomain` property, which +directly enables the service. More information about skolemisation can be found on +the [skolemisation-transformer page](./../ldio-transformers/ldio-skolemisation-transformer) \ No newline at end of file diff --git a/docs/_ldio/ldio-transformers/ldio-skolemisation-transformer.md b/docs/_ldio/ldio-transformers/ldio-skolemisation-transformer.md new file mode 100644 index 000000000..11b402351 --- /dev/null +++ b/docs/_ldio/ldio-transformers/ldio-skolemisation-transformer.md @@ -0,0 +1,75 @@ +--- +layout: default +parent: LDIO Transformers +title: Skolemisation Transformer +--- + +# LDIO Skolemisation Transformer + +***Ldio:SkolemisationTransformer*** + +A transformer which skolemises the incoming model. + +## What is Skolemisation + +In the context of Linked Data, Skolemisation is a process used to handle blank nodes or anonymous nodes in RDF +(Resource Description Framework) graphs. +These nodes, which lack unique identifiers, are frequently employed in RDF/S knowledge bases to represent complex +attributes or resources with known properties but unknown identities. + +Skolemisation in Linked Data involves the transformation of these blank nodes into Skolem Uniform Resource Identifiers ( +URIs). +The process enhances the clarity makes it easier to reference these nodes in future datasets. + +This process is particularly useful when dealing with substantial volumes of unstructured data distributed across +diverse sources. +By improving the accuracy and relevance of RDF summaries in relation to original datasets, +Skolemisation enhances the efficiency and effectiveness of subsequent queries against these summaries. + +In summary, Skolemisation in Linked Data provides a way to handle the complexity introduced by blank nodes in RDF +graphs, +thereby enhancing the clarity, interoperability, and usability of the data. + +### Example + +Suppose we have the following RDF triples with a blank node represented as `_:`: + +``` +_:bnode "The Lord of the Rings" . +_:bnode "J.R.R. Tolkien" . +``` + +In this example, `_:` is a blank node that represents a resource (a book in this case) with known properties (title and +creator) but an unknown identity. + +Through Skolemisation, we can replace the blank node with a Skolem URI. +The Skolem URI is typically a URL that is unique to the blank node and is generated by the system handling the RDF data. +Here’s how it might look: + +``` + "The Lord of the Rings" . + "J.R.R. Tolkien" . +``` + +In this Skolemized version, the blank node has been replaced with the Skolem +URI http://example.com/.well-known/genid/123456. +This URI is unique to the resource previously represented by the blank node, and can now be used to reference this +resource in other datasets. +This is a simple example, but it illustrates the basic process of Skolemisation in Linked Data. + +## Config + +| Property | Description | Required | Default | Example | Supported values | +|----------------|----------------------|----------|---------|--------------------|------------------| +| _skolemDomain_ | Skolemisation domain | true | N/A | http://example.com | Any valid URI | + +### Configuration + +The YAML configuration of this example would be as follows: + +```yaml +transformers: + - name: Ldio:SkolemisationTransformer + config: + skolemDomain: http://example.com +``` \ No newline at end of file diff --git a/ldi-core/http-sparql-out/pom.xml b/ldi-core/http-sparql-out/pom.xml new file mode 100644 index 000000000..612fc92e1 --- /dev/null +++ b/ldi-core/http-sparql-out/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-core + 2.10.0-SNAPSHOT + + + http-sparql-out + + + + be.vlaanderen.informatievlaanderen.ldes.ldi + request-executor + ${project.version} + + + be.vlaanderen.informatievlaanderen.ldes.ldi + skolemisation-transformer + ${project.version} + + + \ No newline at end of file diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HttpSparqlOut.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HttpSparqlOut.java new file mode 100644 index 000000000..8e9573290 --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HttpSparqlOut.java @@ -0,0 +1,53 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.exceptions.WriteActionFailedException; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.PostRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeader; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeaders; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Response; +import be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation.Skolemizer; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.SparqlQuery; +import org.apache.http.HttpHeaders; +import org.apache.jena.rdf.model.Model; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class HttpSparqlOut { + private static final Logger log = LoggerFactory.getLogger(HttpSparqlOut.class); + private final String endpoint; + private final SparqlQuery sparqlQuery; + private final Skolemizer skolemizer; + private final RequestExecutor requestExecutor; + + public HttpSparqlOut(String endpoint, SparqlQuery sparqlQuery, Skolemizer skolemizer, RequestExecutor requestExecutor) { + this.endpoint = endpoint; + this.sparqlQuery = sparqlQuery; + this.skolemizer = skolemizer; + this.requestExecutor = requestExecutor; + } + + public void write(Model model) { + if (model.isEmpty()) { + return; + } + + String query = sparqlQuery.getQueryForModel(skolemizer.skolemize(model)); + + final PostRequest request = new PostRequest(endpoint, new RequestHeaders(List.of( + new RequestHeader(HttpHeaders.CONTENT_TYPE, "application/sparql-update"), + new RequestHeader(HttpHeaders.ACCEPT, "application/json"))), query); + synchronized (requestExecutor) { + Response response = requestExecutor.execute(request); + if (response.isSuccess()) { + log.debug("{} {} {}", request.getMethod(), request.getUrl(), response.getHttpStatus()); + } else { + final String message = "Failed to post model. The request url was %s. The http response obtained from the server has code %s and body \"%s\"." + .formatted(response.getRequestedUrl(), response.getHttpStatus(), response.getBodyAsString().orElse(null)); + throw new WriteActionFailedException(message); + } + } + } +} diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/exceptions/WriteActionFailedException.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/exceptions/WriteActionFailedException.java new file mode 100644 index 000000000..2ff9c7ad7 --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/exceptions/WriteActionFailedException.java @@ -0,0 +1,7 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.exceptions; + +public class WriteActionFailedException extends RuntimeException { + public WriteActionFailedException(String message) { + super(message); + } +} diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/DeleteFunctionBuilder.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/DeleteFunctionBuilder.java new file mode 100644 index 000000000..50404f482 --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/DeleteFunctionBuilder.java @@ -0,0 +1,52 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.factory; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.DeleteFunction; + +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class DeleteFunctionBuilder { + private static final String BASE_TEMPLATE = "DELETE %s { ?s ?p ?o } WHERE { %%s }"; + private final String base; + + private DeleteFunctionBuilder(String base) { + this.base = base; + } + + public static DeleteFunction disabled() { + return DeleteFunction.empty(); + } + + public static DeleteFunctionBuilder create() { + return new DeleteFunctionBuilder(String.format(BASE_TEMPLATE, "")); + } + + public static DeleteFunctionBuilder withGraph(String graph) { + final String graphBase = "FROM <%s>".formatted(graph); + return new DeleteFunctionBuilder(String.format(BASE_TEMPLATE, graphBase)); + } + + public DeleteFunction withDepth(int depth) { + if (depth < 0) { + throw new IllegalArgumentException("Depth must be a positive number"); + } + final String whereConditions = IntStream.range(1, depth + 1) + .mapToObj(DeleteFunctionBuilder::createUnionClause) + .collect(Collectors.joining("", """ + { + ?o0 ?p ?o . + BIND (?o0 AS ?s) + } + """, "FILTER (?o0 IN (%s))")); + final String query = base.formatted(whereConditions); + return DeleteFunction.ofQuery(query); + } + + private static String createUnionClause(int length) { + return IntStream.range(0, length) + .mapToObj(i -> "?o%d ?p%d ?o%d . ".formatted(i, i + 1, i + 1)) + .collect(Collectors.joining("", + " UNION { ", + "?o%d ?p ?o . BIND (?o%d AS ?s)}".formatted(length, length))); + } +} diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/InsertFunctionBuilder.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/InsertFunctionBuilder.java new file mode 100644 index 000000000..a8867b5fb --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/InsertFunctionBuilder.java @@ -0,0 +1,25 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.factory; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.InsertFunction; + +public class InsertFunctionBuilder { + private static final String BASE_TEMPLATE = "INSERT DATA { %s }"; + private final String query; + + private InsertFunctionBuilder(String query) { + this.query = query; + } + + public static InsertFunctionBuilder create() { + return new InsertFunctionBuilder(BASE_TEMPLATE); + } + + public static InsertFunctionBuilder withGraph(String graph) { + final String graphBase = "GRAPH <%s> { %%s }".formatted(graph); + return new InsertFunctionBuilder(String.format(BASE_TEMPLATE, graphBase)); + } + + public InsertFunction build() { + return new InsertFunction(query); + } +} diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/EmptySkolemizer.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/EmptySkolemizer.java new file mode 100644 index 000000000..3301874db --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/EmptySkolemizer.java @@ -0,0 +1,10 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation; + +import org.apache.jena.rdf.model.Model; + +public class EmptySkolemizer implements Skolemizer { + @Override + public Model skolemize(Model model) { + return model; + } +} diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/Skolemizer.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/Skolemizer.java new file mode 100644 index 000000000..eb3b096a6 --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/Skolemizer.java @@ -0,0 +1,8 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation; + +import org.apache.jena.rdf.model.Model; + +@FunctionalInterface +public interface Skolemizer { + Model skolemize(Model model); +} diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/SkolemizerImpl.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/SkolemizerImpl.java new file mode 100644 index 000000000..167a50e61 --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/SkolemizerImpl.java @@ -0,0 +1,17 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.SkolemisationTransformer; +import org.apache.jena.rdf.model.Model; + +public class SkolemizerImpl implements Skolemizer { + private final SkolemisationTransformer skolemisationTransformer; + + public SkolemizerImpl(SkolemisationTransformer skolemisationTransformer) { + this.skolemisationTransformer = skolemisationTransformer; + } + + @Override + public Model skolemize(Model model) { + return skolemisationTransformer.transform(model); + } +} diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/DeleteFunction.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/DeleteFunction.java new file mode 100644 index 000000000..09b3ae031 --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/DeleteFunction.java @@ -0,0 +1,32 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class DeleteFunction { + private final String query; + + private DeleteFunction(String query) { + this.query = query; + } + + public Optional createQueryForResources(List subjects) { + final String joinedSubjects = subjects.stream() + .map("<%s>"::formatted) + .collect(Collectors.joining(",")); + return Optional.ofNullable(query).map(q -> q.formatted(joinedSubjects)); + } + + public Optional createQueryForResources(String... subjects) { + return createQueryForResources(List.of(subjects)); + } + + public static DeleteFunction empty() { + return new DeleteFunction(null); + } + + public static DeleteFunction ofQuery(String query) { + return new DeleteFunction(query); + } +} diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/InsertFunction.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/InsertFunction.java new file mode 100644 index 000000000..b674e1076 --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/InsertFunction.java @@ -0,0 +1,19 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFWriter; + +public class InsertFunction { + private final String query; + + public InsertFunction(String query) { + this.query = query; + } + + public String createQuery(Model model) { + final String triples = RDFWriter.source(model).lang(Lang.NQUADS).asString(); + return query.formatted(triples); + } + +} diff --git a/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SparqlQuery.java b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SparqlQuery.java new file mode 100644 index 000000000..55223cd96 --- /dev/null +++ b/ldi-core/http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SparqlQuery.java @@ -0,0 +1,27 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; + +import java.util.List; + +public class SparqlQuery { + private final InsertFunction insertFunction; + private final DeleteFunction deleteFunction; + + public SparqlQuery(InsertFunction insertFunction, DeleteFunction deleteFunction) { + this.insertFunction = insertFunction; + this.deleteFunction = deleteFunction; + } + + public String getQueryForModel(Model model) { + final List subjects = model.listSubjects() + .filterDrop(RDFNode::isAnon) + .mapWith(RDFNode::asResource) + .mapWith(Resource::getURI) + .toList(); + + return deleteFunction.createQueryForResources(subjects).orElse("") + "\n" + insertFunction.createQuery(model); + } +} diff --git a/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HttpSparqlOutTest.java b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HttpSparqlOutTest.java new file mode 100644 index 000000000..4c3f14e22 --- /dev/null +++ b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HttpSparqlOutTest.java @@ -0,0 +1,80 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.exceptions.WriteActionFailedException; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.PostRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeaders; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Response; +import be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation.Skolemizer; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.SparqlQuery; +import org.apache.jena.rdf.model.Model; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class HttpSparqlOutTest { + private static final String ENDPOINT = "http://example.com/sparql"; + @Mock + private SparqlQuery sparqlQuery; + @Mock + private Skolemizer skolemizer; + @Mock + private RequestExecutor requestExecutor; + @Mock + private Model model; + private HttpSparqlOut httpSparqlOut; + + @BeforeEach + void setUp() { + httpSparqlOut = new HttpSparqlOut(ENDPOINT, sparqlQuery, skolemizer, requestExecutor); + } + + @Test + void given_EmptyModel_when_Write_then_DoNothing() { + when(model.isEmpty()).thenReturn(true); + + httpSparqlOut.write(model); + + verifyNoInteractions(sparqlQuery, skolemizer, requestExecutor); + } + + @Test + void test_Write() { + Response response = mock(); + when(model.isEmpty()).thenReturn(false); + when(skolemizer.skolemize(model)).thenReturn(model); + when(sparqlQuery.getQueryForModel(model)).thenReturn("my-sparql-query"); + when(requestExecutor.execute(any())).thenReturn(response); + when(response.isSuccess()).thenReturn(true); + + httpSparqlOut.write(model); + + verify(sparqlQuery).getQueryForModel(model); + verify(skolemizer).skolemize(model); + verify(requestExecutor).execute(any()); + } + + @Test + void test_Write_when_BadRequestIsReturned() { + when(model.isEmpty()).thenReturn(false); + when(skolemizer.skolemize(model)).thenReturn(model); + when(sparqlQuery.getQueryForModel(model)).thenReturn("my-sparql-query"); + when(requestExecutor.execute(any())).thenReturn(new Response(new PostRequest(ENDPOINT, new RequestHeaders(List.of()), "my-sparql-query"), List.of(), 400, "error")); + + assertThatThrownBy(() -> httpSparqlOut.write(model)) + .isExactlyInstanceOf(WriteActionFailedException.class) + .hasMessage("Failed to post model. The request url was %s. The http response obtained from the server has code %s and body \"%s\".".formatted(ENDPOINT, 400, "error")); + verify(sparqlQuery).getQueryForModel(model); + verify(skolemizer).skolemize(model); + verify(requestExecutor).execute(any()); + } +} \ No newline at end of file diff --git a/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/DeleteFunctionBuilderTest.java b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/DeleteFunctionBuilderTest.java new file mode 100644 index 000000000..fd9e0a5a1 --- /dev/null +++ b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/DeleteFunctionBuilderTest.java @@ -0,0 +1,114 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.factory; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.DeleteFunction; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +class DeleteFunctionBuilderTest { + + @Test + void test_FromDepth1() { + final String expected = """ + DELETE { ?s ?p ?o } + WHERE { + { + ?o0 ?p ?o . + BIND (?o0 AS ?s) + } + UNION { + ?o0 ?p1 ?o1 . + ?o1 ?p ?o . + BIND (?o1 AS ?s) + } + FILTER (?o0 IN (, )) + }"""; + + final DeleteFunction query = DeleteFunctionBuilder.create().withDepth(1); + + assertThat(query.createQueryForResources("http://example.com", "http://example.org")) + .get(InstanceOfAssertFactories.STRING) + .isEqualToIgnoringWhitespace(expected); + } + + @Test + void test_FromDepth3() { + final String expected = """ + DELETE { ?s ?p ?o } + WHERE { + { + ?o0 ?p ?o . + BIND (?o0 AS ?s) + } + UNION { + ?o0 ?p1 ?o1 . + ?o1 ?p ?o . + BIND (?o1 AS ?s) + } + UNION { + ?o0 ?p1 ?o1 . + ?o1 ?p2 ?o2 . + ?o2 ?p ?o . + BIND (?o2 AS ?s) + } + UNION { + ?o0 ?p1 ?o1 . + ?o1 ?p2 ?o2 . + ?o2 ?p3 ?o3 . + ?o3 ?p ?o . + BIND (?o3 AS ?s) + } + FILTER (?o0 IN ()) + }"""; + + final DeleteFunction query = DeleteFunctionBuilder.create().withDepth(3); + + assertThat(query.createQueryForResources("http://example.com")) + .get(InstanceOfAssertFactories.STRING) + .isEqualToIgnoringWhitespace(expected); + } + + @Test + void test_Disabled() { + final DeleteFunction query = DeleteFunctionBuilder.disabled(); + + assertThat(query) + .usingRecursiveComparison() + .isEqualTo(DeleteFunction.empty()); + } + + @Test + void given_Graph_test_FromDepth1() { + final String expected = """ + DELETE FROM { + ?s ?p ?o + } + WHERE { + { + ?o0 ?p ?o . + BIND (?o0 AS ?s) + } + UNION { + ?o0 ?p1 ?o1 . + ?o1 ?p ?o . + BIND (?o1 AS ?s) + } + FILTER (?o0 IN ()) + }"""; + + final DeleteFunction query = DeleteFunctionBuilder.withGraph("http://example.org/graph").withDepth(1); + + assertThat(query.createQueryForResources("http://example.com")) + .get(InstanceOfAssertFactories.STRING) + .isEqualToIgnoringWhitespace(expected); + } + + @Test + void given_DepthIsNegative_when_Build_then_ThrowException() { + assertThatIllegalArgumentException() + .isThrownBy(() -> DeleteFunctionBuilder.create().withDepth(-1)) + .withMessage("Depth must be a positive number"); + } +} \ No newline at end of file diff --git a/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/InsertFunctionBuilderTest.java b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/InsertFunctionBuilderTest.java new file mode 100644 index 000000000..1b6fcd906 --- /dev/null +++ b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/factory/InsertFunctionBuilderTest.java @@ -0,0 +1,30 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.factory; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.InsertFunction; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class InsertFunctionBuilderTest { + @Test + void test_Build() { + final InsertFunction expected = new InsertFunction("INSERT DATA { %s }"); + + final InsertFunction result = InsertFunctionBuilder.create().build(); + + assertThat(result) + .usingRecursiveComparison() + .isEqualTo(expected); + } + + @Test + void given_Graph_test_Build() { + final InsertFunction expected = new InsertFunction("INSERT DATA { GRAPH { %s } }"); + + final InsertFunction result = InsertFunctionBuilder.withGraph("http://example.com").build(); + + assertThat(result) + .usingRecursiveComparison() + .isEqualTo(expected); + } +} \ No newline at end of file diff --git a/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/EmptySkolemizerTest.java b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/EmptySkolemizerTest.java new file mode 100644 index 000000000..61e53f2dc --- /dev/null +++ b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/EmptySkolemizerTest.java @@ -0,0 +1,19 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class EmptySkolemizerTest { + private final Skolemizer skolemizer = new EmptySkolemizer(); + @Test + void test_Skolemize() { + final Model model = ModelFactory.createDefaultModel(); + + final Model result = skolemizer.skolemize(model); + + assertThat(result).isSameAs(model); + } +} \ No newline at end of file diff --git a/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/SkolemizerImplTest.java b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/SkolemizerImplTest.java new file mode 100644 index 000000000..19735a608 --- /dev/null +++ b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/skolemisation/SkolemizerImplTest.java @@ -0,0 +1,34 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.SkolemisationTransformer; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SkolemizerImplTest { + @Mock + private SkolemisationTransformer skolemisationTransformer; + @InjectMocks + private SkolemizerImpl skolemizer; + + @Test + void test_Skolemize() { + final Model input = ModelFactory.createDefaultModel(); + final Model output = ModelFactory.createDefaultModel(); + when(skolemisationTransformer.transform(input)).thenReturn(output); + + final Model result = skolemizer.skolemize(input); + + assertThat(result) + .isSameAs(output) + .isNotSameAs(input); + } +} \ No newline at end of file diff --git a/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/DeleteFunctionTest.java b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/DeleteFunctionTest.java new file mode 100644 index 000000000..6760cf397 --- /dev/null +++ b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/DeleteFunctionTest.java @@ -0,0 +1,53 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects; + +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +class DeleteFunctionTest { + @Test + void test_GetQueryForResource() { + final DeleteFunction query = DeleteFunction.ofQuery("DELETE { ?s ?p ?o } WHERE { ?s ?p ?o FILTER (?s IN (%s)) }"); + + final Optional queryForResource = query.createQueryForResources("http://example.com"); + + assertThat(queryForResource) + .get(InstanceOfAssertFactories.STRING) + .isEqualToIgnoringWhitespace("DELETE { ?s ?p ?o } WHERE { ?s ?p ?o FILTER (?s IN ()) }"); + } + + @Test + void test_GetQueryForResources() { + final DeleteFunction query = DeleteFunction.ofQuery("DELETE { ?s ?p ?o } WHERE { ?s ?p ?o FILTER (?s IN (%s)) }"); + + final Optional queryForResource = query.createQueryForResources("http://example.com", "http://example.org"); + + assertThat(queryForResource) + .get(InstanceOfAssertFactories.STRING) + .isEqualToIgnoringWhitespace("DELETE { ?s ?p ?o } WHERE { ?s ?p ?o FILTER (?s IN (,)) }"); + + } + + @Test + void given_queryWithoutTemplate_test_GetQueryForResource() { + final DeleteFunction query = DeleteFunction.ofQuery("DELETE { ?s ?p ?o } WHERE { ?s ?p ?o VALUES ?s { } }"); + + final Optional queryForResource = query.createQueryForResources("https://fantasy.com"); + + assertThat(queryForResource) + .get(InstanceOfAssertFactories.STRING) + .isEqualToIgnoringWhitespace("DELETE { ?s ?p ?o } WHERE { ?s ?p ?o VALUES ?s { } }"); + } + + @Test + void given_EmptyDeleteFunction() { + final DeleteFunction query = DeleteFunction.empty(); + + final Optional queryForResource = query.createQueryForResources("https://fantasy.com"); + + assertThat(queryForResource).isEmpty(); + } +} \ No newline at end of file diff --git a/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/InsertFunctionTest.java b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/InsertFunctionTest.java new file mode 100644 index 000000000..6237ba66a --- /dev/null +++ b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/InsertFunctionTest.java @@ -0,0 +1,39 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParser; +import org.apache.jena.update.UpdateAction; +import org.assertj.core.api.Condition; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class InsertFunctionTest { + private static final String QUERY = "INSERT DATA { %s }"; + private InsertFunction insertFunction; + + @BeforeEach + void setUp() { + insertFunction = new InsertFunction(QUERY); + } + + @Test + void test_createQuery() { + final Model input = RDFParser.source("jane-doe.nt").lang(Lang.NT).toModel(); + + final String result = insertFunction.createQuery(input); + + assertThat(result).is(queryIsomorphicWith(input)); + } + + private Condition queryIsomorphicWith(Model model) { + return new Condition<>(query -> { + Model output = ModelFactory.createDefaultModel(); + UpdateAction.parseExecute(query, output); + return model.isIsomorphicWith(output); + }, "INSERT query must be isomorphic with expected model"); + } +} \ No newline at end of file diff --git a/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SparqlQueryTest.java b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SparqlQueryTest.java new file mode 100644 index 000000000..fd0c58824 --- /dev/null +++ b/ldi-core/http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SparqlQueryTest.java @@ -0,0 +1,49 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.factory.DeleteFunctionBuilder; +import be.vlaanderen.informatievlaanderen.ldes.ldi.factory.InsertFunctionBuilder; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParser; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class SparqlQueryTest { + private SparqlQuery sparqlQuery; + + @BeforeEach + void setUp() { + sparqlQuery = new SparqlQuery( + InsertFunctionBuilder.create().build(), + DeleteFunctionBuilder.create().withDepth(0) + ); + } + + @Test + void test() { + final Model input = RDFParser.create().fromString(singleTriple()).lang(Lang.NT).toModel(); + + final String result = sparqlQuery.getQueryForModel(input); + + assertThat(result).isEqualToIgnoringWhitespace(expectedQuery()); + } + + private static String expectedQuery() { + return """ + DELETE { ?s ?p ?o } + WHERE { + { + ?o0 ?p ?o . + BIND (?o0 AS ?s) + } + FILTER (?o0 IN ()) + } + INSERT DATA { "Jane Doe" . }"""; + } + + private String singleTriple() { + return " \"Jane Doe\" ."; + } +} \ No newline at end of file diff --git a/ldi-core/http-sparql-out/src/test/resources/jane-doe.nt b/ldi-core/http-sparql-out/src/test/resources/jane-doe.nt new file mode 100644 index 000000000..fa4669c08 --- /dev/null +++ b/ldi-core/http-sparql-out/src/test/resources/jane-doe.nt @@ -0,0 +1,3 @@ + . + "Jane Doe" . + "Professor" . \ No newline at end of file diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index 88d19139c..18120d487 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -27,6 +27,8 @@ ldi-common change-detection-filter ldi-infra-sql + skolemisation-transformer + http-sparql-out diff --git a/ldi-core/skolemisation-transformer/pom.xml b/ldi-core/skolemisation-transformer/pom.xml new file mode 100644 index 000000000..8171912dd --- /dev/null +++ b/ldi-core/skolemisation-transformer/pom.xml @@ -0,0 +1,14 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-core + 2.10.0-SNAPSHOT + + + skolemisation-transformer + + \ No newline at end of file diff --git a/ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SkolemisationTransformer.java b/ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SkolemisationTransformer.java new file mode 100644 index 000000000..44f20fabe --- /dev/null +++ b/ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SkolemisationTransformer.java @@ -0,0 +1,19 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOneToOneTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.SkolemizedModel; +import org.apache.jena.rdf.model.Model; + +public class SkolemisationTransformer implements LdiOneToOneTransformer { + public static final String SKOLEM_URI = "/.well-known/genid/"; + private final String skolemUriTemplate; + + public SkolemisationTransformer(String skolemDomain) { + this.skolemUriTemplate = skolemDomain + SKOLEM_URI + "%s"; + } + + @Override + public Model transform(Model model) { + return new SkolemizedModel(skolemUriTemplate, model).getModel(); + } +} diff --git a/ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/StatementBuilder.java b/ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/StatementBuilder.java new file mode 100644 index 000000000..c992effae --- /dev/null +++ b/ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/services/StatementBuilder.java @@ -0,0 +1,37 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.services; + +import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.rdf.model.impl.StatementImpl; + +public class StatementBuilder { + private final Property predicate; + private Resource subject; + private RDFNode object; + + private StatementBuilder(Property predicate) { + this.predicate = predicate; + } + + public Statement build() { + return new StatementImpl(subject, predicate, object); + } + + public StatementBuilder withSubject(Resource subject) { + this.subject = subject; + return this; + } + + public StatementBuilder withObject(RDFNode object) { + this.object = object; + return this; + } + + public static StatementBuilder withPredicate(Property predicate) { + return new StatementBuilder(predicate); + } + +} + diff --git a/ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SkolemizedModel.java b/ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SkolemizedModel.java new file mode 100644 index 000000000..8bb332f51 --- /dev/null +++ b/ldi-core/skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SkolemizedModel.java @@ -0,0 +1,41 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.services.StatementBuilder; +import org.apache.jena.rdf.model.*; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class SkolemizedModel { + private final String skolemUriTemplate; + private final Model model; + private final Map bnodes = new HashMap<>(); + + public SkolemizedModel(String skolemUriTemplate, Model model) { + this.skolemUriTemplate = skolemUriTemplate; + this.model = model; + } + + public Model getModel() { + if(!hasBNodes()) { + return model; + } + return ModelFactory.createDefaultModel().add(model.listStatements().mapWith(this::getSkolemizedStatement).toList()); + } + + private boolean hasBNodes() { + return model.listStatements().filterKeep(statement -> statement.getSubject().isAnon() || statement.getObject().isAnon()).hasNext(); + } + + private Statement getSkolemizedStatement(Statement statement) { + return StatementBuilder.withPredicate(statement.getPredicate()) + .withSubject(statement.getSubject().isAnon() ? getSkolemizedNode(statement.getSubject()) : statement.getSubject()) + .withObject(statement.getObject().isAnon() ? getSkolemizedNode(statement.getObject()) : statement.getObject()) + .build(); + } + + private Resource getSkolemizedNode(RDFNode rdfNode) { + return bnodes.computeIfAbsent(rdfNode, bnode -> ResourceFactory.createProperty(skolemUriTemplate.formatted(UUID.randomUUID()))); + } +} diff --git a/ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SkolemisationTransformerTest.java b/ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SkolemisationTransformerTest.java new file mode 100644 index 000000000..19bc49027 --- /dev/null +++ b/ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/SkolemisationTransformerTest.java @@ -0,0 +1,31 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.RDFParser; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.util.SkolemizationConditions.*; +import static org.assertj.core.api.Assertions.assertThat; + +class SkolemisationTransformerTest { + private static final String SKOLEMIZATION_DOMAIN = "http://example.com"; + private SkolemisationTransformer skolemisationTransformer; + + @BeforeEach + void setUp() { + skolemisationTransformer = new SkolemisationTransformer(SKOLEMIZATION_DOMAIN); + } + + @Test + void given_ModelWithBNodes_when_Transform_then_ReturnModelWithoutBNodes() { + final Model input = RDFParser.source("mob-hind-model.ttl").toModel(); + + final Model result = skolemisationTransformer.transform(input); + + assertThat(result) + .has(noBlankNodes()) + .has(skolemizedSubjectsWithPrefix(2, SKOLEMIZATION_DOMAIN + SkolemisationTransformer.SKOLEM_URI)) + .has(skolemizedObjectsWithPrefix(2, SKOLEMIZATION_DOMAIN + SkolemisationTransformer.SKOLEM_URI)); + } +} \ No newline at end of file diff --git a/ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/util/SkolemizationConditions.java b/ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/util/SkolemizationConditions.java new file mode 100644 index 000000000..5f3bbad33 --- /dev/null +++ b/ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/util/SkolemizationConditions.java @@ -0,0 +1,25 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.util; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.RDFNode; +import org.assertj.core.api.Condition; + +public class SkolemizationConditions { + public static Condition noBlankNodes() { + return new Condition<>(model -> model.listObjects().toList().stream().noneMatch(RDFNode::isAnon), "Model cannot have blank nodes"); + } + + public static Condition skolemizedSubjectsWithPrefix(int count, String prefix) { + return new Condition<>(actual -> actual.listSubjects() + .filterKeep(object -> object.toString().contains(prefix)) + .toList() + .size() == count,"Expected to find %d subjects with skolemized id", count); + } + + public static Condition skolemizedObjectsWithPrefix(int count, String prefix) { + return new Condition<>(actual -> actual.listObjects() + .filterKeep(object -> object.toString().contains(prefix)) + .toList() + .size() == count,"Expected to find %d objects with skolemized id", count); + } +} diff --git a/ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SkolemizedModelTest.java b/ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SkolemizedModelTest.java new file mode 100644 index 000000000..8f632b690 --- /dev/null +++ b/ldi-core/skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/valueobjects/SkolemizedModelTest.java @@ -0,0 +1,34 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.SkolemisationTransformer; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.RDFParser; +import org.junit.jupiter.api.Test; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.util.SkolemizationConditions.*; +import static org.assertj.core.api.Assertions.assertThat; + +class SkolemizedModelTest { + private static final String SKOLEM_URI_TEMPLATE = "http://example.org" + SkolemisationTransformer.SKOLEM_URI + "%s"; + + @Test + void given_ModelWithBNodes_when_getModel_then_NoBNodesArePresent() { + final Model modelToSkolemize = RDFParser.source("activity.nq").toModel(); + + final Model skolemizedModel = new SkolemizedModel(SKOLEM_URI_TEMPLATE, modelToSkolemize).getModel(); + + assertThat(skolemizedModel) + .has(noBlankNodes()) + .has(skolemizedObjectsWithPrefix(2, SKOLEM_URI_TEMPLATE.replace("%s", ""))) + .has(skolemizedSubjectsWithPrefix(3, SKOLEM_URI_TEMPLATE.replace("%s", ""))); + } + + @Test + void given_ModelWithoutBNodes_when_getModel_then_ThereAreStillNoBNodesPresent() { + final Model modelToSkolemize = RDFParser.source("skolemized-product.nq").toModel(); + + final Model skolemizedModel = new SkolemizedModel(SKOLEM_URI_TEMPLATE, modelToSkolemize).getModel(); + + assertThat(skolemizedModel).has(noBlankNodes()); + } +} \ No newline at end of file diff --git a/ldi-core/skolemisation-transformer/src/test/resources/activity.nq b/ldi-core/skolemisation-transformer/src/test/resources/activity.nq new file mode 100644 index 000000000..6b650681d --- /dev/null +++ b/ldi-core/skolemisation-transformer/src/test/resources/activity.nq @@ -0,0 +1,11 @@ + . + "Sally" . + _:b2 . +_:b0 . +_:b0 . +_:b0 _:b1 . +_:b0 "2015-01-25T12:34:56Z"^^ . +_:b1 . +_:b1 "This is a simple note" . +_:b2 . +_:b2 "Timo" . \ No newline at end of file diff --git a/ldi-core/skolemisation-transformer/src/test/resources/mob-hind-model.ttl b/ldi-core/skolemisation-transformer/src/test/resources/mob-hind-model.ttl new file mode 100644 index 000000000..7285b6789 --- /dev/null +++ b/ldi-core/skolemisation-transformer/src/test/resources/mob-hind-model.ttl @@ -0,0 +1,24 @@ +@prefix dc: . +@prefix geosparql: . +@prefix xsd: . +@prefix adms: . +@prefix skos: . +@prefix gipod: . +@prefix ns0: . +@prefix m8g: . + + + dc:isVersionOf ; + a ; + geosparql:asWKT "POLYGON ((3.7337472847142124 51.04745170597559, 4.359276660355135 50.851907920816956, 4.711285586572245 50.84364854093491, 4.4020885567877315 51.214619167436666, 3.7337472847142124 51.04745170597559))"^^geosparql:wktLiteral ; + dc:created "2023-11-30T21:45:15+01:00"^^xsd:dateTime ; + adms:identifier [ + a adms:Identifier ; + skos:notation "10810464"^^gipod:gipodId ; + adms:schemaAgency "https://gipod.vlaanderen.be"@nl-be + ] ; + ns0:periode [ + m8g:endTime "2022-05-27T17:00:00Z"^^xsd:dateTime ; + m8g:startTime "2022-05-27T07:00:00Z"^^xsd:dateTime ; + a m8g:PeriodOfTime + ] . \ No newline at end of file diff --git a/ldi-core/skolemisation-transformer/src/test/resources/skolemized-product.nq b/ldi-core/skolemisation-transformer/src/test/resources/skolemized-product.nq new file mode 100644 index 000000000..b2a1728bb --- /dev/null +++ b/ldi-core/skolemisation-transformer/src/test/resources/skolemized-product.nq @@ -0,0 +1,13 @@ + . + "Need to sell fast and furiously" . + . + . + . + "Used Tesla Roadster" . + . + "USD" . + "85000"^^ . + "Tesla Roadster" . + . + . + . diff --git a/ldi-nifi/ldi-nifi-common/pom.xml b/ldi-nifi/ldi-nifi-common/pom.xml index 6ffe4990d..6a0e04118 100644 --- a/ldi-nifi/ldi-nifi-common/pom.xml +++ b/ldi-nifi/ldi-nifi-common/pom.xml @@ -27,6 +27,11 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-infra-sql + + be.vlaanderen.informatievlaanderen.ldes.ldi + request-executor + ${project.version} + \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/RequestExecutorProperties.java b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/RequestExecutorProperties.java new file mode 100644 index 000000000..dd6c3bcac --- /dev/null +++ b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/RequestExecutorProperties.java @@ -0,0 +1,155 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; + +public class RequestExecutorProperties { + private RequestExecutorProperties() { + } + + public static final PropertyDescriptor API_KEY_HEADER_PROPERTY = new PropertyDescriptor.Builder() + .name("API_KEY_HEADER_PROPERTY") + .displayName("API-KEY header property") + .description("API header that should be used for the API key") + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .defaultValue("X-API-KEY") + .build(); + + public static final PropertyDescriptor API_KEY_PROPERTY = new PropertyDescriptor.Builder() + .name("API_KEY_PROPERTY") + .displayName("API-KEY property") + .description("API key that should be used to access the API.") + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static final PropertyDescriptor OAUTH_CLIENT_ID = new PropertyDescriptor.Builder() + .name("OAUTH_CLIENT_ID") + .displayName("OAUTH client ID") + .description("Client id used for Oauth2 client credentials flow") + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static final PropertyDescriptor OAUTH_CLIENT_SECRET = new PropertyDescriptor.Builder() + .name("OAUTH_CLIENT_SECRET") + .displayName("OAUTH client secret") + .description("Client secret used for Oauth2 client credentials flow") + .sensitive(true) + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static final PropertyDescriptor OAUTH_TOKEN_ENDPOINT = new PropertyDescriptor.Builder() + .name("OAUTH_TOKEN_ENDPOINT") + .displayName("OAUTH token endpoint") + .description("Token endpoint used for Oauth2 client credentials flow.") + .required(false) + .addValidator(StandardValidators.URL_VALIDATOR) + .build(); + + public static final PropertyDescriptor OAUTH_SCOPE = new PropertyDescriptor.Builder() + .name("OAUTH_SCOPE") + .displayName("OAUTH scope") + .description("Scope used for Oauth2 client credentials flow.") + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static final PropertyDescriptor AUTHORIZATION_STRATEGY = new PropertyDescriptor.Builder() + .name("AUTHORIZATION_STRATEGY") + .displayName("Authorization strategy") + .description("Authorization strategy for the internal http client.") + .required(true) + .defaultValue(AuthStrategy.NO_AUTH.name()) + .allowableValues(Arrays.stream(AuthStrategy.values()).map(Enum::name).collect(Collectors.toSet())) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static final PropertyDescriptor RETRIES_ENABLED = new PropertyDescriptor.Builder() + .name("RETRIES_ENABLED") + .displayName("Retries enabled") + .description("Indicates of retries are enabled when the http request fails.") + .required(false) + .defaultValue(TRUE.toString()) + .allowableValues(FALSE.toString(), TRUE.toString()) + .addValidator(StandardValidators.BOOLEAN_VALIDATOR) + .build(); + + public static final PropertyDescriptor MAX_RETRIES = new PropertyDescriptor.Builder() + .name("MAX_RETRIES") + .displayName("Max retries") + .description("Indicates max number of retries when retries are enabled.") + .required(false) + .defaultValue(String.valueOf(5)) + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .build(); + + public static final PropertyDescriptor STATUSES_TO_RETRY = new PropertyDescriptor.Builder() + .name("STATUSES_TO_RETRY") + .displayName("Statuses to retry") + .description("Custom comma seperated list of http status codes that can trigger a retry in the http client.") + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + + public static String getApiKeyHeader(final ProcessContext context) { + return context.getProperty(API_KEY_HEADER_PROPERTY).getValue(); + } + + public static String getApiKey(final ProcessContext context) { + return context.getProperty(API_KEY_PROPERTY).getValue(); + } + + public static String getOauthClientId(final ProcessContext context) { + return context.getProperty(OAUTH_CLIENT_ID).getValue(); + } + + public static String getOauthClientSecret(final ProcessContext context) { + return context.getProperty(OAUTH_CLIENT_SECRET).getValue(); + } + + public static String getOauthTokenEndpoint(final ProcessContext context) { + return context.getProperty(OAUTH_TOKEN_ENDPOINT).getValue(); + } + + public static String getOauthScope(final ProcessContext context) { + return context.getProperty(OAUTH_SCOPE).getValue(); + } + + public static AuthStrategy getAuthorizationStrategy(final ProcessContext context) { + final String authValue = context.getProperty(AUTHORIZATION_STRATEGY).getValue(); + return AuthStrategy + .from(authValue) + .orElseThrow(() -> new IllegalArgumentException("Unsupported authorization strategy: " + authValue)); + } + + public static boolean retriesEnabled(final ProcessContext context) { + return !FALSE.equals(context.getProperty(RETRIES_ENABLED).asBoolean()); + } + + public static int getMaxRetries(final ProcessContext context) { + return context.getProperty(MAX_RETRIES).asInteger(); + } + + public static List getStatusesToRetry(final ProcessContext context) { + String commaSeperatedValues = context.getProperty(STATUSES_TO_RETRY).getValue(); + if (commaSeperatedValues != null) { + return Stream.of(commaSeperatedValues.split(",")).map(String::trim).map(Integer::parseInt).toList(); + } else { + return new ArrayList<>(); + } + } +} diff --git a/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/services/RequestExecutorSupplier.java b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/services/RequestExecutorSupplier.java new file mode 100644 index 000000000..74757a237 --- /dev/null +++ b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/services/RequestExecutorSupplier.java @@ -0,0 +1,45 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.retry.RetryConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorDecorator; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import io.github.resilience4j.retry.Retry; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; +import org.apache.nifi.processor.ProcessContext; + +import java.util.List; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.RequestExecutorProperties.*; + +public class RequestExecutorSupplier { + private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); + + public RequestExecutor getRequestExecutor(ProcessContext context) { + return RequestExecutorDecorator.decorate(getBaseRequestExecutor(context)).with(getRetry(context)).get(); + } + + private Retry getRetry(final ProcessContext context) { + if (retriesEnabled(context)) { + return RetryConfig.of(getMaxRetries(context), getStatusesToRetry(context)).getRetry(); + } else { + return null; + } + } + + private RequestExecutor getBaseRequestExecutor(final ProcessContext context) { + return switch (getAuthorizationStrategy(context)) { + case NO_AUTH -> requestExecutorFactory.createNoAuthExecutor(); + case API_KEY -> { + List
headers = List.of( + new BasicHeader(getApiKeyHeader(context), getApiKey(context)) + ); + yield requestExecutorFactory.createNoAuthExecutor(headers); + } + case OAUTH2_CLIENT_CREDENTIALS -> + requestExecutorFactory.createClientCredentialsExecutor(getOauthClientId(context), + getOauthClientSecret(context), getOauthTokenEndpoint(context), getOauthScope(context)); + }; + } +} diff --git a/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/pom.xml new file mode 100644 index 000000000..bc5dc51c7 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/pom.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + ldi-nifi-processors + 2.10.0-SNAPSHOT + + + http-sparql-out-processor + nar + + + + be.vlaanderen.informatievlaanderen.ldes.ldi + http-sparql-out + ${project.version} + + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-nifi-common + ${project.version} + + + + com.github.tomakehurst + wiremock-jre8 + + + + \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HttpSparqlOutProcessor.java b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HttpSparqlOutProcessor.java new file mode 100644 index 000000000..8eb78c509 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HttpSparqlOutProcessor.java @@ -0,0 +1,116 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.HttpSparqlOut; +import be.vlaanderen.informatievlaanderen.ldes.ldi.SkolemisationTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldi.factory.DeleteFunctionBuilder; +import be.vlaanderen.informatievlaanderen.ldes.ldi.factory.InsertFunctionBuilder; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.RequestExecutorSupplier; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation.EmptySkolemizer; +import be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation.Skolemizer; +import be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation.SkolemizerImpl; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.DeleteFunction; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.InsertFunction; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.SparqlQuery; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFLanguages; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; + +import java.util.List; +import java.util.Set; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.HttpSparqlOutProcessorProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.RequestExecutorProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.*; + +@SuppressWarnings("java:S2160") // nifi handles equals/hashcode of processors +@Tags({"http", "sparql", "vsds", "ldes"}) +@CapabilityDescription("Transforms blank nodes from LDES members to a skolemized URI") +public class HttpSparqlOutProcessor extends AbstractProcessor { + private HttpSparqlOut httpSparqlOut; + + @Override + protected List getSupportedPropertyDescriptors() { + return List.of( + ENDPOINT, + GRAPH, + REPLACEMENT_ENABLED, + REPLACEMENT_DEPTH, + REPLACEMENT_DELETE_FUNCTION, + SKOLEMISATION_DOMAIN, + API_KEY_HEADER_PROPERTY, + API_KEY_PROPERTY, + OAUTH_CLIENT_ID, + OAUTH_CLIENT_SECRET, + OAUTH_TOKEN_ENDPOINT, + OAUTH_SCOPE, + AUTHORIZATION_STRATEGY, + RETRIES_ENABLED, + MAX_RETRIES, + STATUSES_TO_RETRY + ); + } + + @Override + public Set getRelationships() { + return Set.of(SUCCESS, FAILURE); + } + + @OnScheduled + public void onScheduled(final ProcessContext context) { + final DeleteFunction deleteFunction = createDeleteFunction(context); + final InsertFunction insertFunction = getGraph(context) + .map(InsertFunctionBuilder::withGraph) + .orElseGet(InsertFunctionBuilder::create) + .build(); + final Skolemizer skolemizer = getSkolemisationDomain(context) + .map(SkolemisationTransformer::new) + .map(skolemisationTransformer -> (Skolemizer) new SkolemizerImpl(skolemisationTransformer)) + .orElseGet(EmptySkolemizer::new); + final RequestExecutor requestExecutor = new RequestExecutorSupplier().getRequestExecutor(context); + + final SparqlQuery sparqlQuery = new SparqlQuery(insertFunction, deleteFunction); + httpSparqlOut = new HttpSparqlOut(getEndpoint(context), sparqlQuery, skolemizer, requestExecutor); + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + final FlowFile flowFile = session.get(); + if(flowFile != null) { + try { + final Lang mimeType = RDFLanguages.contentTypeToLang(flowFile.getAttribute("mime.type")); + final Model inputModel = receiveDataAsModel(session, flowFile, mimeType); + + httpSparqlOut.write(inputModel); + + session.transfer(flowFile, SUCCESS); + } catch (Exception e) { + getLogger().error("Error executing http sparql out: {}", e.getMessage()); + session.transfer(flowFile, FAILURE); + } + } + } + + private static DeleteFunction createDeleteFunction(ProcessContext context) { + if(!isReplacementEnabled(context)) { + return DeleteFunctionBuilder.disabled(); + } + return getReplacementDeleteFunction(context) + .map(DeleteFunction::ofQuery) + .orElseGet(() -> getGraph(context) + .map(DeleteFunctionBuilder::withGraph) + .orElseGet(DeleteFunctionBuilder::create) + .withDepth(getReplacementDepth(context)) + ); + } +} diff --git a/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/HttpSparqlOutProcessorProperties.java b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/HttpSparqlOutProcessorProperties.java new file mode 100644 index 000000000..94fdfbf59 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/HttpSparqlOutProcessorProperties.java @@ -0,0 +1,100 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.Optional; + +public class HttpSparqlOutProcessorProperties { + private HttpSparqlOutProcessorProperties() { + } + + public static final PropertyDescriptor ENDPOINT = new PropertyDescriptor.Builder() + .name("ENDPOINT") + .displayName("Endpoint") + .description("Endpoint that must be used to write triples to") + .required(true) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .addValidator(StandardValidators.URL_VALIDATOR) + .build(); + + public static final PropertyDescriptor GRAPH = new PropertyDescriptor.Builder() + .name("GRAPH") + .displayName("Graph") + .description("Graph that must be used to write triples to") + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .addValidator(StandardValidators.URI_VALIDATOR) + .build(); + + public static final PropertyDescriptor SKOLEMISATION_DOMAIN = new PropertyDescriptor.Builder() + .name("SKOLEMISATION_DOMAIN") + .displayName("Skolemisation domain") + .description("Domain to use for skolemisation, if ommitted, then skolemisation is disabled") + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .addValidator(StandardValidators.URI_VALIDATOR) + .build(); + + public static final PropertyDescriptor REPLACEMENT_ENABLED = new PropertyDescriptor.Builder() + .name("REPLACEMENT_ENABLED") + .displayName("Replacement enabled") + .description("Whether replacment should be enabled") + .required(true) + .defaultValue(Boolean.TRUE.toString()) + .allowableValues(Boolean.TRUE.toString(), Boolean.FALSE.toString()) + .build(); + + public static final PropertyDescriptor REPLACEMENT_DEPTH = new PropertyDescriptor.Builder() + .name("REPLACEMENT_DEPTH") + .displayName("Replacement depth") + .description("Number of levels of nested nodes that must be deleted during the replacement process") + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .addValidator(StandardValidators.INTEGER_VALIDATOR) + .defaultValue("10") + .dependsOn(REPLACEMENT_ENABLED, Boolean.TRUE.toString()) + .build(); + + public static final PropertyDescriptor REPLACEMENT_DELETE_FUNCTION = new PropertyDescriptor.Builder() + .name("REPLACEMENT_DELETE_FUNCTION") + .displayName("Replacement delete function") + .description("Custom delete function that must be used during the replacement process instead of the function created by the replacement depth") + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .dependsOn(REPLACEMENT_ENABLED, Boolean.TRUE.toString()) + .build(); + + public static String getEndpoint(ProcessContext processContext) { + return processContext.getProperty(ENDPOINT).getValue(); + } + + public static Optional getGraph(ProcessContext processContext) { + if(processContext.getProperty(GRAPH).isSet()) { + return Optional.of(processContext.getProperty(GRAPH).getValue()); + } + return Optional.empty(); + } + + public static Optional getSkolemisationDomain(ProcessContext processContext) { + if(processContext.getProperty(SKOLEMISATION_DOMAIN).isSet()) { + return Optional.of(processContext.getProperty(SKOLEMISATION_DOMAIN).getValue()); + } + return Optional.empty(); + } + + public static boolean isReplacementEnabled(ProcessContext processContext) { + return Boolean.TRUE.equals(processContext.getProperty(REPLACEMENT_ENABLED).asBoolean()); + } + + public static int getReplacementDepth(ProcessContext processContext) { + return processContext.getProperty(REPLACEMENT_DEPTH).asInteger(); + } + + public static Optional getReplacementDeleteFunction(ProcessContext processContext) { + if(processContext.getProperty(REPLACEMENT_DELETE_FUNCTION).isSet()) { + return Optional.of(processContext.getProperty(REPLACEMENT_DELETE_FUNCTION).getValue()); + } + return Optional.empty(); + } +} diff --git a/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 000000000..02b6f3d2b --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1 @@ +be.vlaanderen.informatievlaanderen.ldes.ldi.processors.HttpSparqlOutProcessor \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HttpSparqlOutProcessorTest.java b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HttpSparqlOutProcessorTest.java new file mode 100644 index 000000000..74a8f03d0 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HttpSparqlOutProcessorTest.java @@ -0,0 +1,207 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; + +import com.github.tomakehurst.wiremock.client.BasicCredentials; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParser; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.AssertionFailedError; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Path; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.HttpSparqlOutProcessorProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.RequestExecutorProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.FAILURE; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.SUCCESS; +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class HttpSparqlOutProcessorTest { + private static final int PORT = 12321; + private static final Map minimalProperties = Map.of( + ENDPOINT, "http://localhost:%s/sparql".formatted(PORT), + GRAPH, "http://example.graph.com" + ); + private static final String MIME_TYPE = "mime.type"; + private TestRunner testRunner; + + @BeforeEach + void setUp() { + testRunner = TestRunners.newTestRunner(HttpSparqlOutProcessor.class); + } + + @Test + void given_InvalidModel_when_SparqlOut_then_AddModelToFailureRelation() { + minimalProperties.forEach(testRunner::setProperty); + + testRunner.enqueue("invalid model", Map.of(MIME_TYPE, Lang.NT.getHeaderString())); + + testRunner.run(); + + assertThat(testRunner.getFlowFilesForRelationship(FAILURE)).hasSize(1); + } + + @Test + void test_EmptyFlowFile() { + minimalProperties.forEach(testRunner::setProperty); + + testRunner.run(); + + assertThat(testRunner.getFlowFilesForRelationship(SUCCESS)).isEmpty(); + assertThat(testRunner.getFlowFilesForRelationship(FAILURE)).isEmpty(); + } + + @Nested + @WireMockTest(httpPort = HttpSparqlOutProcessorTest.PORT) + class HttpInteractions { + private static Path path; + private Model expectedModel; + private static final Lang mimetype = Lang.TURTLE; + + @BeforeAll + static void beforeAll() throws URISyntaxException { + URL resource = HttpSparqlOutProcessorTest.class.getClassLoader().getResource("mob-hind-model.ttl"); + path = Path.of(Objects.requireNonNull(resource).toURI()); + } + + @BeforeEach + void setUp() { + expectedModel = RDFParser.source(path).lang(Lang.TURTLE).toModel(); + stubFor(post(urlEqualTo("/sparql")).willReturn(ok())); + } + + @Test + void given_ValidModel_when_SparqlOut_then_AddModelToSuccessRelation() { + testRunner.setProperty(ENDPOINT, "http://localhost:%s/sparql".formatted(PORT)); + testRunner.setProperty(REPLACEMENT_ENABLED, Boolean.FALSE.toString()); + + final String content = " \"Jane Doe\" .\n"; + Model expected = RDFParser.create().fromString(content).lang(Lang.NT).toModel(); + final String expectedRequestBody = "INSERT DATA { %s }".formatted(content); + + testRunner.enqueue(content, Map.of(MIME_TYPE, Lang.NT.getHeaderString())); + testRunner.run(); + + MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(SUCCESS).getFirst(); + Model result = RDFParser.create().source(flowFile.getContentStream()).lang(Lang.NT).toModel(); + + verify(postRequestedFor(urlEqualTo("/sparql")).withRequestBody(containing(expectedRequestBody))); + assertThat(result).matches(expected::isIsomorphicWith); + } + + @Test + void given_SkolemisationEnabled_when_SparqlOut_then_AddModelToSuccessRelation() throws IOException { + final String skolemDomain = "http://example.com"; + minimalProperties.forEach(testRunner::setProperty); + testRunner.setProperty(SKOLEMISATION_DOMAIN, skolemDomain); + + testRunner.enqueue(path, Map.of(MIME_TYPE, mimetype.getHeaderString())); + testRunner.run(); + + MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(SUCCESS).getFirst(); + Model result = RDFParser.create().source(flowFile.getContentStream()).lang(mimetype).toModel(); + + verify(postRequestedFor(urlEqualTo("/sparql")).withRequestBody(containing("%s/.well-known/genid/".formatted(skolemDomain)))); + assertThat(result).matches(expectedModel::isIsomorphicWith); + } + + @Test + void given_GraphOmitted_when_SparqlOut_then_AddModelToSuccessRelation() throws IOException { + testRunner.setProperty(ENDPOINT, "http://localhost:%s/sparql".formatted(PORT)); + + testRunner.enqueue(path, Map.of(MIME_TYPE, mimetype.getHeaderString())); + testRunner.run(); + + MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(SUCCESS).getFirst(); + Model result = RDFParser.create().source(flowFile.getContentStream()).lang(mimetype).toModel(); + + verify(postRequestedFor(urlEqualTo("/sparql")).withRequestBody(notContaining("FROM"))); + assertThat(result).matches(expectedModel::isIsomorphicWith); + } + + @Test + void given_CustomDeteFunction_when_SparqlOut_then_AddModelToSuccessRelation() throws IOException { + final String customDeleteFunction = "custom function"; + testRunner.setProperty(ENDPOINT, "http://localhost:%s/sparql".formatted(PORT)); + testRunner.setProperty(REPLACEMENT_DELETE_FUNCTION, customDeleteFunction); + + testRunner.enqueue(path, Map.of(MIME_TYPE, mimetype.getHeaderString())); + testRunner.run(); + + MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(SUCCESS).getFirst(); + Model result = RDFParser.create().source(flowFile.getContentStream()).lang(mimetype).toModel(); + + verify(postRequestedFor(urlEqualTo("/sparql")).withRequestBody(containing(customDeleteFunction).and(notContaining("FROM")))); + assertThat(result).matches(expectedModel::isIsomorphicWith); + } + + @Test + void given_EndpointReturns400_when_SparqlOut_then_AddModelToSuccessRelation() throws IOException { + stubFor(post(urlEqualTo("/sparql-fail")).willReturn(aResponse().withStatus(400))); + testRunner.setProperty(ENDPOINT, "http://localhost:%s/sparql-fail".formatted(PORT)); + + testRunner.enqueue(path, Map.of(MIME_TYPE, mimetype.getHeaderString())); + testRunner.run(); + + MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(FAILURE).getFirst(); + Model result = RDFParser.create().source(flowFile.getContentStream()).lang(mimetype).toModel(); + + verify(postRequestedFor(urlEqualTo("/sparql-fail"))); + assertThat(result).matches(expectedModel::isIsomorphicWith); + assertThat(testRunner.getLogger().getErrorMessages()).hasSize(1); + } + + @Test + void given_AuthConfig_when_SparqlOut_then_PostQuerywithHeader() throws IOException { + final BasicCredentials basicCredentials = new BasicCredentials("sparql", "changeme"); + + minimalProperties.forEach(testRunner::setProperty); + testRunner.setProperty(AUTHORIZATION_STRATEGY, "API_KEY"); + testRunner.setProperty(API_KEY_HEADER_PROPERTY, "Authorization"); + testRunner.setProperty(API_KEY_PROPERTY, basicCredentials.asAuthorizationHeaderValue()); + + testRunner.enqueue(path, Map.of(MIME_TYPE, mimetype.getHeaderString())); + testRunner.run(); + + verify(postRequestedFor(urlEqualTo("/sparql")).withBasicAuth(basicCredentials)); + } + } + + @ParameterizedTest + @MethodSource("invalidProperties") + void given_InvalidProperties_when_TryToTestRun_then_ThrowAssertionFailedError(Map properties, int expectedFailures) { + properties.forEach(testRunner::setProperty); + + assertThatThrownBy(testRunner::run) + .isInstanceOf(AssertionFailedError.class) + .hasMessageContaining("%d validation failures", expectedFailures); + } + + static Stream invalidProperties() { + return Stream.of( + Arguments.of(Map.of(), 1), + Arguments.of(Map.of(GRAPH, "http://example.graph.com"), 1), + Arguments.of(Map.of(ENDPOINT, "", GRAPH, "http://"), 2), + Arguments.of(Map.of(ENDPOINT, "http://localhost:8890/sparql", GRAPH, "http://example.graph.com", REPLACEMENT_DEPTH, "ten"), 1), + Arguments.of(Map.of(ENDPOINT, "http://localhost:8890/sparql", GRAPH, "http://example.graph.com", REPLACEMENT_ENABLED, "off"), 1) + ); + } +} \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/test/resources/mob-hind-model.ttl b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/test/resources/mob-hind-model.ttl new file mode 100644 index 000000000..7285b6789 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/src/test/resources/mob-hind-model.ttl @@ -0,0 +1,24 @@ +@prefix dc: . +@prefix geosparql: . +@prefix xsd: . +@prefix adms: . +@prefix skos: . +@prefix gipod: . +@prefix ns0: . +@prefix m8g: . + + + dc:isVersionOf ; + a ; + geosparql:asWKT "POLYGON ((3.7337472847142124 51.04745170597559, 4.359276660355135 50.851907920816956, 4.711285586572245 50.84364854093491, 4.4020885567877315 51.214619167436666, 3.7337472847142124 51.04745170597559))"^^geosparql:wktLiteral ; + dc:created "2023-11-30T21:45:15+01:00"^^xsd:dateTime ; + adms:identifier [ + a adms:Identifier ; + skos:notation "10810464"^^gipod:gipodId ; + adms:schemaAgency "https://gipod.vlaanderen.be"@nl-be + ] ; + ns0:periode [ + m8g:endTime "2022-05-27T17:00:00Z"^^xsd:dateTime ; + m8g:startTime "2022-05-27T07:00:00Z"^^xsd:dateTime ; + a m8g:PeriodOfTime + ] . \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java index d2ab73f7f..8986af2cc 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java @@ -2,14 +2,11 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.RequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers.MemberSupplierWrappersBuilder; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.retry.RetryConfig; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorDecorator; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; -import io.github.resilience4j.retry.Retry; import ldes.client.eventstreamproperties.EventStreamPropertiesFetcher; import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; import ldes.client.eventstreamproperties.valueobjects.PropertiesRequest; @@ -17,8 +14,6 @@ import ldes.client.treenodesupplier.domain.valueobject.*; import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; import ldes.client.treenodesupplier.membersuppliers.MemberSupplierImpl; -import org.apache.http.Header; -import org.apache.http.message.BasicHeader; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFWriter; @@ -46,6 +41,7 @@ import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties.*; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorRelationships.DATA_RELATIONSHIP; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.RequestExecutorProperties.*; import static org.apache.jena.rdf.model.ResourceFactory.createProperty; @SuppressWarnings("java:S2160") // nifi handles equals/hashcode of processors @@ -57,7 +53,6 @@ public class LdesClientProcessor extends AbstractProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(LdesClientProcessor.class); private MemberSupplier memberSupplier; private EventStreamProperties eventStreamProperties; - private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private boolean hasLdesEnded; @Override @@ -99,7 +94,8 @@ public final List getSupportedPropertyDescriptors() { public void onScheduled(final ProcessContext context) { List dataSourceUrls = LdesProcessorProperties.getDataSourceUrl(context); Lang dataSourceFormat = LdesProcessorProperties.getDataSourceFormat(context); - final RequestExecutor requestExecutor = getRequestExecutorWithPossibleRetry(context); + final RequestExecutorSupplier requestExecutorSupplier = new RequestExecutorSupplier(); + final RequestExecutor requestExecutor = requestExecutorSupplier.getRequestExecutor(context); LdesMetaData ldesMetaData = new LdesMetaData(dataSourceUrls, dataSourceFormat); StatePersistence statePersistence = new StatePersistenceFactory().getStatePersistence(context); eventStreamProperties = fetchEventStreamProperties(ldesMetaData, requestExecutor); @@ -117,33 +113,6 @@ public void onScheduled(final ProcessContext context) { context.getName(), dataSourceUrls, dataSourceFormat); } - private RequestExecutor getRequestExecutorWithPossibleRetry(final ProcessContext context) { - return RequestExecutorDecorator.decorate(getRequestExecutor(context)).with(getRetry(context)).get(); - } - - private Retry getRetry(final ProcessContext context) { - if (retriesEnabled(context)) { - return RetryConfig.of(getMaxRetries(context), getStatusesToRetry(context)).getRetry(); - } else { - return null; - } - } - - private RequestExecutor getRequestExecutor(final ProcessContext context) { - return switch (getAuthorizationStrategy(context)) { - case NO_AUTH -> requestExecutorFactory.createNoAuthExecutor(); - case API_KEY -> { - List
headers = List.of( - new BasicHeader(getApiKeyHeader(context), getApiKey(context)) - ); - yield requestExecutorFactory.createNoAuthExecutor(headers); - } - case OAUTH2_CLIENT_CREDENTIALS -> - requestExecutorFactory.createClientCredentialsExecutor(getOauthClientId(context), - getOauthClientSecret(context), getOauthTokenEndpoint(context), getOauthScope(context)); - }; - } - private EventStreamProperties fetchEventStreamProperties(LdesMetaData ldesMetaData, RequestExecutor requestExecutor) { PropertiesRequest request = new PropertiesRequest(ldesMetaData.getStartingNodeUrl(), ldesMetaData.getLang()); return new EventStreamPropertiesFetcher(requestExecutor).fetchEventStreamProperties(request); @@ -161,7 +130,6 @@ public void onTrigger(ProcessContext context, ProcessSession session) throws Pro LOGGER.warn(exception.getMessage()); hasLdesEnded = true; } - } private void processNextMember(ProcessContext context, ProcessSession session) { diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java index 32a86e299..e92de8010 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorProperties.java @@ -1,7 +1,6 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.validators.RDFLanguageValidator; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFLanguages; import org.apache.nifi.components.PropertyDescriptor; @@ -10,11 +9,8 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.CommonProperties.DATA_DESTINATION_FORMAT; import static java.lang.Boolean.FALSE; @@ -73,93 +69,6 @@ private LdesProcessorProperties() { .defaultValue(FALSE.toString()) .build(); - public static final PropertyDescriptor API_KEY_HEADER_PROPERTY = new PropertyDescriptor.Builder() - .name("API_KEY_HEADER_PROPERTY") - .displayName("API-KEY header property") - .description("API header that should be used for the API key") - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .defaultValue("X-API-KEY") - .build(); - - public static final PropertyDescriptor API_KEY_PROPERTY = new PropertyDescriptor.Builder() - .name("API_KEY_PROPERTY") - .displayName("API-KEY property") - .description("API key that should be used to access the API.") - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .build(); - - public static final PropertyDescriptor OAUTH_CLIENT_ID = new PropertyDescriptor.Builder() - .name("OAUTH_CLIENT_ID") - .displayName("OAUTH client ID") - .description("Client id used for Oauth2 client credentials flow") - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .build(); - - public static final PropertyDescriptor OAUTH_CLIENT_SECRET = new PropertyDescriptor.Builder() - .name("OAUTH_CLIENT_SECRET") - .displayName("OAUTH client secret") - .description("Client secret used for Oauth2 client credentials flow") - .sensitive(true) - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .build(); - - public static final PropertyDescriptor OAUTH_TOKEN_ENDPOINT = new PropertyDescriptor.Builder() - .name("OAUTH_TOKEN_ENDPOINT") - .displayName("OAUTH token endpoint") - .description("Token endpoint used for Oauth2 client credentials flow.") - .required(false) - .addValidator(StandardValidators.URL_VALIDATOR) - .build(); - - public static final PropertyDescriptor OAUTH_SCOPE = new PropertyDescriptor.Builder() - .name("OAUTH_SCOPE") - .displayName("OAUTH scope") - .description("Scope used for Oauth2 client credentials flow.") - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .build(); - - public static final PropertyDescriptor AUTHORIZATION_STRATEGY = new PropertyDescriptor.Builder() - .name("AUTHORIZATION_STRATEGY") - .displayName("Authorization strategy") - .description("Authorization strategy for the internal http client.") - .required(true) - .defaultValue(AuthStrategy.NO_AUTH.name()) - .allowableValues(Arrays.stream(AuthStrategy.values()).map(Enum::name).collect(Collectors.toSet())) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .build(); - - public static final PropertyDescriptor RETRIES_ENABLED = new PropertyDescriptor.Builder() - .name("RETRIES_ENABLED") - .displayName("Retries enabled") - .description("Indicates of retries are enabled when the http request fails.") - .required(false) - .defaultValue(TRUE.toString()) - .allowableValues(FALSE.toString(), TRUE.toString()) - .addValidator(StandardValidators.BOOLEAN_VALIDATOR) - .build(); - - public static final PropertyDescriptor MAX_RETRIES = new PropertyDescriptor.Builder() - .name("MAX_RETRIES") - .displayName("Max retries") - .description("Indicates max number of retries when retries are enabled.") - .required(false) - .defaultValue(String.valueOf(5)) - .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) - .build(); - - public static final PropertyDescriptor STATUSES_TO_RETRY = new PropertyDescriptor.Builder() - .name("STATUSES_TO_RETRY") - .displayName("Statuses to retry") - .description("Custom comma seperated list of http status codes that can trigger a retry in the http client.") - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .build(); - public static final PropertyDescriptor USE_VERSION_MATERIALISATION = new PropertyDescriptor.Builder() .name("USE_VERSION_MATERIALISATION") .displayName("Use version materialisation") @@ -230,54 +139,6 @@ public static boolean streamShapeProperty(final ProcessContext context) { return TRUE.equals(context.getProperty(STREAM_SHAPE_PROPERTY).asBoolean()); } - public static String getApiKeyHeader(final ProcessContext context) { - return context.getProperty(API_KEY_HEADER_PROPERTY).getValue(); - } - - public static String getApiKey(final ProcessContext context) { - return context.getProperty(API_KEY_PROPERTY).getValue(); - } - - public static String getOauthClientId(final ProcessContext context) { - return context.getProperty(OAUTH_CLIENT_ID).getValue(); - } - - public static String getOauthClientSecret(final ProcessContext context) { - return context.getProperty(OAUTH_CLIENT_SECRET).getValue(); - } - - public static String getOauthTokenEndpoint(final ProcessContext context) { - return context.getProperty(OAUTH_TOKEN_ENDPOINT).getValue(); - } - - public static String getOauthScope(final ProcessContext context) { - return context.getProperty(OAUTH_SCOPE).getValue(); - } - - public static AuthStrategy getAuthorizationStrategy(final ProcessContext context) { - final String authValue = context.getProperty(AUTHORIZATION_STRATEGY).getValue(); - return AuthStrategy - .from(authValue) - .orElseThrow(() -> new IllegalArgumentException("Unsupported authorization strategy: " + authValue)); - } - - public static boolean retriesEnabled(final ProcessContext context) { - return !FALSE.equals(context.getProperty(RETRIES_ENABLED).asBoolean()); - } - - public static int getMaxRetries(final ProcessContext context) { - return context.getProperty(MAX_RETRIES).asInteger(); - } - - public static List getStatusesToRetry(final ProcessContext context) { - String commaSeperatedValues = context.getProperty(STATUSES_TO_RETRY).getValue(); - if (commaSeperatedValues != null) { - return Stream.of(commaSeperatedValues.split(",")).map(String::trim).map(Integer::parseInt).toList(); - } else { - return new ArrayList<>(); - } - } - public static boolean useVersionMaterialisation(final ProcessContext context) { return TRUE.equals(context.getProperty(USE_VERSION_MATERIALISATION).asBoolean()); } diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java index 485c22590..16bc04fcf 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java @@ -18,17 +18,17 @@ class LdesProcessorPropertiesTest { @Test void test_retriesEnabled() { // default is true - assertTrue(LdesProcessorProperties.retriesEnabled(getMockContext(null))); - assertTrue(LdesProcessorProperties.retriesEnabled(getMockContext("true"))); - assertTrue(LdesProcessorProperties.retriesEnabled(getMockContext("trUe"))); - assertFalse(LdesProcessorProperties.retriesEnabled(getMockContext("false"))); - assertFalse(LdesProcessorProperties.retriesEnabled(getMockContext("FALSE"))); + assertTrue(RequestExecutorProperties.retriesEnabled(getMockContext(null))); + assertTrue(RequestExecutorProperties.retriesEnabled(getMockContext("true"))); + assertTrue(RequestExecutorProperties.retriesEnabled(getMockContext("trUe"))); + assertFalse(RequestExecutorProperties.retriesEnabled(getMockContext("false"))); + assertFalse(RequestExecutorProperties.retriesEnabled(getMockContext("FALSE"))); } @Test void test_getStatusesToRetry() { - assertTrue(LdesProcessorProperties.getStatusesToRetry(getMockContext(null)).isEmpty()); - List statusesToRetry = LdesProcessorProperties.getStatusesToRetry(getMockContext("200, 204")); + assertTrue(RequestExecutorProperties.getStatusesToRetry(getMockContext(null)).isEmpty()); + List statusesToRetry = RequestExecutorProperties.getStatusesToRetry(getMockContext("200, 204")); assertTrue(statusesToRetry.contains(200)); assertTrue(statusesToRetry.contains(204)); assertFalse(statusesToRetry.contains(500)); diff --git a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml index b5a98ada9..9a78aee02 100644 --- a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml @@ -73,6 +73,18 @@ ${project.version} nar + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + skolemisation-transformer-processor + ${project.version} + nar + + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + http-sparql-out-processor + ${project.version} + nar + diff --git a/ldi-nifi/ldi-nifi-processors/pom.xml b/ldi-nifi/ldi-nifi-processors/pom.xml index 5f8eeb377..5ebccadf1 100644 --- a/ldi-nifi/ldi-nifi-processors/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/pom.xml @@ -25,6 +25,8 @@ ldi-processors-bundle change-detection-processor rml-adapter-processor + skolemisation-transformer-processor + http-sparql-out-processor diff --git a/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/pom.xml new file mode 100644 index 000000000..b6f085820 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/pom.xml @@ -0,0 +1,28 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + ldi-nifi-processors + 2.10.0-SNAPSHOT + + + skolemisation-transformer-processor + nar + + + + be.vlaanderen.informatievlaanderen.ldes.ldi + skolemisation-transformer + ${project.version} + + + be.vlaanderen.informatievlaanderen.ldes.ldi + ldi-nifi-common + ${project.version} + + + + \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/SkolemisationTransformerProcessor.java b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/SkolemisationTransformerProcessor.java new file mode 100644 index 000000000..c8353cad3 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/SkolemisationTransformerProcessor.java @@ -0,0 +1,62 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.SkolemisationTransformer; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFLanguages; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; + +import java.util.List; +import java.util.Set; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.SkolemisationTransformerProperties.SKOLEM_DOMAIN; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.SkolemisationTransformerProperties.getSkolemDomain; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.*; + +@SuppressWarnings("java:S2160") // nifi handles equals/hashcode of processors +@Tags({"skolemisation", "vsds", "ldes"}) +@CapabilityDescription("Transforms blank nodes from LDES members to a skolemized URI") +public class SkolemisationTransformerProcessor extends AbstractProcessor { + private SkolemisationTransformer skolemisationTransformer; + + @Override + protected List getSupportedPropertyDescriptors() { + return List.of(SKOLEM_DOMAIN); + } + + @Override + public Set getRelationships() { + return Set.of(SUCCESS, FAILURE); + } + + @OnScheduled + public void onScheduled(final ProcessContext context) { + skolemisationTransformer = new SkolemisationTransformer(getSkolemDomain(context)); + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + final FlowFile flowFile = session.get(); + if(flowFile != null) { + try { + final Lang mimeType = RDFLanguages.contentTypeToLang(flowFile.getAttribute("mime.type")); + final Model inputModel = receiveDataAsModel(session, flowFile, mimeType); + + final Model result = skolemisationTransformer.transform(inputModel); + sendRDFToRelation(session, flowFile, result, SUCCESS, mimeType); + } catch (Exception e) { + getLogger().error("Error executing skolemisation transformation: {}", e.getMessage()); + sendRDFToRelation(session, flowFile, FAILURE); + } + } + } +} diff --git a/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/SkolemisationTransformerProperties.java b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/SkolemisationTransformerProperties.java new file mode 100644 index 000000000..b8f997e31 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/SkolemisationTransformerProperties.java @@ -0,0 +1,23 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.util.StandardValidators; + +public class SkolemisationTransformerProperties { + private SkolemisationTransformerProperties() { + } + + public static final PropertyDescriptor SKOLEM_DOMAIN = new PropertyDescriptor.Builder() + .name("SKOLEM_DOMAIN") + .displayName("Skolem domain") + .description("Skolemization domain that will be used for transforming blank nodes") + .required(true) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .addValidator(StandardValidators.URI_VALIDATOR) + .build(); + + public static String getSkolemDomain(ProcessContext context) { + return context.getProperty(SKOLEM_DOMAIN).getValue(); + } +} diff --git a/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 000000000..79f7087fc --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1 @@ +be.vlaanderen.informatievlaanderen.ldes.ldi.processors.SkolemisationTransformerProcessor diff --git a/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/SkolemisationTransformerProcessorTest.java b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/SkolemisationTransformerProcessorTest.java new file mode 100644 index 000000000..4d8a3f81d --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/SkolemisationTransformerProcessorTest.java @@ -0,0 +1,104 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.SkolemisationTransformerProperties; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParser; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.assertj.core.api.Condition; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Map; +import java.util.Objects; +import java.util.function.Predicate; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.FAILURE; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager.SUCCESS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class SkolemisationTransformerProcessorTest { + public static final String MIME_TYPE_KEY = "mime.type"; + private TestRunner testRunner; + + @BeforeEach + void setUp() { + testRunner = TestRunners.newTestRunner(SkolemisationTransformerProcessor.class); + } + + @Test + void test_WrongPropertyName() { + testRunner.setProperty("SKOLEMIZATION_DOMAIN", "http://example.com"); + + assertThatThrownBy(() -> testRunner.run()) + .isInstanceOf(AssertionError.class) + .hasMessageContaining("Processor has 2 validation failures:"); + } + + @ParameterizedTest + @EmptySource + @ValueSource(strings = {" ", "http://"}) + void test_InvalidProperties(String skolemDomain) { + testRunner.setProperty("SKOLEM_DOMAIN", skolemDomain); + + assertThatThrownBy(() -> testRunner.run()) + .isInstanceOf(AssertionError.class) + .hasMessageContaining("Processor has 1 validation failures"); + } + + @Test + void given_ValidModel_when_Skolemise_then_AddSkolemisedModelToSuccessRelation() throws URISyntaxException, IOException { + Lang mimetype = Lang.TURTLE; + testRunner.setProperty(SkolemisationTransformerProperties.SKOLEM_DOMAIN, "http://example.com"); + + URI uri = Objects.requireNonNull(getClass().getClassLoader().getResource("mob-hind-model.ttl")).toURI(); + testRunner.enqueue(Path.of(uri), Map.of(MIME_TYPE_KEY, mimetype.getHeaderString())); + + testRunner.run(); + + MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(SUCCESS).getFirst(); + Model result = RDFParser.create().source(flowFile.getContentStream()).lang(mimetype).toModel(); + + assertThat(result).has(noBlankNodes()); + } + + @Test + void test_EmptyFlowFile() { + testRunner.setProperty(SkolemisationTransformerProperties.SKOLEM_DOMAIN, "http://example.com"); + + testRunner.run(); + + assertThat(testRunner.getFlowFilesForRelationship(SUCCESS)).isEmpty(); + assertThat(testRunner.getFlowFilesForRelationship(FAILURE)).isEmpty(); + } + + @Test + void given_InvalidModel_when_Skolemise_AddModelToFailureRelation() { + testRunner.setProperty(SkolemisationTransformerProperties.SKOLEM_DOMAIN, "http://example.com"); + + testRunner.enqueue("invalid model", Map.of(MIME_TYPE_KEY, Lang.TURTLE.getHeaderString())); + + testRunner.run(); + + assertThat(testRunner.getFlowFilesForRelationship("failure")).hasSize(1); + } + + private Condition noBlankNodes() { + final Predicate isStmtWithBNode = statement -> statement.getSubject().isAnon() || statement.getObject().isAnon(); + return new Condition<>( + model -> !model.listStatements().filterKeep(isStmtWithBNode).hasNext(), + "Model cannot have blank nodes" + ); + } +} \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/test/resources/mob-hind-model.ttl b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/test/resources/mob-hind-model.ttl new file mode 100644 index 000000000..7285b6789 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/src/test/resources/mob-hind-model.ttl @@ -0,0 +1,24 @@ +@prefix dc: . +@prefix geosparql: . +@prefix xsd: . +@prefix adms: . +@prefix skos: . +@prefix gipod: . +@prefix ns0: . +@prefix m8g: . + + + dc:isVersionOf ; + a ; + geosparql:asWKT "POLYGON ((3.7337472847142124 51.04745170597559, 4.359276660355135 50.851907920816956, 4.711285586572245 50.84364854093491, 4.4020885567877315 51.214619167436666, 3.7337472847142124 51.04745170597559))"^^geosparql:wktLiteral ; + dc:created "2023-11-30T21:45:15+01:00"^^xsd:dateTime ; + adms:identifier [ + a adms:Identifier ; + skos:notation "10810464"^^gipod:gipodId ; + adms:schemaAgency "https://gipod.vlaanderen.be"@nl-be + ] ; + ns0:periode [ + m8g:endTime "2022-05-27T17:00:00Z"^^xsd:dateTime ; + m8g:startTime "2022-05-27T07:00:00Z"^^xsd:dateTime ; + a m8g:PeriodOfTime + ] . \ No newline at end of file diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index 97ab124a3..f09fdfd15 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -380,11 +380,21 @@ - ldio-virtuoso-out + ldio-http-sparql-out be.vlaanderen.informatievlaanderen.ldes.ldio - ldio-virtuoso-out + ldio-http-sparql-out + ${project.version} + + + + + ldio-skolemisation-transformer + + + be.vlaanderen.informatievlaanderen.ldes.ldio + ldio-skolemisation-transformer ${project.version} diff --git a/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/pom.xml similarity index 54% rename from ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/pom.xml rename to ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/pom.xml index 2b2c27e7e..6cb08e24d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/pom.xml @@ -1,26 +1,32 @@ - + 4.0.0 - ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio + ldio-connectors 2.10.0-SNAPSHOT - 4.0.0 - ldio-virtuoso-out + ldio-http-sparql-out - org.springframework - spring-web + be.vlaanderen.informatievlaanderen.ldes.ldi + http-sparql-out + ${project.version} be.vlaanderen.informatievlaanderen.ldes.ldio ldio-request-executor ${project.version} - compile + + + com.github.tomakehurst + wiremock-jre8-standalone + ${wiremock-jre8.version} + test diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpSparqlOut.java b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpSparqlOut.java new file mode 100644 index 000000000..3a7a0a0df --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpSparqlOut.java @@ -0,0 +1,27 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.HttpSparqlOut; +import be.vlaanderen.informatievlaanderen.ldes.ldi.exceptions.WriteActionFailedException; +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOutput; +import org.apache.jena.rdf.model.Model; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LdioHttpSparqlOut implements LdiOutput { + public static final String NAME = "Ldio:HttpSparqlOut"; + private static final Logger log = LoggerFactory.getLogger(LdioHttpSparqlOut.class); + private final HttpSparqlOut httpSparqlOut; + + public LdioHttpSparqlOut(HttpSparqlOut httpSparqlOut) { + this.httpSparqlOut = httpSparqlOut; + } + + @Override + public void accept(Model model) { + try { + httpSparqlOut.write(model); + } catch (WriteActionFailedException e) { + log.error(e.getMessage()); + } + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutAutoConfig.java new file mode 100644 index 000000000..abeef8697 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutAutoConfig.java @@ -0,0 +1,68 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.HttpSparqlOut; +import be.vlaanderen.informatievlaanderen.ldes.ldi.SkolemisationTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldi.factory.DeleteFunctionBuilder; +import be.vlaanderen.informatievlaanderen.ldes.ldi.factory.InsertFunctionBuilder; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation.EmptySkolemizer; +import be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation.Skolemizer; +import be.vlaanderen.informatievlaanderen.ldes.ldi.skolemisation.SkolemizerImpl; +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOutput; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.DeleteFunction; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.InsertFunction; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.SparqlQuery; +import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpSparqlOut; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioOutputConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpSparqlOut.NAME; + +@Configuration +public class LdioHttpSparqlOutAutoConfig { + + @SuppressWarnings("java:S6830") + @Bean(name = NAME) + public LdioHttpSparqlOutConfigurator ldiHttpSparqlOutConfigurator() { + return new LdioHttpSparqlOutConfigurator(); + } + + public static class LdioHttpSparqlOutConfigurator implements LdioOutputConfigurator { + + @Override + public LdiOutput configure(ComponentProperties properties) { + final LdioHttpSparqlOutProperties httpSparqlOutProperties = new LdioHttpSparqlOutProperties(properties); + final DeleteFunction deleteFunction = createDeleteFunction(httpSparqlOutProperties); + final InsertFunction insertFunction = httpSparqlOutProperties.getGraph() + .map(InsertFunctionBuilder::withGraph) + .orElseGet(InsertFunctionBuilder::create) + .build(); + final Skolemizer skolemizer = httpSparqlOutProperties.getSkolemisationDomain() + .map(SkolemisationTransformer::new) + .map(skolemisationTransformer -> (Skolemizer) new SkolemizerImpl(skolemisationTransformer)) + .orElseGet(EmptySkolemizer::new); + final RequestExecutor requestExecutor = new LdioRequestExecutorSupplier().getRequestExecutor(properties); + + final SparqlQuery sparqlQuery = new SparqlQuery(insertFunction, deleteFunction); + final HttpSparqlOut httpSparqlOut = new HttpSparqlOut(httpSparqlOutProperties.getEndpoint(), sparqlQuery, skolemizer, requestExecutor); + + return new LdioHttpSparqlOut(httpSparqlOut); + } + + private static DeleteFunction createDeleteFunction(LdioHttpSparqlOutProperties properties) { + if (!properties.isReplacementEnabled()) { + return DeleteFunctionBuilder.disabled(); + } + return properties.getReplacementDeleteFunction() + .map(DeleteFunction::ofQuery) + .orElseGet(() -> properties.getGraph() + .map(DeleteFunctionBuilder::withGraph) + .orElseGet(DeleteFunctionBuilder::create) + .withDepth(properties.getReplacementDepth()) + ); + } + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutProperties.java b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutProperties.java new file mode 100644 index 000000000..14a6343ee --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutProperties.java @@ -0,0 +1,52 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; + +import java.util.Optional; + +public class LdioHttpSparqlOutProperties { + public static final String ENDPOINT = "endpoint"; + public static final String GRAPH = "graph"; + public static final String SKOLEMISATION_SKOLEM_DOMAIN = "skolemisation.skolemDomain"; + public static final String REPLACEMENT_DEPTH = "replacement.depth"; + public static final String REPLACEMENT_ENABLED = "replacement.enabled"; + public static final String REPLACEMENT_DELETE_FUNCTION = "replacement.deleteFunction"; + public static final boolean DEFAULT_REPLACEMENT_ENABLED = true; + public static final int DEFAULT_REPLACEMENT_DEPTH = 10; + + private final ComponentProperties componentProperties; + + public LdioHttpSparqlOutProperties(ComponentProperties componentProperties) { + this.componentProperties = componentProperties; + } + + public String getEndpoint() { + return componentProperties.getProperty(ENDPOINT); + } + + public Optional getGraph() { + return componentProperties.getOptionalProperty(GRAPH); + } + + public Optional getSkolemisationDomain() { + return componentProperties.getOptionalProperty(SKOLEMISATION_SKOLEM_DOMAIN); + } + + public boolean isReplacementEnabled() { + return componentProperties + .getOptionalBoolean(REPLACEMENT_ENABLED) + .orElse(DEFAULT_REPLACEMENT_ENABLED); + } + + public int getReplacementDepth() { + return componentProperties + .getOptionalInteger(REPLACEMENT_DEPTH) + .orElse(DEFAULT_REPLACEMENT_DEPTH); + } + + public Optional getReplacementDeleteFunction() { + return componentProperties.getOptionalProperty(REPLACEMENT_DELETE_FUNCTION); + } + + +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpSparqlOutTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpSparqlOutTest.java new file mode 100644 index 000000000..5e7e78344 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpSparqlOutTest.java @@ -0,0 +1,145 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOutput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpSparqlOutAutoConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpSparqlOutProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.ConfigPropertyMissingException; +import com.github.tomakehurst.wiremock.client.BasicCredentials; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFParser; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@WireMockTest(httpPort = LdioHttpSparqlOutTest.PORT) +class LdioHttpSparqlOutTest { + static final int PORT = 12121; + private static final String PIPELINE_NAME = "test-pipeline"; + private static final String ENDPOINT = "http://localhost:%s/sparql".formatted(PORT); + private static final Map minimalProperties = Map.of( + LdioHttpSparqlOutProperties.ENDPOINT, ENDPOINT, + "graph", "http://example.graph.com" + ); + private static Model model; + private final LdioHttpSparqlOutAutoConfig.LdioHttpSparqlOutConfigurator configurator = new LdioHttpSparqlOutAutoConfig().ldiHttpSparqlOutConfigurator(); + private LdiOutput ldioHttpSparqlOut; + + @BeforeAll + static void beforeAll() { + model = RDFParser.source("mob-hind-model.ttl").lang(Lang.TTL).toModel(); + } + + @BeforeEach + void setUp() { + stubFor(post(urlEqualTo("/sparql")).willReturn(ok())); + } + + @Test + void given_ValidModel_when_SparqlOut_then_PostQuery() { + final Map properties = Map.of( + LdioHttpSparqlOutProperties.ENDPOINT, ENDPOINT, + LdioHttpSparqlOutProperties.REPLACEMENT_ENABLED, Boolean.FALSE.toString() + ); + final String content = " \"Jane Doe\" .\n"; + final Model contentModel = RDFParser.create().fromString(content).lang(Lang.NQUADS).toModel(); + final String expectedRequestBody = "INSERT DATA { %s }".formatted(content); + ldioHttpSparqlOut = configurator.configure(new ComponentProperties(PIPELINE_NAME, LdioHttpSparqlOut.NAME, properties)); + + ldioHttpSparqlOut.accept(contentModel); + + verify(postRequestedFor(urlEqualTo("/sparql")).withRequestBody(containing(expectedRequestBody))); + } + + @Test + void given_SkolemisationEnabled_when_SparqlOut_then_PostQuery() { + final String skolemDomain = "http://example.com"; + final Map properties = extendProperties(Map.of("skolemisation.skolemDomain", skolemDomain)); + ldioHttpSparqlOut = configurator.configure(new ComponentProperties(PIPELINE_NAME, LdioHttpSparqlOut.NAME, properties)); + + ldioHttpSparqlOut.accept(model); + + verify(postRequestedFor(urlEqualTo("/sparql")).withRequestBody(containing("%s/.well-known/genid/".formatted(skolemDomain)))); + } + + @Test + void given_GraphOmitted_when_SparqlOut_then_PostQuery() { + final Map properties = Map.of(LdioHttpSparqlOutProperties.ENDPOINT, ENDPOINT); + ldioHttpSparqlOut = configurator.configure(new ComponentProperties(PIPELINE_NAME, LdioHttpSparqlOut.NAME, properties)); + + ldioHttpSparqlOut.accept(model); + + verify(postRequestedFor(urlEqualTo("/sparql")).withRequestBody(notContaining("FROM"))); + } + + @Test + void given_CustomDeteFunction_when_SparqlOut_then_PostQuery() { + final String customDeleteFunction = "custom function"; + final Map properties = extendProperties(Map.of(LdioHttpSparqlOutProperties.REPLACEMENT_DELETE_FUNCTION, customDeleteFunction)); + ldioHttpSparqlOut = configurator.configure(new ComponentProperties(PIPELINE_NAME, LdioHttpSparqlOut.NAME, properties)); + + ldioHttpSparqlOut.accept(model); + + verify(postRequestedFor(urlEqualTo("/sparql")).withRequestBody(containing(customDeleteFunction).and(notContaining("FROM")))); + } + + @ParameterizedTest + @ArgumentsSource(AuthPropertiesProvider.class) + void given_AuthConfig_when_SparqlOut_then_PostQueryWithHeader(Map authProperties) { + final Map properties = extendProperties(authProperties); + ldioHttpSparqlOut = configurator.configure(new ComponentProperties(PIPELINE_NAME, LdioHttpSparqlOut.NAME, properties)); + + ldioHttpSparqlOut.accept(model); + + verify(postRequestedFor(urlEqualTo("/sparql")).withBasicAuth(AuthPropertiesProvider.basicCredentials)); + } + + @Test + void when_EndpointIsNotConfigured_then_ThrowMissingConfigPropertyException() { + ComponentProperties componentProperties = new ComponentProperties(PIPELINE_NAME, LdioHttpSparqlOut.NAME); + assertThatThrownBy(() -> configurator.configure(componentProperties)) + .isExactlyInstanceOf(ConfigPropertyMissingException.class) + .hasMessage("Pipeline \"test-pipeline\": \"Ldio:HttpSparqlOut\" : Missing value for property \"endpoint\" ."); + } + + private Map extendProperties(Map properties) { + final Map extendedProperties = new HashMap<>(minimalProperties); + extendedProperties.putAll(properties); + return extendedProperties; + } + + private static class AuthPropertiesProvider implements ArgumentsProvider { + static final String USERNAME = "sparql"; + static final String PASSWORD = "changeme"; + static final BasicCredentials basicCredentials = new BasicCredentials(USERNAME, PASSWORD); + + @Override + public Stream provideArguments(ExtensionContext extensionContext) throws Exception { + return Stream.of( + Map.of( + "auth.type", "API_KEY", + "auth.api-key", basicCredentials.asAuthorizationHeaderValue(), + "auth.api-key-header", "Authorization" + ), + Map.of( + "http.headers.0.key", "Authorization", + "http.headers.0.value", basicCredentials.asAuthorizationHeaderValue() + ) + ).map(Arguments::of); + } + } +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutPropertiesTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutPropertiesTest.java new file mode 100644 index 000000000..71c964ebc --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpSparqlOutPropertiesTest.java @@ -0,0 +1,117 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.ConfigPropertyMissingException; +import org.assertj.core.api.Condition; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpSparqlOutProperties.*; +import static org.assertj.core.api.Assertions.*; + +class LdioHttpSparqlOutPropertiesTest { + @Test + void test_Defaults() { + final String endpoint = "http://localhost:8888/sparql"; + final ComponentProperties componentProperties = new ComponentProperties("test-pipeline", "cname", Map.of("endpoint", endpoint)); + + final LdioHttpSparqlOutProperties httpSparqlOutProperties = new LdioHttpSparqlOutProperties(componentProperties); + + assertThat(httpSparqlOutProperties) + .hasFieldOrPropertyWithValue("endpoint", endpoint) + .has(emptyProperty(LdioHttpSparqlOutProperties::getSkolemisationDomain)) + .has(emptyProperty(LdioHttpSparqlOutProperties::getGraph)) + .is(replacementEnabled()) + .has(replacementDepth(10)) + .has(noReplacementDeleteFunction()); + } + + @Test + void given_MissingEndpoint_when_GetEndpoint_then_ThrowException() { + final ComponentProperties componentProperties = new ComponentProperties("test-pipeline", "cname"); + + final LdioHttpSparqlOutProperties httpSparqlOutProperties = new LdioHttpSparqlOutProperties(componentProperties); + + assertThatThrownBy(httpSparqlOutProperties::getEndpoint) + .isInstanceOf(ConfigPropertyMissingException.class) + .hasMessage("Pipeline \"test-pipeline\": \"cname\" : Missing value for property \"endpoint\" ."); + } + + @Test + void test_CustomSkolemDomain() { + final Map properties = Map.of(SKOLEMISATION_SKOLEM_DOMAIN, "http://example.org"); + final ComponentProperties componentProperties = new ComponentProperties("test-pipeline", "cname", properties); + + final LdioHttpSparqlOutProperties httpSparqlOutProperties = new LdioHttpSparqlOutProperties(componentProperties); + + assertThat(httpSparqlOutProperties.getSkolemisationDomain()).contains("http://example.org"); + } + + @Test + void test_CustomGraph() { + final Map properties = Map.of(GRAPH, "http://example.org"); + final ComponentProperties componentProperties = new ComponentProperties("test-pipeline", "cname", properties); + + final LdioHttpSparqlOutProperties httpSparqlOutProperties = new LdioHttpSparqlOutProperties(componentProperties); + + assertThat(httpSparqlOutProperties.getGraph()).contains("http://example.org"); + } + + @Nested + class Replacement { + @Test + void test_CustomReplacementDepth() { + final Map properties = Map.of(REPLACEMENT_DEPTH, "15"); + final ComponentProperties componentProperties = new ComponentProperties("test-pipeline", "cname", properties); + + final LdioHttpSparqlOutProperties httpSparqlOutProperties = new LdioHttpSparqlOutProperties(componentProperties); + + assertThat(httpSparqlOutProperties) + .is(replacementEnabled()) + .has(replacementDepth(15)) + .has(noReplacementDeleteFunction()); + } + + @Test + void test_DisabledReplacement() { + final Map properties = Map.of(REPLACEMENT_ENABLED, Boolean.FALSE.toString()); + final ComponentProperties componentProperties = new ComponentProperties("test-pipeline", "cname", properties); + + final LdioHttpSparqlOutProperties httpSparqlOutProperties = new LdioHttpSparqlOutProperties(componentProperties); + + assertThat(httpSparqlOutProperties).is(not(replacementEnabled())); + } + + @Test + void test_DeleteFunction() { + final String deleteFunction = "DELETE *"; + final Map properties = Map.of(REPLACEMENT_DELETE_FUNCTION, deleteFunction); + final ComponentProperties componentProperties = new ComponentProperties("test-pipeline", "cname", properties); + + final LdioHttpSparqlOutProperties httpSparqlOutProperties = new LdioHttpSparqlOutProperties(componentProperties); + + assertThat(httpSparqlOutProperties).is(replacementEnabled()); + assertThat(httpSparqlOutProperties.getReplacementDeleteFunction()).contains(deleteFunction); + } + } + + private Condition emptyProperty(Function> extractor) { + return new Condition<>(properties -> extractor.apply(properties).isEmpty(), "Expected property to be empty"); + } + + private Condition replacementEnabled() { + return new Condition<>(LdioHttpSparqlOutProperties::isReplacementEnabled, "replacement.enabled expected to be true"); + } + + private Condition replacementDepth(int depth) { + return new Condition<>(properties -> properties.getReplacementDepth() == depth, "replacement.depth expected to be %d", depth); + } + + private Condition noReplacementDeleteFunction() { + return new Condition<>(properties -> properties.getReplacementDeleteFunction().isEmpty(), "replacement.deleteFunction expected to be empty"); + } +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/resources/mob-hind-model.ttl b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/resources/mob-hind-model.ttl new file mode 100644 index 000000000..7285b6789 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/src/test/resources/mob-hind-model.ttl @@ -0,0 +1,24 @@ +@prefix dc: . +@prefix geosparql: . +@prefix xsd: . +@prefix adms: . +@prefix skos: . +@prefix gipod: . +@prefix ns0: . +@prefix m8g: . + + + dc:isVersionOf ; + a ; + geosparql:asWKT "POLYGON ((3.7337472847142124 51.04745170597559, 4.359276660355135 50.851907920816956, 4.711285586572245 50.84364854093491, 4.4020885567877315 51.214619167436666, 3.7337472847142124 51.04745170597559))"^^geosparql:wktLiteral ; + dc:created "2023-11-30T21:45:15+01:00"^^xsd:dateTime ; + adms:identifier [ + a adms:Identifier ; + skos:notation "10810464"^^gipod:gipodId ; + adms:schemaAgency "https://gipod.vlaanderen.be"@nl-be + ] ; + ns0:periode [ + m8g:endTime "2022-05-27T17:00:00Z"^^xsd:dateTime ; + m8g:startTime "2022-05-27T07:00:00Z"^^xsd:dateTime ; + a m8g:PeriodOfTime + ] . \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/pom.xml new file mode 100644 index 000000000..17af0a7c0 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.ldes.ldio + ldio-connectors + 2.10.0-SNAPSHOT + + + ldio-skolemisation-transformer + + + + be.vlaanderen.informatievlaanderen.ldes.ldi + skolemisation-transformer + ${project.version} + + + + + ${project.artifactId} + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + default-jar + none + + + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + + \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSkolemisationTransformer.java b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSkolemisationTransformer.java new file mode 100644 index 000000000..e922098b7 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSkolemisationTransformer.java @@ -0,0 +1,19 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.SkolemisationTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioTransformer; +import org.apache.jena.rdf.model.Model; + +public class LdioSkolemisationTransformer extends LdioTransformer { + public static final String NAME = "Ldio:SkolemisationTransformer"; + private final SkolemisationTransformer skolemisationTransformer; + + public LdioSkolemisationTransformer(SkolemisationTransformer skolemisationTransformer) { + this.skolemisationTransformer = skolemisationTransformer; + } + + @Override + public void apply(Model model) { + this.next(skolemisationTransformer.transform(model)); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioSkolemisationTransformerAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioSkolemisationTransformerAutoConfig.java new file mode 100644 index 000000000..575fc7025 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioSkolemisationTransformerAutoConfig.java @@ -0,0 +1,27 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.SkolemisationTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioSkolemisationTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioTransformerConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class LdioSkolemisationTransformerAutoConfig { + + @SuppressWarnings("java:S6830") + @Bean(LdioSkolemisationTransformer.NAME) + public LdioTransformerConfigurator ldioConfigurator() { + return new LdioSkolemisationTransformerConfigurator(); + } + + public static class LdioSkolemisationTransformerConfigurator implements LdioTransformerConfigurator { + @Override + public LdioTransformer configure(ComponentProperties config) { + String skolemDomain = config.getProperty("skolem-domain"); + return new LdioSkolemisationTransformer(new SkolemisationTransformer(skolemDomain)); + } + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSkolemisationTransformerTest.java b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSkolemisationTransformerTest.java new file mode 100644 index 000000000..dabea0450 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioSkolemisationTransformerTest.java @@ -0,0 +1,28 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.SkolemisationTransformer; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +class LdioSkolemisationTransformerTest { + @Mock + private SkolemisationTransformer skolemisationTransformer; + @InjectMocks + private LdioSkolemisationTransformer ldioSkolemisationTransformer; + + @Test + void test_Apply() { + Model model = ModelFactory.createDefaultModel(); + ldioSkolemisationTransformer.apply(model); + + verify(skolemisationTransformer).transform(model); + } +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioSkolemisationTransformerAutoConfigTest.java b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioSkolemisationTransformerAutoConfigTest.java new file mode 100644 index 000000000..81df6b104 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/config/LdioSkolemisationTransformerAutoConfigTest.java @@ -0,0 +1,42 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioSkolemisationTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioTransformerConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.ConfigPropertyMissingException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class LdioSkolemisationTransformerAutoConfigTest { + private LdioTransformerConfigurator ldioSkolemisationTransformerConfigurator; + + @BeforeEach + void setUp() { + ldioSkolemisationTransformerConfigurator = new LdioSkolemisationTransformerAutoConfig().ldioConfigurator(); + } + + @Test + void test_MissingProperty() { + final ComponentProperties properties = new ComponentProperties("pipeline", "component"); + + assertThatThrownBy(() -> ldioSkolemisationTransformerConfigurator.configure(properties)) + .isInstanceOf(ConfigPropertyMissingException.class) + .hasMessage("Pipeline \"pipeline\": \"component\" : Missing value for property \"skolem-domain\" ."); + } + + @Test + void test_Configure() { + final ComponentProperties properties = new ComponentProperties("pipeline", "component", Map.of("skolem-domain", "http://exampe.com")); + + final LdioTransformer result = ldioSkolemisationTransformerConfigurator.configure(properties); + + assertThat(result).isInstanceOf(LdioSkolemisationTransformer.class); + } + +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVirtuosoOut.java b/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVirtuosoOut.java deleted file mode 100644 index 29b9f73a4..000000000 --- a/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVirtuosoOut.java +++ /dev/null @@ -1,60 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.LdiRdfWriter; -import be.vlaanderen.informatievlaanderen.ldes.ldi.rdf.formatter.LdiRdfWriterProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.PostRequest; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeader; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeaders; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Response; -import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOutput; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.riot.Lang; -import org.apache.jena.riot.RDFWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpHeaders; - -import java.io.ByteArrayOutputStream; -import java.util.List; - -public class LdioVirtuosoOut implements LdiOutput { - public static final String NAME = "Ldio:VirtuosoOut"; - private static final Logger log = LoggerFactory.getLogger(LdioVirtuosoOut.class); - - private final RequestExecutor requestExecutor; - private final String endpoint; - private final String graph; - private final LdiRdfWriter ldiRdfWriter; - - public LdioVirtuosoOut(RequestExecutor requestExecutor, String endpoint, String graph) { - this.requestExecutor = requestExecutor; - this.endpoint = endpoint; - this.graph = graph; - this.ldiRdfWriter = LdiRdfWriter.getRdfWriter(new LdiRdfWriterProperties().withLang(Lang.NQUADS)); - } - - @Override - public void accept(Model linkedDataModel) { - if (!linkedDataModel.isEmpty()) { - final ByteArrayOutputStream output = new ByteArrayOutputStream(); - ldiRdfWriter.writeToOutputStream(linkedDataModel, output); - String modelAsNQuads = RDFWriter.source(linkedDataModel).lang(Lang.NQUADS).asString(); - String body = "INSERT INTO <%s> { %s }".formatted(graph, modelAsNQuads); - - final PostRequest request = new PostRequest(endpoint, new RequestHeaders(List.of( - new RequestHeader(HttpHeaders.CONTENT_TYPE, "application/sparql-update"), - new RequestHeader(HttpHeaders.ACCEPT, "application/json"))), body); - synchronized (requestExecutor) { - Response response = requestExecutor.execute(request); - if (response.isSuccess()) { - log.debug("{} {} {}", request.getMethod(), request.getUrl(), response.getHttpStatus()); - } else { - log.atError().log("Failed to post model. The request url was {}. " + - "The http response obtained from the server has code {} and body \"{}\".", - response.getRequestedUrl(), response.getHttpStatus(), response.getBodyAsString().orElse(null)); - } - } - } - } -} diff --git a/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVirtuosoOutAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVirtuosoOutAutoConfig.java deleted file mode 100644 index 60e628e35..000000000 --- a/ldi-orchestrator/ldio-connectors/ldio-virtuoso-out/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVirtuosoOutAutoConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio.config; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; -import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVirtuosoOut; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioOutputConfigurator; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVirtuosoOut.NAME; - -@Configuration -public class LdioVirtuosoOutAutoConfig { - - @SuppressWarnings("java:S6830") - @Bean(NAME) - public LdioOutputConfigurator ldiHttpOutConfigurator() { - return new LdioVirtuosoOutConfigurator(); - } - - public static class LdioVirtuosoOutConfigurator implements LdioOutputConfigurator { - - @Override - public LdiComponent configure(ComponentProperties config) { - final RequestExecutor requestExecutor = new LdioRequestExecutorSupplier().getRequestExecutor(config); - - String graph = config.getProperty("graph"); - String endpoint = config.getProperty("endpoint"); - - return new LdioVirtuosoOut(requestExecutor, endpoint, graph); - } - } -} diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index 248ae8ae5..ef2160f2c 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -40,7 +40,8 @@ ldio-file-out ldio-repository-sink ldio-change-detection-filter - ldio-virtuoso-out + ldio-skolemisation-transformer + ldio-http-sparql-out