From 98ceeef4d5fd6190479f3d0130ff612f1ab856ec Mon Sep 17 00:00:00 2001 From: Jonas Bulcke <127748878+jobulcke@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:06:12 +0100 Subject: [PATCH] fix: root page was falsely deleted (#1464) * fix: root page was falsely deleted * feat: cleaner db query fix: root page was marked as immutable too soon * fix: broken file name reference --- docs/how-to-run.md | 2 +- .../factory/RootBucketCreatorImpl.java | 2 +- .../repository/BucketRepository.java | 2 +- .../RootBucketCreatorImplTest.java | 2 +- .../postgres/BucketPostgresRepository.java | 6 +- .../BucketPostgresRepositoryTest.java | 2 +- .../db/changelog/3_6_1/add_is_root_column.xml | 20 +++++ .../db/changelog/3_6_1/add_is_root_values.sql | 7 ++ .../changelog/3_6_1/alter_open_pages_view.sql | 16 ++++ .../resources/db/changelog/3_6_1/master.xml | 9 ++ .../main/resources/db/changelog/master.xml | 1 + .../postgres/entity/CompactionPageEntity.java | 3 + .../CompactionPageEntityRepository.java | 1 + .../postgres/entity/PageEntity.java | 3 + .../repository/PageEntityRepository.java | 2 +- .../ldes/server/CompactionServiceSteps.java | 90 +++++++++++++++++++ .../resources/application-postgres-test.yml | 4 +- .../eventstreams/compaction/observations.ttl | 31 +++++++ .../input/members/observation.template.json | 20 +++++ .../features/maintenance/compaction.feature | 16 +++- 20 files changed, 224 insertions(+), 15 deletions(-) create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_column.xml create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_values.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/alter_open_pages_view.sql create mode 100644 ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/master.xml create mode 100644 ldes-server-integration-test/src/test/resources/data/input/eventstreams/compaction/observations.ttl create mode 100644 ldes-server-integration-test/src/test/resources/data/input/members/observation.template.json diff --git a/docs/how-to-run.md b/docs/how-to-run.md index 2a80d54f6..8826916cd 100644 --- a/docs/how-to-run.md +++ b/docs/how-to-run.md @@ -136,7 +136,7 @@ Here is an explanation provided for all the possibilities on how to tweak and co ldes-server.compaction-duration Defines how long the redundant compacted fragments will remain on the server No - PD7 + P7D Maintenance diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootBucketCreatorImpl.java b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootBucketCreatorImpl.java index 272843a1d..c3fd3907c 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootBucketCreatorImpl.java +++ b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/factory/RootBucketCreatorImpl.java @@ -16,7 +16,7 @@ public RootBucketCreatorImpl(BucketRepository bucketRepository) { @Override public void createRootBucketForView(ViewName viewName) { if (bucketRepository.retrieveRootBucket(viewName).isEmpty()) { - bucketRepository.insertBucket(Bucket.createRootBucketForView(viewName)); + bucketRepository.insertRootBucket(Bucket.createRootBucketForView(viewName)); } } } diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java index b2adaec89..ed400e58d 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java +++ b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/repository/BucketRepository.java @@ -6,6 +6,6 @@ import java.util.Optional; public interface BucketRepository { - Bucket insertBucket(Bucket bucket); + Bucket insertRootBucket(Bucket bucket); Optional retrieveRootBucket(ViewName viewName); } diff --git a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketCreatorImplTest.java b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketCreatorImplTest.java index d8cde4cf0..ebee82945 100644 --- a/ldes-server-fragmentation/ldes-server-fragmentation-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketCreatorImplTest.java +++ b/ldes-server-fragmentation/ldes-server-fragmentation-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/RootBucketCreatorImplTest.java @@ -29,7 +29,7 @@ void when_RootFragmentDoesNotExist_ItIsCreatedAndSaved() { InOrder inOrder = inOrder(bucketRepository); inOrder.verify(bucketRepository).retrieveRootBucket(VIEW_NAME); - inOrder.verify(bucketRepository).insertBucket(Bucket.createRootBucketForView(VIEW_NAME)); + inOrder.verify(bucketRepository).insertRootBucket(Bucket.createRootBucketForView(VIEW_NAME)); inOrder.verifyNoMoreInteractions(); } diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java index b9edd5f64..3a80d763a 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepository.java @@ -31,10 +31,10 @@ public BucketPostgresRepository(ViewEntityRepository viewEntityRepository, Bucke @Override @Transactional - public Bucket insertBucket(Bucket bucket) { + public Bucket insertRootBucket(Bucket bucket) { String sql = """ - INSERT INTO pages (bucket_id, expiration, partial_url) - VALUES (:bucketId, NULL, :partialUrl) + INSERT INTO pages (bucket_id, expiration, partial_url, is_root) + VALUES (:bucketId, NULL, :partialUrl, true) ON CONFLICT DO NOTHING """; ViewEntity view = viewEntityRepository.findByViewName(bucket.getViewName().getCollectionName(), bucket.getViewName().getViewName()) diff --git a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java index 2ad37cf08..e36c07155 100644 --- a/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java +++ b/ldes-server-infra-postgres/postgres-fragmentation-repository/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/fragmentation/postgres/BucketPostgresRepositoryTest.java @@ -41,7 +41,7 @@ void test_Insertion() { final Bucket bucketToSave = new Bucket(BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME); final Bucket expectedSavedBucket = new Bucket(1L, BucketDescriptor.of(new BucketDescriptorPair("key", "value"), new BucketDescriptorPair("k", "v")), VIEW_NAME, List.of(), 0); - final Bucket result = bucketPostgresRepository.insertBucket(bucketToSave); + final Bucket result = bucketPostgresRepository.insertRootBucket(bucketToSave); assertThat(result) .usingRecursiveComparison() diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_column.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_column.xml new file mode 100644 index 000000000..63f420607 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_column.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_values.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_values.sql new file mode 100644 index 000000000..d0836c3fd --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/add_is_root_values.sql @@ -0,0 +1,7 @@ +WITH root_page_partial_urls AS ( + SELECT CONCAT('/', c.name, '/', v.name) AS partial_url + FROM collections c + JOIN views v ON c.collection_id = v.collection_id +) +UPDATE pages SET is_root = true, immutable = false +WHERE partial_url IN (SELECT partial_url FROM root_page_partial_urls); \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/alter_open_pages_view.sql b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/alter_open_pages_view.sql new file mode 100644 index 000000000..ed34ebcb0 --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/alter_open_pages_view.sql @@ -0,0 +1,16 @@ +DROP VIEW IF EXISTS open_pages; + +CREATE VIEW open_pages AS +SELECT DISTINCT ON (p.bucket_id) p.page_id, + p.bucket_id, + p.partial_url, + v.page_size, + count(pm.member_id) AS assigned_members +FROM pages p + JOIN buckets b ON b.bucket_id = p.bucket_id + JOIN views v ON v.view_id = b.view_id + LEFT JOIN page_members pm ON pm.page_id = p.page_id +WHERE NOT p.immutable +GROUP BY p.page_id, v.page_size, p.bucket_id, p.is_root +ORDER BY p.bucket_id, p.is_root +; diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/master.xml new file mode 100644 index 000000000..67f0e9a8b --- /dev/null +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/3_6_1/master.xml @@ -0,0 +1,9 @@ + + + + + + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml index a46826b4d..9d4083462 100644 --- a/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml +++ b/ldes-server-infra-postgres/postgres-liquibase/src/main/resources/db/changelog/master.xml @@ -12,5 +12,6 @@ + \ No newline at end of file diff --git a/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/entity/CompactionPageEntity.java b/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/entity/CompactionPageEntity.java index c85cf9539..a6808b624 100644 --- a/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/entity/CompactionPageEntity.java +++ b/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/entity/CompactionPageEntity.java @@ -27,6 +27,9 @@ public class CompactionPageEntity { @Column(name = "partial_url", nullable = false, unique = true) private String partialUrl; + @Column(name = "is_root", nullable = false, columnDefinition = "BOOLEAN") + private boolean isRoot; + public CompactionPageEntity() { } diff --git a/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/repository/CompactionPageEntityRepository.java b/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/repository/CompactionPageEntityRepository.java index b7a09057a..581352782 100644 --- a/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/repository/CompactionPageEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-maintenance-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/maintenance/postgres/repository/CompactionPageEntityRepository.java @@ -22,6 +22,7 @@ SELECT p.page_id as fragmentId, COUNT(*) AS size, r.to_page_id AS toPage, p.buck WHERE c.name = :collectionName AND v.name = :viewName AND p.expiration IS NULL AND p.immutable + AND NOT p.is_root GROUP BY p.page_id, r.to_page_id HAVING COUNT(*) < :capacityPerPage """, nativeQuery = true) diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java index 7bd32d38c..bc21bfb7e 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/entity/PageEntity.java @@ -30,6 +30,9 @@ public class PageEntity { @Column(name = "partial_url", nullable = false, unique = true) private String partialUrl; + @Column(name = "is_root", nullable = false, columnDefinition = "BOOLEAN") + private boolean isRoot; + @OneToMany(mappedBy = "fromPage") private List relations; diff --git a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java index 02054ee99..281d6e14d 100644 --- a/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java +++ b/ldes-server-infra-postgres/postgres-pagination-repository/src/main/java/be/vlaanderen/informatievlaanderen/ldes/server/pagination/postgres/repository/PageEntityRepository.java @@ -7,7 +7,7 @@ public interface PageEntityRepository extends JpaRepository { @Modifying - @Query(value = "UPDATE pages SET immutable = true WHERE page_id = ?", nativeQuery = true) + @Query(value = "UPDATE pages SET immutable = true WHERE page_id = ? AND NOT is_root", nativeQuery = true) void setPageImmutable(long pageId); @Modifying diff --git a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java index d22bbdf79..323215a13 100644 --- a/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java +++ b/ldes-server-integration-test/src/test/java/be/vlaanderen/informatievlaanderen/ldes/server/CompactionServiceSteps.java @@ -2,8 +2,15 @@ import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageEntity; import be.vlaanderen.informatievlaanderen.ldes.server.pagination.postgres.entity.PageRelationEntity; +import be.vlaanderen.informatievlaanderen.ldes.server.resultactionsextensions.ResponseToModelConverter; +import io.cucumber.java.After; +import io.cucumber.java.Before; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import org.apache.jena.rdf.model.ResourceFactory; +import org.apache.jena.riot.Lang; +import org.apache.jena.vocabulary.RDF; import java.io.IOException; import java.net.URISyntaxException; @@ -17,17 +24,34 @@ import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Collectors; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @SuppressWarnings("java:S3415") public class CompactionServiceSteps extends LdesServerIntegrationTest { private int versionIncremeter = 1; + private ScheduledExecutorService executorService; + private Future seedingTask; + + @Before + public void setup() { + executorService = Executors.newSingleThreadScheduledExecutor(); + } + + @After + public void cleanup() { + executorService.shutdown(); + } @And("I ingest {int} members of different versions") public void ingestDifferentVersions(int amount) throws Exception { @@ -144,4 +168,70 @@ private boolean isValidUuid(String pageNumber) { return false; } } + + @And("I start seeding {int} members every {int} seconds") + public void iStartSeedingMembersEverySeconds(int numberOfMembers, int seconds) { + seedingTask = executorService.scheduleAtFixedRate(() -> ingestNumberOfVersionObjects(numberOfMembers), 0, seconds, SECONDS); + } + + private void ingestNumberOfVersionObjects(int numberOfMembers) { + try { + String memberTemplate = readMemberTemplate("data/input/members/observation.template.json"); + for (int i = 0; i < numberOfMembers; i++) { + String memberContent = memberTemplate + .replace("ID", String.valueOf(i)) + .replace("DATETIME", getCurrentTimestamp()); + mockMvc.perform(post("/observations") + .contentType(Lang.JSONLD.getHeaderString()) + .content(memberContent)) + .andExpect(status().is2xxSuccessful()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Then("I wait until {int} members are ingested") + public void iWaitUntilMembersAreIngested(int number) { + await().atMost(Duration.ofMinutes(3)) + .pollInterval(Duration.ofSeconds(15)) + .untilAsserted(() -> assertThat(jdbcTemplate.queryForObject("SELECT COUNT(*) FROM members", Long.class)).isEqualTo(number)); + } + + @Then("I stop seeding members") + public void iStopSeedingMembers() { + seedingTask.cancel(true); + } + + + @When("I wait until the first page does not exits anymore") + public void iWaitUntilThePageWithPageNumberDoesNotExitsAnymore() { + await() + .atMost(Duration.ofMinutes(2)) + .pollInterval(Duration.ofSeconds(5)) + .untilAsserted(() -> mockMvc.perform(get("/observations/time-based?pageNumber=1")) + .andExpect(status().isNotFound())); + } + + @And("I only have one open page") + public void iOnlyHaveOneOpenPage() { + final Long openPageCount = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM open_pages", Long.class); + assertThat(openPageCount).isEqualTo(1); + } + + @Then("the root page points to a compacted page") + public void theRootPagePointsToACompactedPage() throws Exception { + final var response = mockMvc.perform(get("/observations/time-based").accept(Lang.NQ.getHeaderString())) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse(); + final String pageNumber = new ResponseToModelConverter(response).convert() + .listSubjectsWithProperty(RDF.type, ResourceFactory.createProperty("https://w3id.org/tree#Relation")) + .nextResource() + .listProperties(ResourceFactory.createProperty("https://w3id.org/tree#node")) + .nextStatement() + .getResource() + .getLocalName(); + assertThatNoException().isThrownBy(() -> UUID.fromString(pageNumber)); + } } diff --git a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml index cc3f2571b..8ebf6316c 100644 --- a/ldes-server-integration-test/src/test/resources/application-postgres-test.yml +++ b/ldes-server-integration-test/src/test/resources/application-postgres-test.yml @@ -1,8 +1,8 @@ ldes-server: host-name: "http://localhost:8080" - compaction-duration: "*/10 * * * * *" fragmentation-cron: "*/10 * * * * *" - maintenance-cron: "*/10 * * * * *" + maintenance-cron: "*/5 * * * * *" + compaction-duration: PT2M springdoc.swaggerui.path: "/swagger" management: diff --git a/ldes-server-integration-test/src/test/resources/data/input/eventstreams/compaction/observations.ttl b/ldes-server-integration-test/src/test/resources/data/input/eventstreams/compaction/observations.ttl new file mode 100644 index 000000000..468618430 --- /dev/null +++ b/ldes-server-integration-test/src/test/resources/data/input/eventstreams/compaction/observations.ttl @@ -0,0 +1,31 @@ +@prefix ldes: . +@prefix tree: . +@prefix sh: . +@prefix ex: . +@prefix xsd: . +@prefix dcterms: . + + a ldes:EventStream ; + ldes:timestampPath dcterms:created ; + ldes:versionOfPath dcterms:isVersionOf ; + tree:shape [ a sh:NodeShape ] ; + tree:view ; + ldes:eventSource [ + a ldes:EventSource ; + ldes:retentionPolicy [ + a ldes:DurationAgoPolicy ; + tree:value "PT2M"^^xsd:duration + ] ; + ] . + + a tree:Node ; + tree:viewDescription [ + a tree:ViewDescription ; + tree:fragmentationStrategy () ; + tree:pageSize "7"^^xsd:integer ; + ldes:retentionPolicy [ + a ldes:DurationAgoPolicy ; + tree:value "PT90S"^^xsd:duration + ] + ] +. \ No newline at end of file diff --git a/ldes-server-integration-test/src/test/resources/data/input/members/observation.template.json b/ldes-server-integration-test/src/test/resources/data/input/members/observation.template.json new file mode 100644 index 000000000..799083b4f --- /dev/null +++ b/ldes-server-integration-test/src/test/resources/data/input/members/observation.template.json @@ -0,0 +1,20 @@ +{ + "@context": { + "@base": "http://example.org/id/", + "@vocab": "http://purl.org/dc/terms/", + "isVersionOf": { + "@type": "@id" + } + }, + "@id": "ID#DATETIME", + "@type": [ + "something" + ], + "created": [ + { + "@value": "DATETIME", + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + ], + "isVersionOf": "http://example.org/id/ID" +} \ No newline at end of file diff --git a/ldes-server-integration-test/src/test/resources/features/maintenance/compaction.feature b/ldes-server-integration-test/src/test/resources/features/maintenance/compaction.feature index 959f09446..c067384d5 100644 --- a/ldes-server-integration-test/src/test/resources/features/maintenance/compaction.feature +++ b/ldes-server-integration-test/src/test/resources/features/maintenance/compaction.feature @@ -1,13 +1,11 @@ Feature: LDES Server Compaction - Background: + Scenario: Execution Compaction Given I create the eventstream "data/input/eventstreams/compaction/mobility-hindrances_paginated_5.ttl" And I ingest 6 members of different versions And I ingest 5 members of the same version And I ingest 5 members of the same version And I ingest 3 members of different versions - - Scenario: Execution Compaction Then I wait until all members are fragmented Then wait until no fragments can be compacted And verify there are 5 pages @@ -21,4 +19,14 @@ Feature: LDES Server Compaction And verify the following pages no longer exist | 2 | | 3 | - And the background processes did not fail \ No newline at end of file + And the background processes did not fail + + Scenario: Retention Compaction And Deletion works fine together + Given I create the eventstream "data/input/eventstreams/compaction/observations.ttl" + And I start seeding 5 members every 15 seconds + When I wait until the first page does not exits anymore + Then the root page points to a compacted page + And I only have one open page + Then I stop seeding members + And I delete the eventstream "observations" + And the background processes did not fail