From b02b2906154e3c63680bc3cba934b55614ec6b53 Mon Sep 17 00:00:00 2001 From: G TanSndil <72496065+GtanSndil@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:54:47 +0200 Subject: [PATCH] 684 complter la suppression dun jeu de donnes (#705) * feat: delete dataset completion delete the catalogRecord associated with the dataset delete any alternative identifier associated with the dataset block deletion if a derived dataset exists * feat: clean database after dataset deletion * test: let see with Fabrice * resolve conflicts * refactor: delete unuseful code --- .../datasets/DatasetQueries.java | 33 +++++ .../datasets/DatasetServiceImpl.java | 58 ++++++++- .../fr/insee/rmes/exceptions/ErrorCodes.java | 1 + ...teDatasetQualifiedDerivationWhiteNode.ftlh | 9 ++ ...eleteDatasetTemporalCoverageWhiteNode.ftlh | 9 ++ .../dataset/getDatasetDerivedFrom.ftlh | 21 +++ .../request/dataset/getDerivedDataset.ftlh | 9 ++ .../datasets/DatasetServiceImplTest.java | 12 +- .../TestDatasetsResourcesEnvProd.java | 120 +----------------- 9 files changed, 149 insertions(+), 123 deletions(-) create mode 100644 src/main/resources/request/dataset/deleteDatasetQualifiedDerivationWhiteNode.ftlh create mode 100644 src/main/resources/request/dataset/deleteDatasetTemporalCoverageWhiteNode.ftlh create mode 100644 src/main/resources/request/dataset/getDatasetDerivedFrom.ftlh create mode 100644 src/main/resources/request/dataset/getDerivedDataset.ftlh diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/datasets/DatasetQueries.java b/src/main/java/fr/insee/rmes/bauhaus_services/datasets/DatasetQueries.java index aed98b575..1466cf0c0 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/datasets/DatasetQueries.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/datasets/DatasetQueries.java @@ -73,4 +73,37 @@ public static String getDatasetContributors(IRI iri, String datasetsGraph) throw Map params = Map.of("GRAPH", datasetsGraph, "IRI", iri, "PREDICATE", "dc:contributor"); return FreeMarkerUtils.buildRequest("common/", "getContributors.ftlh", params); } + + public static String getDerivedDataset(String id, String datasetsGraph) throws RmesException { + HashMap params = new HashMap<>(); + params.put(DATASET_GRAPH, datasetsGraph); + params.put("ID", id); + return FreeMarkerUtils.buildRequest(ROOT_DIRECTORY, "getDerivedDataset.ftlh", params); + } + + public static String deleteTempWhiteNode(String id, String datasetsGraph) throws RmesException { + HashMap params = new HashMap<>(); + params.put(DATASET_GRAPH, datasetsGraph); + params.put("ID", id); + return FreeMarkerUtils.buildRequest(ROOT_DIRECTORY, "deleteDatasetTemporalCoverageWhiteNode.ftlh", params); + } + + public static String getDatasetDerivedFrom(String id, String datasetsGraph) throws RmesException { + HashMap params = new HashMap<>(); + params.put(DATASET_GRAPH, datasetsGraph); + params.put("ID", id); + return FreeMarkerUtils.buildRequest(ROOT_DIRECTORY, "getDatasetDerivedFrom.ftlh", params); + } + + + + public static String deleteDatasetQualifiedDerivationWhiteNode(String id, String datasetsGraph) throws RmesException { + HashMap params = new HashMap<>(); + params.put(DATASET_GRAPH, datasetsGraph); + params.put("ID", id); + return FreeMarkerUtils.buildRequest(ROOT_DIRECTORY, "deleteDatasetQualifiedDerivationWhiteNode.ftlh", params); + } + + + } \ No newline at end of file diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/datasets/DatasetServiceImpl.java b/src/main/java/fr/insee/rmes/bauhaus_services/datasets/DatasetServiceImpl.java index 2df06611d..035dfc8ab 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/datasets/DatasetServiceImpl.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/datasets/DatasetServiceImpl.java @@ -26,14 +26,19 @@ import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; - +import org.eclipse.rdf4j.model.BNode; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.model.Resource; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.regex.Pattern; + import static fr.insee.rmes.exceptions.ErrorCodes.DATASET_PATCH_INCORRECT_BODY; @Service @@ -277,24 +282,41 @@ public void patchDataset(String datasetId, PatchDataset patchDataset) throws Rme @Override public void deleteDatasetId(String datasetId) throws RmesException{ Dataset dataset = getDatasetByID(datasetId); - if (!isUnpublished(dataset)){ - throw new RmesBadRequestException(ErrorCodes.DATASET_DELETE_ONLY_UNPUBLISHED, "Only unpublished datasets can be deleted"); + if (isPublished(dataset)){ + throw new RmesBadRequestException(ErrorCodes.DATASET_DELETE_ONLY_UNPUBLISHED, "Only unpublished datasets can be deleted"); } if (hasDistribution(dataset)) { throw new RmesBadRequestException(ErrorCodes.DATASET_DELETE_ONLY_WITHOUT_DISTRIBUTION, "Only dataset without any distribution can be deleted"); } + if (hasDerivedDataset(dataset)) { + throw new RmesBadRequestException(ErrorCodes.DATASET_DELETE_ONLY_WITHOUT_DERIVED_DATASET, "Only dataset without any derived dataset can be deleted"); + } + IRI datasetIRI = RdfUtils.createIRI(getDatasetsBaseUri()); IRI graph = getDatasetIri(datasetId); String datasetURI = getDatasetsBaseUri() + "/" + datasetId; + IRI catalogRecordIRI = RdfUtils.createIRI(getCatalogRecordBaseUri() + "/" + datasetId); + IRI datasetAdmsIri = RdfUtils.createIRI(getDatasetsAdmsBaseUri() + "/" + datasetId); + + if (hasTemporalCoverage(dataset)){ + deleteTemporalWhiteNode(datasetId); + } + if (isDerivedFromADataset(dataset)){ + deleteQualifiedDerivationWhiteNode(datasetId); + } repoGestion.deleteObject(RdfUtils.toURI(datasetURI)); + repoGestion.deleteObject(catalogRecordIRI); + repoGestion.deleteObject(datasetAdmsIri); repoGestion.deleteTripletByPredicate(datasetIRI, DCAT.DATASET, graph); + } - private boolean isUnpublished(Dataset dataset) { - return "Unpublished".equalsIgnoreCase(dataset.getValidationState()); + + private boolean isPublished(Dataset dataset) { + return !"Unpublished".equalsIgnoreCase(dataset.getValidationState()); } private boolean hasDistribution(Dataset dataset) throws RmesException { @@ -302,6 +324,32 @@ private boolean hasDistribution(Dataset dataset) throws RmesException { return !getDistributions(datasetId).equals("[]"); } + private boolean hasDerivedDataset(Dataset dataset) throws RmesException { + String datasetId = dataset.getId(); + JSONObject datasetDerivation = this.repoGestion.getResponseAsObject(DatasetQueries.getDerivedDataset(datasetId, getDatasetsGraph())); + return (datasetDerivation.has("id")); + } + + private boolean hasTemporalCoverage(Dataset dataset) { + return !(dataset.getTemporalCoverageDataType() == null); + } + + private HttpStatus deleteTemporalWhiteNode(String id) throws RmesException { + HttpStatus result = repoGestion.executeUpdate(DatasetQueries.deleteTempWhiteNode(id, getDatasetsGraph())); + return result; + } + + private boolean isDerivedFromADataset(Dataset dataset) throws RmesException { + String datasetId = dataset.getId(); + JSONObject datasetDerivedFrom = this.repoGestion.getResponseAsObject(DatasetQueries.getDatasetDerivedFrom(datasetId, getDatasetsGraph())); + return (!datasetDerivedFrom.optString("wasDerivedFromS").isEmpty()); + } + + private HttpStatus deleteQualifiedDerivationWhiteNode(String id) throws RmesException { + HttpStatus result = repoGestion.executeUpdate(DatasetQueries.deleteDatasetQualifiedDerivationWhiteNode(id, getDatasetsGraph())); + return result; + } + private void persistCatalogRecord(Dataset dataset) throws RmesException { Resource graph = RdfUtils.createIRI(getDatasetsGraph()); IRI catalogRecordIRI = RdfUtils.createIRI(getCatalogRecordBaseUri() + "/" + dataset.getId()); diff --git a/src/main/java/fr/insee/rmes/exceptions/ErrorCodes.java b/src/main/java/fr/insee/rmes/exceptions/ErrorCodes.java index d7c67f4fa..9f1d4781d 100644 --- a/src/main/java/fr/insee/rmes/exceptions/ErrorCodes.java +++ b/src/main/java/fr/insee/rmes/exceptions/ErrorCodes.java @@ -125,6 +125,7 @@ public class ErrorCodes { public static final int DISTRIBUTION_DELETE_ONLY_UNPUBLISHED = 1203; public static final int DATASET_DELETE_ONLY_UNPUBLISHED = 1203 ; public static final int DATASET_DELETE_ONLY_WITHOUT_DISTRIBUTION = 1204 ; + public static final int DATASET_DELETE_ONLY_WITHOUT_DERIVED_DATASET = 1205 ; /* diff --git a/src/main/resources/request/dataset/deleteDatasetQualifiedDerivationWhiteNode.ftlh b/src/main/resources/request/dataset/deleteDatasetQualifiedDerivationWhiteNode.ftlh new file mode 100644 index 000000000..e8f21423b --- /dev/null +++ b/src/main/resources/request/dataset/deleteDatasetQualifiedDerivationWhiteNode.ftlh @@ -0,0 +1,9 @@ +DELETE WHERE { +GRAPH + { +?uri rdf:type dcat:Dataset . + ?uri prov:qualifiedDerivation ?node . + ?uri dcterms:identifier '${ID}' . + ?node rdf:type prov:Derivation . + } +} \ No newline at end of file diff --git a/src/main/resources/request/dataset/deleteDatasetTemporalCoverageWhiteNode.ftlh b/src/main/resources/request/dataset/deleteDatasetTemporalCoverageWhiteNode.ftlh new file mode 100644 index 000000000..d483e07b1 --- /dev/null +++ b/src/main/resources/request/dataset/deleteDatasetTemporalCoverageWhiteNode.ftlh @@ -0,0 +1,9 @@ +DELETE WHERE { +GRAPH <${DATASET_GRAPH}> + { +?uri rdf:type dcat:Dataset . + ?uri dcterms:temporal ?node . + ?uri dcterms:identifier '${ID}' . + ?node rdf:type dcterms:PeriodOfTime . + } +} \ No newline at end of file diff --git a/src/main/resources/request/dataset/getDatasetDerivedFrom.ftlh b/src/main/resources/request/dataset/getDatasetDerivedFrom.ftlh new file mode 100644 index 000000000..861c9ee21 --- /dev/null +++ b/src/main/resources/request/dataset/getDatasetDerivedFrom.ftlh @@ -0,0 +1,21 @@ +SELECT DISTINCT +?id +?uri +(GROUP_CONCAT(DISTINCT ?wasDerivedFromS; separator=" , ") AS ?wasDerivedFromS) + +FROM + +WHERE { + ?uri a dcat:Dataset ; + dcterms:identifier '${ID}' ; + dcterms:identifier ?id ; + + OPTIONAL {?uri prov:wasDerivedFrom ?wasDerivedFrom . + ?wasDerivedFrom dcterms:identifier ?wasDerivedFromS + } . + + +} +GROUP BY ?id ?uri + + diff --git a/src/main/resources/request/dataset/getDerivedDataset.ftlh b/src/main/resources/request/dataset/getDerivedDataset.ftlh new file mode 100644 index 000000000..2639ee006 --- /dev/null +++ b/src/main/resources/request/dataset/getDerivedDataset.ftlh @@ -0,0 +1,9 @@ +SELECT DISTINCT ?id +FROM <${DATASET_GRAPH}> +WHERE { + ?uri a dcat:Dataset ; + dcterms:identifier ?id . + ?uri prov:wasDerivedFrom ?wasDerivedFrom . + ?wasDerivedFrom dcterms:identifier '${ID}' . +} +GROUP BY ?id \ No newline at end of file diff --git a/src/test/java/fr/insee/rmes/bauhaus_services/datasets/DatasetServiceImplTest.java b/src/test/java/fr/insee/rmes/bauhaus_services/datasets/DatasetServiceImplTest.java index e19c2959b..395bf20dd 100644 --- a/src/test/java/fr/insee/rmes/bauhaus_services/datasets/DatasetServiceImplTest.java +++ b/src/test/java/fr/insee/rmes/bauhaus_services/datasets/DatasetServiceImplTest.java @@ -89,6 +89,7 @@ class DatasetServiceImplTest { } ]"""; public static final String EMPTY_ARRAY = "[]"; + public static final String QUASI_EMPTY_OBJECT = "{\"uri\":\"\"}"; @Test void shouldReturnDatasets() throws RmesException { @@ -669,6 +670,7 @@ void shouldDeleteDataSet() throws RmesException{ "}\n" + "]"); JSONArray empty_array = new JSONArray(EMPTY_ARRAY); + JSONObject quasi_empty_object = new JSONObject(QUASI_EMPTY_OBJECT); String stringDatasetURI="http://bauhaus/catalogues/entreeCatalogue/idtest"; IRI datasetUri= RdfUtils.toURI(stringDatasetURI); @@ -689,6 +691,12 @@ void shouldDeleteDataSet() throws RmesException{ distributionQueriesMock.when(() -> DistributionQueries.getDatasetDistributions(any(), any())).thenReturn("query3 "); when(repositoryGestion.getResponseAsArray("query3 ")).thenReturn(empty_array); + datasetQueriesMock.when(() -> DatasetQueries.getDerivedDataset(any(), any())).thenReturn("query4 "); + when(repositoryGestion.getResponseAsObject("query4 ")).thenReturn(quasi_empty_object); + + datasetQueriesMock.when(() -> DatasetQueries.getDatasetDerivedFrom(any(), any())).thenReturn("query5 "); + when(repositoryGestion.getResponseAsObject("query5 ")).thenReturn(quasi_empty_object); + rdfUtilsMock.when(() -> RdfUtils.createIRI(any(String.class))).thenReturn(datasetUri); rdfUtilsMock.when(() -> RdfUtils.toURI(any(String.class))).thenReturn(datasetUri); @@ -697,10 +705,10 @@ void shouldDeleteDataSet() throws RmesException{ datasetService.deleteDatasetId("idTest"); - verify(repositoryGestion, times(1)).deleteObject(uriCaptor.capture()); + verify(repositoryGestion, times(3)).deleteObject(uriCaptor.capture()); Assertions.assertEquals(datasetUri, uriCaptor.getValue()); - verify(repositoryGestion, times(1)).deleteObject(datasetUri); + verify(repositoryGestion, times(3)).deleteObject(datasetUri); verify(repositoryGestion, times(1)).deleteTripletByPredicate(any(IRI.class), eq(DCAT.DATASET), any(IRI.class)); } } diff --git a/src/test/java/fr/insee/rmes/integration/authorizations/TestDatasetsResourcesEnvProd.java b/src/test/java/fr/insee/rmes/integration/authorizations/TestDatasetsResourcesEnvProd.java index 941361dae..b2730e0bd 100644 --- a/src/test/java/fr/insee/rmes/integration/authorizations/TestDatasetsResourcesEnvProd.java +++ b/src/test/java/fr/insee/rmes/integration/authorizations/TestDatasetsResourcesEnvProd.java @@ -1,11 +1,7 @@ package fr.insee.rmes.integration.authorizations; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import fr.insee.rmes.bauhaus_services.StampAuthorizationChecker; import fr.insee.rmes.bauhaus_services.datasets.DatasetService; -import fr.insee.rmes.bauhaus_services.datasets.DatasetServiceImpl; -import fr.insee.rmes.bauhaus_services.rdf_utils.RepositoryGestion; import fr.insee.rmes.config.Config; import fr.insee.rmes.config.auth.UserProviderFromSecurityContext; import fr.insee.rmes.config.auth.roles.Roles; @@ -14,22 +10,15 @@ import fr.insee.rmes.config.auth.security.DefaultSecurityContext; import fr.insee.rmes.config.auth.security.OpenIDConnectSecurityContext; import fr.insee.rmes.config.auth.user.Stamp; -import fr.insee.rmes.exceptions.RmesException; -import fr.insee.rmes.model.ValidationStatus; import fr.insee.rmes.model.dataset.Dataset; -import fr.insee.rmes.model.dataset.PatchDataset; import fr.insee.rmes.webservice.dataset.DatasetResources; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; -import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; -import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.security.oauth2.jwt.JwtDecoder; import org.springframework.test.web.servlet.MockMvc; @@ -37,12 +26,8 @@ import java.util.List; import static fr.insee.rmes.integration.authorizations.TokenForTestsConfiguration.*; -import static fr.insee.rmes.model.ValidationStatus.UNPUBLISHED; -import static org.assertj.core.api.Assertions.assertThat; -import static org.hamcrest.Matchers.containsString; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @WebMvcTest(controllers = DatasetResources.class, @@ -74,6 +59,8 @@ class TestDatasetsResourcesEnvProd { private JwtDecoder jwtDecoder; @MockBean StampAuthorizationChecker stampAuthorizationChecker; + @MockBean + DatasetService datasetService; private static Dataset dataset; @@ -81,7 +68,8 @@ class TestDatasetsResourcesEnvProd { private final String timbre = "XX59-YYY"; int datasetId=10; - ValidationStatus status= UNPUBLISHED; + + @Test void shouldGetDatasetsWithAnyRole() throws Exception { @@ -317,18 +305,6 @@ void shouldNotDeleteADataset() throws Exception { .andExpect(status().isForbidden()); } - @Test - void shouldNotDeleteNotUnpublishedDataset() throws Exception { - configureJwtDecoderMock(jwtDecoder, idep, timbre, List.of(Roles.ADMIN)); - dataset=new Dataset(); - dataset.setValidationState("Published"); - mvc.perform(delete("/datasets/" + datasetId).header("Authorization", "Bearer toto") - .contentType(MediaType.APPLICATION_JSON) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(result-> assertThat(result.getResponse().getStatus()).isIn(HttpStatus.BAD_REQUEST.value())) - .andExpect(content().string(containsString("Only unpublished datasets can be deleted"))); - } - @Test void shouldPatchADatasetIfAdmin() throws Exception { configureJwtDecoderMock(jwtDecoder, idep, timbre, List.of(Roles.ADMIN)); @@ -372,92 +348,4 @@ void shouldNotPatchADataset() throws Exception { } - static class DatasetServiceImplStub extends DatasetServiceImpl{ - - - public DatasetServiceImplStub(){ - super.repoGestion= Mockito.mock(RepositoryGestion.class); - // repoGestion.deleteObject(RdfUtils.toURI(datasetURI)); - // repoGestion.deleteTripletByPredicate(datasetIRI, DCAT.DATASET, graph); - } - - @Override - public Dataset getDatasetByID(String id) { - final ObjectMapper mapper=new ObjectMapper(); - return dataset; - } - - @Override - public String getDistributions(String id) { - return "[]"; - } - - @Override - protected String getDatasetsBaseUri(){ - return "http://bauhaus/datasets"; - } - - } - - @TestConfiguration - static class ConfigureDatasetServiceForTest{ - - @Bean - DatasetService datasetService(){ - final DatasetServiceImplStub realInstance = new DatasetServiceImplStub(); - return new DatasetService() { - @Override - public String getDatasets() throws RmesException { - return ""; - } - - @Override - public Dataset getDatasetByID(String id) throws RmesException { - return null; - } - - @Override - public String update(String datasetId, String body) throws RmesException { - return ""; - } - - @Override - public String create(String body) throws RmesException { - return ""; - } - - @Override - public String getDistributions(String id) throws RmesException { - return ""; - } - - @Override - public String getArchivageUnits() throws RmesException { - return ""; - } - - @Override - public void patchDataset(String datasetId, PatchDataset patchDataset) throws RmesException { - - } - - @Override - public String getDatasetsForDistributionCreation(String stamp) throws RmesException { - return ""; - } - - @Override - public String publishDataset(String id) throws RmesException { - return ""; - } - - @Override - public void deleteDatasetId(String datasetId) throws RmesException { - realInstance.deleteDatasetId(datasetId); - } - }; - } - - } - }