From 796bd2cd50570094a3874ffa2d1ff2e8cc4c1742 Mon Sep 17 00:00:00 2001 From: Alice Lambois Date: Wed, 23 Oct 2024 16:41:34 +0200 Subject: [PATCH] Rename services and extract volumetrics --- .../controller/rest/ResponseController.java | 29 +- .../controller/rest/UtilsController.java | 34 +- .../rest/ResponseControllerTest.java | 243 ++---------- .../controller/rest/UtilsControllerTest.java | 372 ++++++++++++++++++ 4 files changed, 446 insertions(+), 232 deletions(-) create mode 100644 src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java diff --git a/src/main/java/fr/insee/genesis/controller/rest/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/ResponseController.java index 41ce1b7..c7b77c1 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/ResponseController.java @@ -22,7 +22,6 @@ import fr.insee.genesis.domain.model.surveyunit.Variable; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; -import fr.insee.genesis.domain.service.volumetry.VolumetryLogService; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.exceptions.NoDataError; @@ -58,7 +57,7 @@ import java.util.Set; import java.util.stream.Stream; -@RequestMapping(path = "/response" ) +@RequestMapping(path = "/responses" ) @Controller @Tag(name = "Response services for interrogations", description = "A **response** is considered the entire set of data associated with an interrogation (idUE x idQuestionnaire). \n\n These data may have different state (collected, edited, external, ...) ") @Slf4j @@ -68,22 +67,20 @@ public class ResponseController { public static final String S_S = "%s/%s"; private final SurveyUnitApiPort surveyUnitService; private final SurveyUnitQualityService surveyUnitQualityService; - private final VolumetryLogService volumetryLogService; private final FileUtils fileUtils; private final ControllerUtils controllerUtils; @Autowired - public ResponseController(SurveyUnitApiPort surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, VolumetryLogService volumetryLogService, FileUtils fileUtils, ControllerUtils controllerUtils) { + public ResponseController(SurveyUnitApiPort surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils, ControllerUtils controllerUtils) { this.surveyUnitService = surveyUnitService; this.surveyUnitQualityService = surveyUnitQualityService; - this.volumetryLogService = volumetryLogService; this.fileUtils = fileUtils; this.controllerUtils = controllerUtils; } //SAVE @Operation(summary = "Save one file of responses to Genesis Database, passing its path as a parameter") - @PutMapping(path = "/save/lunatic-xml/one-file") + @PutMapping(path = "/lunatic-xml/save-one") public ResponseEntity saveResponsesFromXmlFile(@RequestParam("pathLunaticXml") String xmlFile, @RequestParam(value = "pathSpecFile") String metadataFilePath, @RequestParam(value = "mode") Mode modeSpecified, @@ -118,7 +115,7 @@ public ResponseEntity saveResponsesFromXmlFile(@RequestParam("pathLunati } @Operation(summary = "Save multiple files to Genesis Database from the campaign root folder") - @PutMapping(path = "/save/lunatic-xml") + @PutMapping(path = "/lunatic-xml/save-folder") public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam("campaignName") String campaignName, @RequestParam(value = "mode", required = false) Mode modeSpecified, @RequestParam(value = "withDDI", defaultValue = "true") boolean withDDI @@ -140,7 +137,7 @@ public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam(" //SAVE ALL @Operation(summary = "Save all files to Genesis Database (differential data folder only), regardless of the campaign") - @PutMapping(path = "/save/lunatic-xml/all-campaigns") + @PutMapping(path = "/lunatic-xml/save-all-campaigns") public ResponseEntity saveResponsesFromAllCampaignFolders(){ List errors = new ArrayList<>(); List campaignFolders = fileUtils.listAllSpecsFolders(); @@ -170,17 +167,11 @@ public ResponseEntity saveResponsesFromAllCampaignFolders(){ } } - @Operation(summary = "Record volumetrics of each campaign in a folder") - @PutMapping(path = "/save-volumetry/all-campaigns") - public ResponseEntity saveVolumetry() throws IOException { - volumetryLogService.writeVolumetries(surveyUnitService); - volumetryLogService.cleanOldFiles(); - return ResponseEntity.ok("Volumetric saved"); - } + //DELETE @Operation(summary = "Delete all responses associated with a questionnaire") - @DeleteMapping(path = "/delete-responses/by-questionnaire") + @DeleteMapping(path = "/delete/by-questionnaire") public ResponseEntity deleteAllResponsesByQuestionnaire(@RequestParam("idQuestionnaire") String idQuestionnaire) { log.info("Try to delete all responses of questionnaire : {}", idQuestionnaire); Long ndDocuments = surveyUnitService.deleteByIdQuestionnaire(idQuestionnaire); @@ -199,7 +190,7 @@ public ResponseEntity> findResponsesByUEAndQuestionnaire(@ @Operation(summary = "Retrieve responses for an interrogation, using IdUE and IdQuestionnaire from Genesis Database with the latest value for each available state of every variable") @GetMapping(path = "/get-responses/by-ue-and-questionnaire/latest-states") - public ResponseEntity findResponsesByUEAndQuestionnaireLastestStates( + public ResponseEntity findResponsesByUEAndQuestionnaireLatestStates( @RequestParam("idUE") String idUE, @RequestParam("idQuestionnaire") String idQuestionnaire) { SurveyUnitDto response = surveyUnitService.findLatestValuesByStateByIdAndByIdQuestionnaire(idUE, idQuestionnaire); @@ -236,8 +227,8 @@ public ResponseEntity> getLatestByUE(@RequestParam("idUE") return ResponseEntity.ok(responses); } - @Operation(summary = "Retrieve response latest state with IdUE and IdQuestionnaire in one object in the output") - @GetMapping(path = "/get-simplified-response/by-ue-and-questionnaire/latest") + @Operation(summary = "Retrieve responses for an interrogation, using IdUE and IdQuestionnaire from Genesis Database. It returns only the latest value of each variable regardless of the state. The result is in one object in the output") + @GetMapping(path = "/get-simplified-response/by-ue-questionnaire-and-mode/latest") public ResponseEntity getLatestByUEOneObject(@RequestParam("idUE") String idUE, @RequestParam("idQuestionnaire") String idQuestionnaire, @RequestParam("mode") Mode mode) { diff --git a/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java b/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java index a39d1ad..c7d8683 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java @@ -1,21 +1,41 @@ package fr.insee.genesis.controller.rest; +import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.volumetry.VolumetryLogService; import fr.insee.genesis.domain.utils.XMLSplitter; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -@RequestMapping(path = "/utils") +import java.io.IOException; + +@RequestMapping(path = "/") @Controller @Slf4j +@Tag(name = "Technical services") public class UtilsController { + private final VolumetryLogService volumetryLogService; + private final SurveyUnitApiPort surveyUnitService; + + + @Autowired + public UtilsController(SurveyUnitApiPort surveyUnitService,VolumetryLogService volumetryLogService) { + this.surveyUnitService = surveyUnitService; + this.volumetryLogService = volumetryLogService; + + } + + + @Operation(summary = "Split a XML file into smaller ones") - @PutMapping(path = "/split/lunatic-xml") + @PutMapping(path = "/utils/split/lunatic-xml") public ResponseEntity saveResponsesFromXmlFile(@RequestParam("inputFolder") String inputFolder, @RequestParam("outputFolder") String outputFolder, @RequestParam("filename") String filename, @@ -23,6 +43,14 @@ public ResponseEntity saveResponsesFromXmlFile(@RequestParam("inputFolde throws Exception { log.info("Split XML file : {} into {} SU by file", filename, nbSU); XMLSplitter.split(inputFolder, filename, outputFolder, "SurveyUnit", nbSU); - return ResponseEntity.ok("Test"); + return ResponseEntity.ok("File split"); + } + + @Operation(summary = "Record volumetrics of each campaign in a folder") + @PutMapping(path = "/volumetrics/save-all-campaigns") + public ResponseEntity saveVolumetry() throws IOException { + volumetryLogService.writeVolumetries(surveyUnitService); + volumetryLogService.cleanOldFiles(); + return ResponseEntity.ok("Volumetric saved"); } } diff --git a/src/test/java/fr/insee/genesis/controller/rest/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/ResponseControllerTest.java index ff6c1f7..dc673f2 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ResponseControllerTest.java @@ -2,20 +2,19 @@ import cucumber.TestConstants; import fr.insee.genesis.Constants; -import fr.insee.genesis.controller.dto.SurveyUnitSimplified; +import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire; +import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; import fr.insee.genesis.controller.dto.SurveyUnitDto; -import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; -import fr.insee.genesis.domain.service.volumetry.VolumetryLogService; +import fr.insee.genesis.controller.dto.SurveyUnitId; +import fr.insee.genesis.controller.dto.SurveyUnitSimplified; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire; import fr.insee.genesis.domain.model.surveyunit.CollectedVariable; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; -import fr.insee.genesis.controller.dto.SurveyUnitId; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.Variable; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; @@ -31,15 +30,12 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.LocalDate; import java.time.LocalDateTime; import java.time.Month; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Set; -import java.util.stream.Stream; class ResponseControllerTest { //Given @@ -60,7 +56,6 @@ static void init() { responseControllerStatic = new ResponseController( surveyUnitApiPort , new SurveyUnitQualityService() - , new VolumetryLogService(new ConfigStub()) , fileUtils , new ControllerUtils(fileUtils) ); @@ -74,24 +69,8 @@ void reset() throws IOException { //MongoDB stub management surveyUnitPersistencePortStub.getMongoStub().clear(); - List externalVariableList = new ArrayList<>(); - Variable variable = Variable.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); - externalVariableList.add(variable); + addAdditionalDtoToMongoStub("TESTIDCAMPAIGN", defaultIdQuest ,LocalDateTime.of(2023, 1, 1, 0, 0, 0),LocalDateTime.of(2024, 1, 1, 0, 0, 0)); - List collectedVariableList = new ArrayList<>(); - CollectedVariable collectedVariable = new CollectedVariable("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); - collectedVariableList.add(collectedVariable); - surveyUnitPersistencePortStub.getMongoStub().add(SurveyUnitModel.builder() - .idCampaign("TESTIDCAMPAIGN") - .mode(Mode.WEB) - .idUE(defaultIdUE) - .idQuest(defaultIdQuest) - .state(DataState.COLLECTED) - .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0, 0)) - .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) - .externalVariables(externalVariableList) - .collectedVariables(collectedVariableList) - .build()); //Test file management //Clean DONE folder @@ -333,25 +312,7 @@ void getAllResponsesByQuestionnaireTestSequential() throws IOException { surveyUnitPersistencePortStub.getMongoStub().clear(); for (int i = 0; i < Constants.BATCH_SIZE + 2; i++) { - List externalVariableList = new ArrayList<>(); - Variable variable = Variable.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); - externalVariableList.add(variable); - - List collectedVariableList = new ArrayList<>(); - CollectedVariable collectedVariable = new CollectedVariable("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); - collectedVariableList.add(collectedVariable); - - surveyUnitPersistencePortStub.getMongoStub().add(SurveyUnitModel.builder() - .idCampaign("TESTIDCAMPAIGN") - .mode(Mode.WEB) - .idUE(defaultIdUE + i) - .idQuest(defaultIdQuest) - .state(DataState.COLLECTED) - .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0, 0)) - .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) - .externalVariables(externalVariableList) - .collectedVariables(collectedVariableList) - .build()); + addAdditionalDtoToMongoStub("TESTIDCAMPAIGN", defaultIdUE + i,LocalDateTime.of(2023, 1, 1, 0, 0, 0),LocalDateTime.of(2024, 1, 1, 0, 0, 0)); } //When @@ -370,7 +331,7 @@ void getAllResponsesByQuestionnaireTestSequential() throws IOException { @Test void getLatestByUETest() { - addAdditionnalDtoToMongoStub(); + addAdditionalDtoToMongoStub(); ResponseEntity> response = responseControllerStatic.getLatestByUE(defaultIdUE, defaultIdQuest); @@ -429,7 +390,7 @@ void getModesByCampaignTest() { @Test void getCampaignsTest() { - addAdditionnalDtoToMongoStub("TESTCAMPAIGN2","TESTQUESTIONNAIRE2"); + addAdditionalDtoToMongoStub("TESTCAMPAIGN2","TESTQUESTIONNAIRE2"); ResponseEntity> response = responseControllerStatic.getCampaigns(); @@ -440,7 +401,7 @@ void getCampaignsTest() { @Test void getQuestionnairesTest() { - addAdditionnalDtoToMongoStub("TESTQUESTIONNAIRE2"); + addAdditionalDtoToMongoStub("TESTQUESTIONNAIRE2"); ResponseEntity> response = responseControllerStatic.getQuestionnaires(); @@ -451,7 +412,7 @@ void getQuestionnairesTest() { @Test void getQuestionnairesByCampaignTest() { - addAdditionnalDtoToMongoStub("TESTQUESTIONNAIRE2"); + addAdditionalDtoToMongoStub("TESTQUESTIONNAIRE2"); ResponseEntity> response = responseControllerStatic.getQuestionnairesByCampaign("TESTIDCAMPAIGN"); @@ -462,8 +423,8 @@ void getQuestionnairesByCampaignTest() { @Test void getCampaignsWithQuestionnairesTest() { - addAdditionnalDtoToMongoStub("TESTQUESTIONNAIRE2"); - addAdditionnalDtoToMongoStub("TESTCAMPAIGN2","TESTQUESTIONNAIRE2"); + addAdditionalDtoToMongoStub("TESTQUESTIONNAIRE2"); + addAdditionalDtoToMongoStub("TESTCAMPAIGN2","TESTQUESTIONNAIRE2"); ResponseEntity> response = responseControllerStatic.getCampaignsWithQuestionnaires(); @@ -485,8 +446,8 @@ void getCampaignsWithQuestionnairesTest() { @Test void getQuestionnairesWithCampaignsTest() { - addAdditionnalDtoToMongoStub("TESTQUESTIONNAIRE2"); - addAdditionnalDtoToMongoStub("TESTCAMPAIGN2","TESTQUESTIONNAIRE2"); + addAdditionalDtoToMongoStub("TESTQUESTIONNAIRE2"); + addAdditionalDtoToMongoStub("TESTCAMPAIGN2","TESTQUESTIONNAIRE2"); ResponseEntity> response = responseControllerStatic.getQuestionnairesWithCampaigns(); @@ -508,119 +469,13 @@ void getQuestionnairesWithCampaignsTest() { ).findFirst().get().getCampaigns()).containsExactly("TESTIDCAMPAIGN", "TESTCAMPAIGN2"); } - @Test - void saveVolumetryTest() throws IOException { - //WHEN - ResponseEntity response = responseControllerStatic.saveVolumetry(); - - //THEN - Path logFilePath = Path.of( - new ConfigStub().getLogFolder()) - .resolve(Constants.VOLUMETRY_FOLDER_NAME) - .resolve(LocalDate.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) - + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); - Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(logFilePath).exists().content().isNotEmpty().contains("TESTIDCAMPAIGN;1"); - - //CLEAN - Files.deleteIfExists(logFilePath); - } - - @Test - void saveVolumetryTest_overwrite() throws IOException { - //WHEN - responseControllerStatic.saveVolumetry(); - ResponseEntity response = responseControllerStatic.saveVolumetry(); - - //THEN - Path logFilePath = Path.of( - new ConfigStub().getLogFolder()) - .resolve(Constants.VOLUMETRY_FOLDER_NAME) - .resolve(LocalDate.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) - + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); - Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(logFilePath).exists().content().isNotEmpty().contains("TESTIDCAMPAIGN;1").doesNotContain("TESTIDCAMPAIGN;1\nTESTIDCAMPAIGN;1"); - - //CLEAN - Files.deleteIfExists(logFilePath); - } - - @Test - void saveVolumetryTest_additionnal_campaign() throws IOException { - //Given - addAdditionnalDtoToMongoStub("TESTIDCAMPAIGN2","TESTQUEST2"); - - //WHEN - ResponseEntity response = responseControllerStatic.saveVolumetry(); - - //THEN - Path logFilePath = Path.of( - new ConfigStub().getLogFolder()) - .resolve(Constants.VOLUMETRY_FOLDER_NAME) - .resolve(LocalDate.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) - + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); - Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(logFilePath).exists().content().isNotEmpty().contains("TESTIDCAMPAIGN;1").contains("TESTIDCAMPAIGN2;1"); - - //CLEAN - Files.deleteIfExists(logFilePath); - } - @Test - void saveVolumetryTest_additionnal_campaign_and_document() throws IOException { - //Given - addAdditionnalDtoToMongoStub("TESTQUEST"); - addAdditionnalDtoToMongoStub("TESTIDCAMPAIGN2","TESTQUEST2"); - - //WHEN - ResponseEntity response = responseControllerStatic.saveVolumetry(); - - //THEN - Path logFilePath = Path.of( - new ConfigStub().getLogFolder()) - .resolve(Constants.VOLUMETRY_FOLDER_NAME) - .resolve(LocalDate.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) - + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); - Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(logFilePath).exists().content().isNotEmpty().contains("TESTIDCAMPAIGN;2").contains("TESTIDCAMPAIGN2;1"); - - //CLEAN - Files.deleteIfExists(logFilePath); - } - - @Test - void cleanOldVolumetryLogFiles() throws IOException { - //GIVEN - Path oldLogFilePath = Path.of( - new ConfigStub().getLogFolder()) - .resolve(Constants.VOLUMETRY_FOLDER_NAME) - .resolve(LocalDate.of(2000,1,1).format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) - + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); - - Files.createDirectories(oldLogFilePath.getParent()); - Files.write(oldLogFilePath, "test".getBytes()); - - //WHEN - ResponseEntity response = responseControllerStatic.saveVolumetry(); - - //THEN - Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(oldLogFilePath).doesNotExist(); - - //CLEAN - try(Stream stream = Files.walk(oldLogFilePath.getParent())){ - for(Path filePath : stream.filter(path -> path.getFileName().toString().endsWith(".csv")).toList()){ - Files.deleteIfExists(filePath); - } - } - } - // Perret tests @Test void getPerretSurveyDataTest(){ //GIVEN //Recent Collected already in stub //Old Collected - addAdditionnalDtoToMongoStub(DataState.COLLECTED, + addAdditionalDtoToMongoStub(DataState.COLLECTED, "C OLD C", // "E OLD C", LocalDateTime.of(1999,2,2,0,0,0), @@ -628,7 +483,7 @@ void getPerretSurveyDataTest(){ ); //Recent Edited - addAdditionnalDtoToMongoStub(DataState.EDITED, + addAdditionalDtoToMongoStub(DataState.EDITED, "C NEW E", "E NEW E", LocalDateTime.of(2025,2,2,0,0,0), @@ -636,7 +491,7 @@ void getPerretSurveyDataTest(){ ); //Old Edited - addAdditionnalDtoToMongoStub(DataState.EDITED, + addAdditionalDtoToMongoStub(DataState.EDITED, "C OLD E", "E OLD E", LocalDateTime.of(1999,2,2,0,0,0), @@ -645,7 +500,7 @@ void getPerretSurveyDataTest(){ //WHEN - ResponseEntity response = responseControllerStatic.findResponsesByUEAndQuestionnaireLastestStates( + ResponseEntity response = responseControllerStatic.findResponsesByUEAndQuestionnaireLatestStates( defaultIdUE, defaultIdQuest ); @@ -708,53 +563,19 @@ void getPerretSurveyDataTest(){ // Utilities - private void addAdditionnalDtoToMongoStub() { - List externalVariableList = new ArrayList<>(); - Variable variable = Variable.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); - externalVariableList.add(variable); - - List collectedVariableList = new ArrayList<>(); - CollectedVariable collectedVariable = new CollectedVariable("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); - collectedVariableList.add(collectedVariable); - - SurveyUnitModel recentDTO = SurveyUnitModel.builder() - .idCampaign("TESTIDCAMPAIGN") - .mode(Mode.WEB) - .idUE(defaultIdUE) - .idQuest(defaultIdQuest) - .state(DataState.COLLECTED) - .fileDate(LocalDateTime.of(2023, 2, 2, 0, 0, 0)) - .recordDate(LocalDateTime.of(2024, 2, 2, 0, 0, 0)) - .externalVariables(externalVariableList) - .collectedVariables(collectedVariableList) - .build(); - surveyUnitPersistencePortStub.getMongoStub().add(recentDTO); + private void addAdditionalDtoToMongoStub() { + addAdditionalDtoToMongoStub(defaultIdQuest); } - private void addAdditionnalDtoToMongoStub(String idQuestionnaire) { - List externalVariableList = new ArrayList<>(); - Variable variable = Variable.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); - externalVariableList.add(variable); - - List collectedVariableList = new ArrayList<>(); - CollectedVariable collectedVariable = new CollectedVariable("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); - collectedVariableList.add(collectedVariable); + private void addAdditionalDtoToMongoStub(String idQuestionnaire) { + addAdditionalDtoToMongoStub("TESTIDCAMPAIGN",idQuestionnaire); + } - SurveyUnitModel recentDTO = SurveyUnitModel.builder() - .idCampaign("TESTIDCAMPAIGN") - .mode(Mode.WEB) - .idUE(defaultIdUE) - .idQuest(idQuestionnaire) - .state(DataState.COLLECTED) - .fileDate(LocalDateTime.of(2023, 2, 2, 0, 0, 0)) - .recordDate(LocalDateTime.of(2024, 2, 2, 0, 0, 0)) - .externalVariables(externalVariableList) - .collectedVariables(collectedVariableList) - .build(); - surveyUnitPersistencePortStub.getMongoStub().add(recentDTO); + private void addAdditionalDtoToMongoStub(String idCampaign, String idQuestionnaire) { + addAdditionalDtoToMongoStub(idCampaign,idQuestionnaire,LocalDateTime.of(2023, 2, 2, 0, 0, 0),LocalDateTime.of(2024, 2, 2, 0, 0, 0)); } - private void addAdditionnalDtoToMongoStub(String idCampaign, String idQuestionnaire) { + private void addAdditionalDtoToMongoStub(String idCampaign, String idQuestionnaire, LocalDateTime fileDate, LocalDateTime recordDate) { List externalVariableList = new ArrayList<>(); Variable variable = Variable.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); externalVariableList.add(variable); @@ -769,15 +590,17 @@ private void addAdditionnalDtoToMongoStub(String idCampaign, String idQuestionna .idUE(defaultIdUE) .idQuest(idQuestionnaire) .state(DataState.COLLECTED) - .fileDate(LocalDateTime.of(2023, 2, 2, 0, 0, 0)) - .recordDate(LocalDateTime.of(2024, 2, 2, 0, 0, 0)) + .fileDate(fileDate) + .recordDate(recordDate) .externalVariables(externalVariableList) .collectedVariables(collectedVariableList) .build(); surveyUnitPersistencePortStub.getMongoStub().add(recentDTO); } - private void addAdditionnalDtoToMongoStub(DataState state, + + + private void addAdditionalDtoToMongoStub(DataState state, String collectedVariableValue, String externalVariableValue, LocalDateTime fileDate, diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java new file mode 100644 index 0000000..9867067 --- /dev/null +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -0,0 +1,372 @@ +package fr.insee.genesis.controller.rest; + +import cucumber.TestConstants; +import fr.insee.genesis.Constants; +import fr.insee.genesis.controller.dto.SurveyUnitId; +import fr.insee.genesis.domain.model.surveyunit.CollectedVariable; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.Variable; +import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.domain.service.volumetry.VolumetryLogService; +import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.ResponseEntity; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + +class UtilsControllerTest { + //Given + static UtilsController utilsControllerStatic; + static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; + + static List surveyUnitIdList; + //Constants + static final String defaultIdUE = "TESTIDUE"; + static final String defaultIdQuest = "TESTIDQUESTIONNAIRE"; + + @BeforeAll + static void init() { + surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); + SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService(surveyUnitPersistencePortStub); + + utilsControllerStatic = new UtilsController( + surveyUnitApiPort + , new VolumetryLogService(new ConfigStub()) + ); + + surveyUnitIdList = new ArrayList<>(); + surveyUnitIdList.add(new SurveyUnitId(defaultIdUE)); + } + + @BeforeEach + void reset() throws IOException { + //MongoDB stub management + surveyUnitPersistencePortStub.getMongoStub().clear(); + + List externalVariableList = new ArrayList<>(); + Variable variable = Variable.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); + externalVariableList.add(variable); + + List collectedVariableList = new ArrayList<>(); + CollectedVariable collectedVariable = new CollectedVariable("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); + collectedVariableList.add(collectedVariable); + surveyUnitPersistencePortStub.getMongoStub().add(SurveyUnitModel.builder() + .idCampaign("TESTIDCAMPAIGN") + .mode(Mode.WEB) + .idUE(defaultIdUE) + .idQuest(defaultIdQuest) + .state(DataState.COLLECTED) + .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0, 0)) + .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) + .externalVariables(externalVariableList) + .collectedVariables(collectedVariableList) + .build()); + + //Test file management + //Clean DONE folder + Path testResourcesPath = Path.of(TestConstants.TEST_RESOURCES_DIRECTORY); + if (testResourcesPath.resolve("DONE").toFile().exists()) + Files.walk(testResourcesPath.resolve("DONE")) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + + //Recreate data files + //SAMPLETEST-PARADATA-v1 + //Root + if (!testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("data.complete.validated.STPDv1.20231122164209.xml") + .toFile().exists() + ){ + Files.copy( + testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("reponse-platine") + .resolve("data.complete.partial.STPDv1.20231122164209.xml") + , testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("data.complete.partial.STPDv1.20231122164209.xml") + ); + Files.copy( + testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("reponse-platine") + .resolve("data.complete.validated.STPDv1.20231122164209.xml") + , testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("data.complete.validated.STPDv1.20231122164209.xml") + ); + } + //Differential data + if (!testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("differential") + .resolve("data") + .resolve("data.complete.validated.STPDv1.20231122164209.xml") + .toFile().exists() + ) { + Files.copy( + testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("reponse-platine") + .resolve("data.complete.partial.STPDv1.20231122164209.xml") + , testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("differential") + .resolve("data") + .resolve("data.complete.partial.STPDv1.20231122164209.xml") + ); + Files.copy( + testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("reponse-platine") + .resolve("data.complete.validated.STPDv1.20231122164209.xml") + , testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("differential") + .resolve("data") + .resolve("data.complete.validated.STPDv1.20231122164209.xml") + ); + } + //SAMPLETEST-PARADATA-v2 + if (!testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("data.complete.validated.STPDv2.20231122164209.xml") + .toFile().exists() + ){ + Files.copy( + testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("reponse-platine") + .resolve("data.complete.partial.STPDv2.20231122164209.xml") + , testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("data.complete.partial.STPDv2.20231122164209.xml") + ); + Files.copy( + testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("reponse-platine") + .resolve("data.complete.validated.STPDv2.20231122164209.xml") + , testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("data.complete.validated.STPDv2.20231122164209.xml") + ); + } + //SAMPLETEST-NO-COLLECTED + if (!testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-NO-COLLECTED") + .resolve("differential") + .resolve("data") + .resolve("data_diff_no_collected.xml") + .toFile().exists() + ){ + Files.copy( + testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-NO-COLLECTED") + .resolve("data_diff_no_collected.xml") + , testResourcesPath + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-NO-COLLECTED") + .resolve("differential") + .resolve("data") + .resolve("data_diff_no_collected.xml") + ); + } + } + + + + + @Test + void saveVolumetryTest() throws IOException { + //WHEN + ResponseEntity response = utilsControllerStatic.saveVolumetry(); + + //THEN + Path logFilePath = Path.of( + new ConfigStub().getLogFolder()) + .resolve(Constants.VOLUMETRY_FOLDER_NAME) + .resolve(LocalDate.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) + + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + Assertions.assertThat(logFilePath).exists().content().isNotEmpty().contains("TESTIDCAMPAIGN;1"); + + //CLEAN + Files.deleteIfExists(logFilePath); + } + + @Test + void saveVolumetryTest_overwrite() throws IOException { + //WHEN + utilsControllerStatic.saveVolumetry(); + ResponseEntity response = utilsControllerStatic.saveVolumetry(); + + //THEN + Path logFilePath = Path.of( + new ConfigStub().getLogFolder()) + .resolve(Constants.VOLUMETRY_FOLDER_NAME) + .resolve(LocalDate.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) + + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + Assertions.assertThat(logFilePath).exists().content().isNotEmpty().contains("TESTIDCAMPAIGN;1").doesNotContain("TESTIDCAMPAIGN;1\nTESTIDCAMPAIGN;1"); + + //CLEAN + Files.deleteIfExists(logFilePath); + } + + @Test + void saveVolumetryTest_additionnal_campaign() throws IOException { + //Given + addAdditionalDtoToMongoStub("TESTIDCAMPAIGN2","TESTQUEST2"); + + //WHEN + ResponseEntity response = utilsControllerStatic.saveVolumetry(); + + //THEN + Path logFilePath = Path.of( + new ConfigStub().getLogFolder()) + .resolve(Constants.VOLUMETRY_FOLDER_NAME) + .resolve(LocalDate.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) + + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + Assertions.assertThat(logFilePath).exists().content().isNotEmpty().contains("TESTIDCAMPAIGN;1").contains("TESTIDCAMPAIGN2;1"); + + //CLEAN + Files.deleteIfExists(logFilePath); + } + @Test + void saveVolumetryTest_additionnal_campaign_and_document() throws IOException { + //Given + addAdditionalDtoToMongoStub("TESTQUEST"); + addAdditionalDtoToMongoStub("TESTIDCAMPAIGN2","TESTQUEST2"); + + //WHEN + ResponseEntity response = utilsControllerStatic.saveVolumetry(); + + //THEN + Path logFilePath = Path.of( + new ConfigStub().getLogFolder()) + .resolve(Constants.VOLUMETRY_FOLDER_NAME) + .resolve(LocalDate.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) + + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + Assertions.assertThat(logFilePath).exists().content().isNotEmpty().contains("TESTIDCAMPAIGN;2").contains("TESTIDCAMPAIGN2;1"); + + //CLEAN + Files.deleteIfExists(logFilePath); + } + + @Test + void cleanOldVolumetryLogFiles() throws IOException { + //GIVEN + Path oldLogFilePath = Path.of( + new ConfigStub().getLogFolder()) + .resolve(Constants.VOLUMETRY_FOLDER_NAME) + .resolve(LocalDate.of(2000,1,1).format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)) + + Constants.VOLUMETRY_FILE_SUFFIX + ".csv"); + + Files.createDirectories(oldLogFilePath.getParent()); + Files.write(oldLogFilePath, "test".getBytes()); + + //WHEN + ResponseEntity response = utilsControllerStatic.saveVolumetry(); + + //THEN + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + Assertions.assertThat(oldLogFilePath).doesNotExist(); + + //CLEAN + try(Stream stream = Files.walk(oldLogFilePath.getParent())){ + for(Path filePath : stream.filter(path -> path.getFileName().toString().endsWith(".csv")).toList()){ + Files.deleteIfExists(filePath); + } + } + } + + + + + // Utilities + + private void addAdditionalDtoToMongoStub(String idQuestionnaire) { + addAdditionalDtoToMongoStub("TESTIDCAMPAIGN",idQuestionnaire); + } + + private void addAdditionalDtoToMongoStub(String idCampaign, String idQuestionnaire) { + List externalVariableList = new ArrayList<>(); + Variable variable = Variable.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); + externalVariableList.add(variable); + + List collectedVariableList = new ArrayList<>(); + CollectedVariable collectedVariable = new CollectedVariable("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); + collectedVariableList.add(collectedVariable); + + SurveyUnitModel recentDTO = SurveyUnitModel.builder() + .idCampaign(idCampaign) + .mode(Mode.WEB) + .idUE(defaultIdUE) + .idQuest(idQuestionnaire) + .state(DataState.COLLECTED) + .fileDate(LocalDateTime.of(2023, 2, 2, 0, 0, 0)) + .recordDate(LocalDateTime.of(2024, 2, 2, 0, 0, 0)) + .externalVariables(externalVariableList) + .collectedVariables(collectedVariableList) + .build(); + surveyUnitPersistencePortStub.getMongoStub().add(recentDTO); + } + +}