diff --git a/.gitignore b/.gitignore index 87359273..9005c2ab 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,5 @@ src/bin/ hs_err_pid* # test generated files -src/test/resources/OUT/* \ No newline at end of file +src/test/resources/OUT/* +/src/test/resources/DONE/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 212b9c0a..6f653b8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 1.2.2 - [NOT-YET-RELEASED] +### Added +- Treat all campaigns endpoint +### Fixed +- Fixed health check always returning version n.a + ## 1.2.1 - [2024-04-24] First production version diff --git a/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java b/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java index 89d37d3c..6f604eaf 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/HealthCheckController.java @@ -1,35 +1,57 @@ package fr.insee.genesis.controller.rest; +import fr.insee.genesis.domain.ports.api.ScheduleApiPort; +import fr.insee.genesis.domain.ports.api.SurveyUnitUpdateApiPort; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.info.BuildProperties; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.ResponseEntity; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.util.Optional; - @RequestMapping("/health-check") @RestController public class HealthCheckController { + private final SurveyUnitUpdateApiPort surveyUnitUpdateApiPort; + private final ScheduleApiPort scheduleApiPort; + @Value("${fr.insee.genesis.version}") + private String projectVersion; + + @Autowired + public HealthCheckController(SurveyUnitUpdateApiPort surveyUnitUpdateApiPort, ScheduleApiPort scheduleApiPort) { + this.surveyUnitUpdateApiPort = surveyUnitUpdateApiPort; + this.scheduleApiPort = scheduleApiPort; + } - @Autowired(required = false) - private Optional buildProperties; + @GetMapping("") + public ResponseEntity healthcheck() { + return ResponseEntity.ok( + """ + OK + + Version %s + User %s + """ + .formatted( + projectVersion, + SecurityContextHolder.getContext().getAuthentication().getName() + )); + } - @GetMapping("") - public ResponseEntity healthcheck(){ - return ResponseEntity.ok( - """ - OK - - Version %s - User %s - """ + @GetMapping("mongoDb") + public ResponseEntity healthcheckMongo() { + return ResponseEntity.ok( + """ + MongoDB OK + + %s Responses + %s Schedules + """ .formatted( - buildProperties.map(BuildProperties::getVersion).orElse("n.a"), - SecurityContextHolder.getContext().getAuthentication().getName() - ) ); - } + surveyUnitUpdateApiPort.countResponses(), + scheduleApiPort.countSchedules() + )); + } } diff --git a/src/main/java/fr/insee/genesis/controller/rest/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/ResponseController.java index 6f13a08d..343a7285 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/ResponseController.java @@ -11,7 +11,12 @@ import fr.insee.genesis.controller.sources.xml.LunaticXmlDataSequentialParser; import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.domain.dtos.*; +import fr.insee.genesis.domain.dtos.CollectedVariableDto; +import fr.insee.genesis.domain.dtos.Mode; +import fr.insee.genesis.domain.dtos.SurveyUnitDto; +import fr.insee.genesis.domain.dtos.SurveyUnitId; +import fr.insee.genesis.domain.dtos.SurveyUnitUpdateDto; +import fr.insee.genesis.domain.dtos.VariableDto; import fr.insee.genesis.domain.ports.api.SurveyUnitUpdateApiPort; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; @@ -23,8 +28,18 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; - +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLStreamException; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -55,9 +70,9 @@ public ResponseController(SurveyUnitUpdateApiPort surveyUnitService, SurveyUnitQ @Operation(summary = "Save one file of responses in Genesis Database with its path") @PutMapping(path = "/save/lunatic-xml/one-file") - public ResponseEntity saveResponsesFromXmlFile( @RequestParam("pathLunaticXml") String xmlFile, - @RequestParam("pathDDI") String ddiFile, - @RequestParam(value = "mode", required = true) Mode modeSpecified) + public ResponseEntity saveResponsesFromXmlFile(@RequestParam("pathLunaticXml") String xmlFile, + @RequestParam("pathDDI") String ddiFile, + @RequestParam(value = "mode") Mode modeSpecified) throws Exception { log.info(String.format("Try to read DDI file : %s", ddiFile)); @@ -71,55 +86,11 @@ public ResponseEntity saveResponsesFromXmlFile( @RequestParam("pathL log.info(String.format("Try to read Xml file : %s", xmlFile)); Path filepath = Paths.get(xmlFile); - LunaticXmlCampaign campaign; - - if(filepath.toFile().length()/1024/1024 <= Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL){ - // DOM method - LunaticXmlDataParser parser = new LunaticXmlDataParser(); - try { - campaign = parser.parseDataFile(filepath); - } catch (GenesisException e) { - return ResponseEntity.status(e.getStatus()).body(e.getMessage()); - } - - List suDtos = new ArrayList<>(); - for (LunaticXmlSurveyUnit su : campaign.getSurveyUnits()) { - suDtos.addAll(LunaticXmlAdapter.convert(su, variablesMap, campaign.getIdCampaign(),modeSpecified)); - } - surveyUnitQualityService.verifySurveyUnits(suDtos,variablesMap); - - log.info("Saving {} survey units updates", suDtos.size()); - surveyUnitService.saveSurveyUnits(suDtos); - log.info("Survey units updates saved"); - - log.info("File {} treated", xmlFile); - return new ResponseEntity<>("Test", HttpStatus.OK); - - }else{ - //Sequential method - log.warn("File size > " + Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL + "MB! Parsing XML file using sequential method..."); - try(final InputStream stream = new FileInputStream(filepath.toFile())){ - LunaticXmlDataSequentialParser parser = new LunaticXmlDataSequentialParser(filepath,stream); - int suCount = 0; - - campaign = parser.getCampaign(); - LunaticXmlSurveyUnit su = parser.readNextSurveyUnit(); - - while(su != null){ - List suDtos = new ArrayList<>(LunaticXmlAdapter.convert(su, variablesMap, campaign.getIdCampaign(),modeSpecified)); - - surveyUnitQualityService.verifySurveyUnits(suDtos,variablesMap); - surveyUnitService.saveSurveyUnits(suDtos); - suCount++; - - su = parser.readNextSurveyUnit(); - } - log.info("Saved {} survey units updates", suCount); - } - - log.info("File {} treated", xmlFile); - return new ResponseEntity<>("Test", HttpStatus.OK); + if (filepath.toFile().length() / 1024 / 1024 <= Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL) { + return treatXmlFileWithMemory(filepath, modeSpecified, variablesMap); + } else { + return treatXmlFileSequentially(filepath, modeSpecified, variablesMap); } } @@ -128,84 +99,50 @@ public ResponseEntity saveResponsesFromXmlFile( @RequestParam("pathL public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam("campaignName") String campaignName, @RequestParam(value = "mode", required = false) Mode modeSpecified) throws Exception { + List errors = new ArrayList<>(); + log.info("Try to import data for campaign : {}", campaignName); - List modesList; + try { - modesList = controllerUtils.getModesList(campaignName, modeSpecified); - } catch(GenesisException e) { + List modesList = controllerUtils.getModesList(campaignName, modeSpecified); + for (Mode currentMode : modesList) { + treatCampaignWithMode(campaignName, currentMode, errors); + } + } catch (GenesisException e) { return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } + return ResponseEntity.ok("Data saved"); + } + @Operation(summary = "Save all files in Genesis Database") + @PutMapping(path = "/save/lunatic-xml/all-campaigns") + public ResponseEntity saveResponsesFromAllCampaignFolders(){ List errors = new ArrayList<>(); + List campaignFolders = fileUtils.listAllSpecsFolders(); + + if (campaignFolders.isEmpty()) { + return ResponseEntity.ok("No campaign to save"); + } + + for (File campaignFolder: campaignFolders) { + String campaignName = campaignFolder.getName(); + log.info("Try to import data for campaign : {}", campaignName); - for(Mode currentMode : modesList) { - log.info("Try to import data for mode : {}", currentMode.getModeName()); - String dataFolder = fileUtils.getDataFolder(campaignName, currentMode.getFolder()); - List dataFiles = fileUtils.listFiles(dataFolder); - log.info("Numbers of files to load in folder {} : {}", dataFolder, dataFiles.size()); - if (dataFiles.isEmpty()) { - errors.add(new NoDataError("No data file found",Mode.getEnumFromModeName(currentMode.getModeName()))); - log.info("No data file found in folder " + dataFolder); - } - VariablesMap variablesMap; try { - Path ddiFilePath = fileUtils.findDDIFile(campaignName, currentMode.getModeName()); - variablesMap = DDIReader.getVariablesFromDDI(ddiFilePath.toFile().toURI().toURL()); - } catch (GenesisException e) { - return ResponseEntity.status(e.getStatus()).body(e.getMessage()); - } catch(Exception e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(e.getMessage()); - } - for (String fileName : dataFiles.stream().filter(s -> s.endsWith(".xml")).toList()) { - String filepathString = String.format("%s/%s", dataFolder, fileName); - Path filepath = Paths.get(filepathString); - log.info("Try to read Xml file : {}", fileName); - - LunaticXmlCampaign campaign; - - if(filepath.toFile().length()/1024/1024 <= Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL){ - //DOM method - LunaticXmlDataParser parser = new LunaticXmlDataParser(); - campaign = parser.parseDataFile(filepath); - - List suDtos = new ArrayList<>(); - for (LunaticXmlSurveyUnit su : campaign.getSurveyUnits()) { - suDtos.addAll(LunaticXmlAdapter.convert(su, variablesMap, campaign.getIdCampaign(),currentMode)); - } - surveyUnitQualityService.verifySurveyUnits(suDtos,variablesMap); - - log.info("Saving {} survey units updates", suDtos.size()); - surveyUnitService.saveSurveyUnits(suDtos); - log.info("Survey units updates saved"); - }else{ - //Sequential method - log.warn("File size > " + Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL + "MB! Parsing XML file using sequential method..."); - try(final InputStream stream = new FileInputStream(filepath.toFile())){ - LunaticXmlDataSequentialParser parser = new LunaticXmlDataSequentialParser(filepath,stream); - int suCount = 0; - - campaign = parser.getCampaign(); - LunaticXmlSurveyUnit su = parser.readNextSurveyUnit(); - - while(su != null){ - List suDtos = new ArrayList<>(LunaticXmlAdapter.convert(su, variablesMap, campaign.getIdCampaign(),currentMode)); - - surveyUnitQualityService.verifySurveyUnits(suDtos,variablesMap); - surveyUnitService.saveSurveyUnits(suDtos); - suCount++; - - su = parser.readNextSurveyUnit(); - } - - log.info("Saving {} survey units updates", suCount); - log.info("Survey units updates saved"); - } + List modesList = controllerUtils.getModesList(campaignName, null); //modeSpecified null = all modes + for (Mode currentMode : modesList) { + treatCampaignWithMode(campaignName, currentMode, errors); } - log.info("File {} saved", fileName); - fileUtils.moveDataFile(campaignName, currentMode.getFolder(), fileName); + } catch (Exception e) { + log.warn("Error for campaign " + campaignName + ": " + e.getMessage()); + errors.add(new GenesisError(e.getMessage())); } } - return new ResponseEntity<>("Data saved", HttpStatus.OK); + if (errors.isEmpty()) { + return ResponseEntity.ok("Data saved"); + } else { + return ResponseEntity.status(209).body("Data saved with " + errors.size() + " errors"); + } } @Operation(summary = "Delete all responses of one questionnaire") @@ -213,16 +150,16 @@ public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam(" public ResponseEntity deleteAllResponsesByQuestionnaire(@RequestParam("idQuestionnaire") String idQuestionnaire) { log.info("Try to delete all responses of questionnaire : " + idQuestionnaire); Long ndDocuments = surveyUnitService.deleteByIdQuestionnaire(idQuestionnaire); - log.info("{} responses deleted",ndDocuments); - return new ResponseEntity<>(String.format("%d responses deleted",ndDocuments), HttpStatus.OK); + log.info("{} responses deleted", ndDocuments); + return ResponseEntity.ok(String.format("%d responses deleted", ndDocuments)); } @Operation(summary = "Retrieve responses with IdUE and IdQuestionnaire from Genesis Database") @GetMapping(path = "/get-responses/by-ue-and-questionnaire") - public ResponseEntity> findResponsesByUEAndQuestionnaire( @RequestParam("idUE") String idUE, - @RequestParam("idQuestionnaire") String idQuestionnaire) { + public ResponseEntity> findResponsesByUEAndQuestionnaire(@RequestParam("idUE") String idUE, + @RequestParam("idQuestionnaire") String idQuestionnaire) { List responses = surveyUnitService.findByIdsUEAndQuestionnaire(idUE, idQuestionnaire); - return new ResponseEntity<>(responses, HttpStatus.OK); + return ResponseEntity.ok(responses); } @Operation(summary = "Retrieve all responses of one questionnaire") @@ -237,52 +174,52 @@ public ResponseEntity findAllResponsesByQuestionnaire(@RequestParam("idQue String filepathString = String.format("OUT/%s/OUT_ALL_%s.json", idQuestionnaire, LocalDateTime.now().toString().replace(":", "")); Path filepath = Path.of(fileUtils.getDataFolderSource(), filepathString); - try (Stream responsesStream = surveyUnitService.findByIdQuestionnaire(idQuestionnaire);) { + try (Stream responsesStream = surveyUnitService.findByIdQuestionnaire(idQuestionnaire)) { fileUtils.writeSuUpdatesInFile(filepath, responsesStream); } catch (IOException e) { log.error("Error while writing file", e); - return new ResponseEntity<>(filepath, HttpStatus.INTERNAL_SERVER_ERROR); + return ResponseEntity.internalServerError().body(filepath); } log.info("End of extraction, responses extracted : {}", idUEsResponses.size()); - return new ResponseEntity<>(filepath, HttpStatus.OK); + return ResponseEntity.ok(filepath); } @Operation(summary = "Retrieve responses latest state with IdUE and IdQuestionnaire") @GetMapping(path = "/get-responses/by-ue-and-questionnaire/latest") - public ResponseEntity> getLatestByUE ( @RequestParam("idUE") String idUE, - @RequestParam("idQuestionnaire") String idQuestionnaire) { + public ResponseEntity> getLatestByUE(@RequestParam("idUE") String idUE, + @RequestParam("idQuestionnaire") String idQuestionnaire) { List responses = surveyUnitService.findLatestByIdAndByMode(idUE, idQuestionnaire); - return new ResponseEntity<>(responses, HttpStatus.OK); + return ResponseEntity.ok(responses); } @Operation(summary = "Retrieve response latest state with IdUE and IdQuestionnaire in one object in the output") @GetMapping(path = "/get-simplified-response/by-ue-and-questionnaire/latest") - public ResponseEntity getLatestByUEOneObject ( @RequestParam("idUE") String idUE, - @RequestParam("idQuestionnaire") String idQuestionnaire, - @RequestParam("mode") Mode mode) { + public ResponseEntity getLatestByUEOneObject(@RequestParam("idUE") String idUE, + @RequestParam("idQuestionnaire") String idQuestionnaire, + @RequestParam("mode") Mode mode) { List responses = surveyUnitService.findLatestByIdAndByMode(idUE, idQuestionnaire); List outputVariables = new ArrayList<>(); List outputExternalVariables = new ArrayList<>(); - responses.stream().filter(rep->rep.getMode().equals(mode)).forEach(response -> { + responses.stream().filter(rep -> rep.getMode().equals(mode)).forEach(response -> { outputVariables.addAll(response.getCollectedVariables()); outputExternalVariables.addAll(response.getExternalVariables()); }); - return new ResponseEntity<>(SurveyUnitUpdateSimplified.builder() + return ResponseEntity.ok(SurveyUnitUpdateSimplified.builder() .idQuest(responses.getFirst().getIdQuest()) .idCampaign(responses.getFirst().getIdCampaign()) .idUE(responses.getFirst().getIdUE()) .variablesUpdate(outputVariables) .externalVariables(outputExternalVariables) - .build(), HttpStatus.OK); + .build()); } @Operation(summary = "Retrieve all responses for a questionnaire and a list of UE", - description = "Return the latest state for each variable for the given ids and a given questionnaire.
" + - "For a given id the endpoint returns a document by collection mode(if there is more than one).") + description = "Return the latest state for each variable for the given ids and a given questionnaire.
" + + "For a given id the endpoint returns a document by collection mode(if there is more than one).") @PostMapping(path = "/get-simplified-responses/by-ue-and-questionnaire/latest") - public ResponseEntity> getLatestForUEList( @RequestParam("idQuestionnaire") String idQuestionnaire, - @RequestBody List idUEs) { + public ResponseEntity> getLatestForUEList(@RequestParam("idQuestionnaire") String idQuestionnaire, + @RequestBody List idUEs) { List results = new ArrayList<>(); List modes = surveyUnitService.findModesByIdQuestionnaire(idQuestionnaire); idUEs.forEach(idUE -> { @@ -290,7 +227,7 @@ public ResponseEntity> getLatestForUEList( @Req modes.forEach(mode -> { List outputVariables = new ArrayList<>(); List outputExternalVariables = new ArrayList<>(); - responses.stream().filter(rep->rep.getMode().equals(mode)).forEach(response -> { + responses.stream().filter(rep -> rep.getMode().equals(mode)).forEach(response -> { outputVariables.addAll(response.getCollectedVariables()); outputExternalVariables.addAll(response.getExternalVariables()); }); @@ -306,20 +243,106 @@ public ResponseEntity> getLatestForUEList( @Req } }); }); - return new ResponseEntity<>(results, HttpStatus.OK); + return ResponseEntity.ok(results); } - @Operation (summary = "Retrieve all IdUEs for a given questionnaire") + @Operation(summary = "Retrieve all IdUEs for a given questionnaire") @GetMapping(path = "/get-idUEs/by-questionnaire") public ResponseEntity> getAllIdUEsByQuestionnaire(@RequestParam("idQuestionnaire") String idQuestionnaire) { List responses = surveyUnitService.findDistinctIdUEsByIdQuestionnaire(idQuestionnaire); - return new ResponseEntity<>(responses, HttpStatus.OK); + return ResponseEntity.ok(responses); } - @Operation (summary = "List sources used for a given questionnaire") + @Operation(summary = "List sources used for a given questionnaire") @GetMapping(path = "/get-modes/by-questionnaire") public ResponseEntity> getModesByQuestionnaire(@RequestParam("idQuestionnaire") String idQuestionnaire) { List modes = surveyUnitService.findModesByIdQuestionnaire(idQuestionnaire); - return new ResponseEntity<>(modes, HttpStatus.OK); + return ResponseEntity.ok(modes); + } + + //Utilities + + private void treatCampaignWithMode(String campaignName, Mode mode, List errors) throws IOException, ParserConfigurationException, SAXException, XMLStreamException { + log.info("Try to import data for mode : {}", mode.getModeName()); + String dataFolder = fileUtils.getDataFolder(campaignName, mode.getFolder()); + List dataFiles = fileUtils.listFiles(dataFolder); + log.info("Numbers of files to load in folder {} : {}", dataFolder, dataFiles.size()); + if (dataFiles.isEmpty()) { + errors.add(new NoDataError("No data file found", Mode.getEnumFromModeName(mode.getModeName()))); + log.info("No data file found in folder {}", dataFolder); + } + VariablesMap variablesMap; + try { + Path ddiFilePath = fileUtils.findDDIFile(campaignName, mode.getModeName()); + variablesMap = DDIReader.getVariablesFromDDI(ddiFilePath.toUri().toURL()); + } catch (Exception e) { + errors.add(new GenesisError(e.getMessage())); + return; + } + for (String fileName : dataFiles.stream().filter(s -> s.endsWith(".xml")).toList()) { + String filepathString = String.format("%s/%s", dataFolder, fileName); + Path filepath = Paths.get(filepathString); + log.info("Try to read Xml file : {}", fileName); + + if (filepath.toFile().length() / 1024 / 1024 <= Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL) { + treatXmlFileWithMemory(filepath, mode, variablesMap); + } else { + treatXmlFileSequentially(filepath, mode, variablesMap); + } + log.debug("File {} saved", fileName); + fileUtils.moveDataFile(campaignName, mode.getFolder(), fileName); + } + } + + private ResponseEntity treatXmlFileWithMemory(Path filepath, Mode modeSpecified, VariablesMap variablesMap) throws IOException, ParserConfigurationException, SAXException { + LunaticXmlCampaign campaign; + // DOM method + LunaticXmlDataParser parser = new LunaticXmlDataParser(); + try { + campaign = parser.parseDataFile(filepath); + } catch (GenesisException e) { + return ResponseEntity.status(e.getStatus()).body(e.getMessage()); + } + + List suDtos = new ArrayList<>(); + for (LunaticXmlSurveyUnit su : campaign.getSurveyUnits()) { + suDtos.addAll(LunaticXmlAdapter.convert(su, variablesMap, campaign.getIdCampaign(), modeSpecified)); + } + surveyUnitQualityService.verifySurveyUnits(suDtos, variablesMap); + + log.debug("Saving {} survey units updates", suDtos.size()); + surveyUnitService.saveSurveyUnits(suDtos); + log.debug("Survey units updates saved"); + + log.info("File {} treated with {} survey units", filepath.getFileName(), suDtos.size()); + return ResponseEntity.ok().build(); + } + + private ResponseEntity treatXmlFileSequentially(Path filepath, Mode modeSpecified, VariablesMap variablesMap) throws IOException, XMLStreamException { + LunaticXmlCampaign campaign; + //Sequential method + log.warn("File size > " + Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL + "MB! Parsing XML file using sequential method..."); + try (final InputStream stream = new FileInputStream(filepath.toFile())) { + LunaticXmlDataSequentialParser parser = new LunaticXmlDataSequentialParser(filepath, stream); + int suCount = 0; + + campaign = parser.getCampaign(); + LunaticXmlSurveyUnit su = parser.readNextSurveyUnit(); + + while (su != null) { + List suDtos = new ArrayList<>(LunaticXmlAdapter.convert(su, variablesMap, campaign.getIdCampaign(), modeSpecified)); + + surveyUnitQualityService.verifySurveyUnits(suDtos, variablesMap); + surveyUnitService.saveSurveyUnits(suDtos); + suCount++; + + su = parser.readNextSurveyUnit(); + } + + log.info("Saved {} survey units updates", suCount); + } + + log.info("File {} treated", filepath.getFileName()); + return ResponseEntity.ok().build(); } } diff --git a/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java b/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java index aad349f5..5d456ba4 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java @@ -10,7 +10,6 @@ import io.swagger.v3.oas.annotations.Parameter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; @@ -45,8 +44,7 @@ public ResponseEntity getAllSchedules() { List storedSurveySchedules = scheduleApiPort.getAllSchedules(); log.info("Returning " + storedSurveySchedules.size() + " schedule documents..."); - - return new ResponseEntity<>(storedSurveySchedules, HttpStatus.OK); + return ResponseEntity.ok(storedSurveySchedules); } @Operation(summary = "Schedule a Kraftwerk execution") @@ -63,10 +61,10 @@ public ResponseEntity addSchedule( scheduleApiPort.addSchedule(surveyName, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate); }catch (InvalidCronExpressionException e){ log.warn("Returned error for wrong frequency : " + frequency); - return new ResponseEntity<>("Wrong frequency syntax",HttpStatus.BAD_REQUEST); + return ResponseEntity.badRequest().body("Wrong frequency syntax"); } log.info("New schedule created for survey " + surveyName); - return new ResponseEntity<>(HttpStatus.OK); + return ResponseEntity.ok().build(); } @Operation(summary = "Update last execution date (for Bangles only)", hidden = true) @@ -80,8 +78,8 @@ public ResponseEntity updateSurveyLastExecution( log.info(surveyName + " last execution updated !"); }catch (NotFoundException e){ log.warn("Survey " + surveyName + " not found !"); - return new ResponseEntity<>(HttpStatus.NOT_FOUND); + return ResponseEntity.notFound().build(); } - return new ResponseEntity<>(HttpStatus.OK); + return ResponseEntity.ok().build(); } } diff --git a/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java b/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java index 6ea9235c..42055cec 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/UtilsController.java @@ -3,7 +3,6 @@ import fr.insee.genesis.controller.utils.XMLSplitter; import io.swagger.v3.oas.annotations.Operation; import lombok.extern.slf4j.Slf4j; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PutMapping; @@ -24,6 +23,6 @@ public ResponseEntity saveResponsesFromXmlFile(@RequestParam("inputFolde throws Exception { log.info("Split XML file : " + filename + " into " + nbSU + " SU by file"); XMLSplitter.split(inputFolder, filename, outputFolder, "SurveyUnit", nbSU); - return new ResponseEntity<>("Test", HttpStatus.OK); + return ResponseEntity.ok("Test"); } } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ScheduleApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ScheduleApiPort.java index 1a4d7d7a..e9010279 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/ScheduleApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/ScheduleApiPort.java @@ -19,4 +19,6 @@ void addSchedule(String surveyName, LocalDateTime scheduleEndDateString) throws InvalidCronExpressionException; void updateLastExecutionName(String surveyName) throws NotFoundException; + + long countSchedules(); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitUpdateApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitUpdateApiPort.java index 41b31353..fdf0779b 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitUpdateApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitUpdateApiPort.java @@ -30,4 +30,6 @@ public interface SurveyUnitUpdateApiPort { List findModesByIdQuestionnaire(String idQuestionnaire); Long deleteByIdQuestionnaire(String idQuestionnaire); + + long countResponses(); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitUpdatePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitUpdatePersistencePort.java index 05a57f43..d9b4ab35 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitUpdatePersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitUpdatePersistencePort.java @@ -22,5 +22,5 @@ public interface SurveyUnitUpdatePersistencePort { Long deleteByIdQuestionnaire(String idQuestionnaire); - + long count(); } diff --git a/src/main/java/fr/insee/genesis/domain/service/ScheduleImpl.java b/src/main/java/fr/insee/genesis/domain/service/ScheduleImpl.java index da3e9949..3a3f5f97 100644 --- a/src/main/java/fr/insee/genesis/domain/service/ScheduleImpl.java +++ b/src/main/java/fr/insee/genesis/domain/service/ScheduleImpl.java @@ -75,5 +75,12 @@ public void updateLastExecutionName(String surveyName) throws NotFoundException throw new NotFoundException(); } + + + } + + @Override + public long countSchedules() { + return scheduleMongoDBRepository.count(); } } diff --git a/src/main/java/fr/insee/genesis/domain/service/SurveyUnitUpdateImpl.java b/src/main/java/fr/insee/genesis/domain/service/SurveyUnitUpdateImpl.java index ab891db7..51373042 100644 --- a/src/main/java/fr/insee/genesis/domain/service/SurveyUnitUpdateImpl.java +++ b/src/main/java/fr/insee/genesis/domain/service/SurveyUnitUpdateImpl.java @@ -125,6 +125,11 @@ public Long deleteByIdQuestionnaire(String idQuestionnaire) { return surveyUnitUpdatePersistencePort.deleteByIdQuestionnaire(idQuestionnaire); } + @Override + public long countResponses() { + return surveyUnitUpdatePersistencePort.count(); + } + private static List getDistinctsModes(List surveyUnits) { List sources = new ArrayList<>(); surveyUnits.forEach(surveyUnitDto -> sources.add(surveyUnitDto.getMode())); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitUpdateMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitUpdateMongoAdapter.java index d85d8d27..cafb92e7 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitUpdateMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitUpdateMongoAdapter.java @@ -66,6 +66,11 @@ public Long deleteByIdQuestionnaire(String idQuestionnaire) { return mongoRepository.deleteByIdQuestionnaire(idQuestionnaire); } + @Override + public long count() { + return mongoRepository.count(); + } + public List findIdUEsByIdQuestionnaire(String idQuestionnaire) { List surveyUnits = mongoRepository.findIdUEsByIdQuestionnaire(idQuestionnaire); return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListDto(surveyUnits); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/ScheduleMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/ScheduleMongoDBRepository.java index 241da743..5ba427ff 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/ScheduleMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/ScheduleMongoDBRepository.java @@ -16,4 +16,6 @@ public interface ScheduleMongoDBRepository extends MongoRepository findByIdQuestionnaire(String idQuestionnaire); + long count(); } \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java b/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java index f52b0a86..1a64798e 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java +++ b/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java @@ -11,6 +11,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.stream.Stream; @@ -139,7 +140,15 @@ public String getDataFolder(String campaign, String dataSource) { /** * Get the path of the folder where the specifications files are stored - * @param campaign + * @return Path of the specifications folder + */ + public String getSpecFolder() { + return String.format("%s/%s", specFolderSource, "specs"); + } + + /** + * Get the path of the folder where the specifications files are stored for specific campaign + * @param campaign name of campaign * @return Path of the specifications folder */ public String getSpecFolder(String campaign) { @@ -206,4 +215,18 @@ public void writeSuUpdatesInFile(Path filePath, Stream resp throw e.getCause(); } } + + /** + * List all folders in the specs folder + * @return List of specs folders + */ + public List listAllSpecsFolders() { + File[] objs = new File(getSpecFolder()).listFiles(); + if (objs == null) { + return List.of(); + } + return Arrays.stream(objs) + .filter(File::isDirectory) + .toList(); + } } diff --git a/src/test/java/cucumber/TestConstants.java b/src/test/java/cucumber/TestConstants.java index eca092f5..7b49456a 100644 --- a/src/test/java/cucumber/TestConstants.java +++ b/src/test/java/cucumber/TestConstants.java @@ -10,7 +10,7 @@ public class TestConstants { //Functional tests public static final String FUNCTIONAL_TESTS_INPUT_DIRECTORY = TEST_RESOURCES_DIRECTORY + "/IN"; public static final String FUNCTIONAL_TESTS_WEB_DIRECTORY = FUNCTIONAL_TESTS_INPUT_DIRECTORY + "/WEB"; - public static final String FUNCTIONAL_TESTS_DDI_DIRECTORY = FUNCTIONAL_TESTS_INPUT_DIRECTORY + "/specs"; + public static final String FUNCTIONAL_TESTS_DDI_DIRECTORY = TEST_RESOURCES_DIRECTORY + "/specs"; public static final String FUNCTIONAL_TESTS_TEMP_DIRECTORY = TEST_RESOURCES_DIRECTORY + "/functional_tests/temp"; public static final String FUNCTIONAL_TESTS_OUTPUT_DIRECTORY = TEST_RESOURCES_DIRECTORY + "/functional_tests/out"; public static final String FUNCTIONAL_TESTS_API_URL = "http://localhost:8080"; diff --git a/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java new file mode 100644 index 00000000..3de53531 --- /dev/null +++ b/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java @@ -0,0 +1,67 @@ +package fr.insee.genesis.controller.rest; + +import fr.insee.genesis.domain.dtos.CollectedVariableDto; +import fr.insee.genesis.domain.dtos.DataState; +import fr.insee.genesis.domain.dtos.Mode; +import fr.insee.genesis.domain.dtos.SurveyUnitUpdateDto; +import fr.insee.genesis.domain.dtos.VariableDto; +import fr.insee.genesis.domain.ports.api.SurveyUnitUpdateApiPort; +import fr.insee.genesis.domain.service.SurveyUnitUpdateImpl; +import fr.insee.genesis.stubs.ScheduleApiPortStub; +import fr.insee.genesis.stubs.SurveyUnitUpdatePersistencePortStub; +import lombok.extern.slf4j.Slf4j; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.http.ResponseEntity; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +class HealthCheckControllerTest { + static HealthCheckController healthCheckController; + + static SurveyUnitUpdatePersistencePortStub surveyUnitUpdatePersistencePortStub; + static ScheduleApiPortStub scheduleApiPortStub; + + @BeforeAll + static void init() { + surveyUnitUpdatePersistencePortStub = new SurveyUnitUpdatePersistencePortStub(); + SurveyUnitUpdateApiPort surveyUnitUpdateApiPort = new SurveyUnitUpdateImpl(surveyUnitUpdatePersistencePortStub); + List externalVariableDtoList = new ArrayList<>(); + VariableDto variableDto = VariableDto.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); + externalVariableDtoList.add(variableDto); + + List collectedVariableDtoList = new ArrayList<>(); + CollectedVariableDto collectedVariableDto = new CollectedVariableDto("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); + collectedVariableDtoList.add(collectedVariableDto); + surveyUnitUpdatePersistencePortStub.getMongoStub().add(SurveyUnitUpdateDto.builder() + .idCampaign("TESTIDCAMPAIGN") + .mode(Mode.WEB) + .idUE("TESTIDUE") + .idQuest("TESTIDQUESTIONNAIRE") + .state(DataState.COLLECTED) + .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0, 0)) + .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) + .externalVariables(externalVariableDtoList) + .collectedVariables(collectedVariableDtoList) + .build()); + + + scheduleApiPortStub = new ScheduleApiPortStub(); + + healthCheckController = new HealthCheckController( + surveyUnitUpdateApiPort, + scheduleApiPortStub + ); + } + + @Test + void mongoCountTest() { + ResponseEntity response = healthCheckController.healthcheckMongo(); + log.info(response.getBody()); + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + } +} diff --git a/src/test/java/fr/insee/genesis/controller/rest/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/ResponseControllerTest.java index d1f85151..9d98a8a4 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ResponseControllerTest.java @@ -5,7 +5,12 @@ import fr.insee.genesis.controller.responses.SurveyUnitUpdateSimplified; import fr.insee.genesis.controller.service.SurveyUnitQualityService; import fr.insee.genesis.controller.utils.ControllerUtils; -import fr.insee.genesis.domain.dtos.*; +import fr.insee.genesis.domain.dtos.CollectedVariableDto; +import fr.insee.genesis.domain.dtos.DataState; +import fr.insee.genesis.domain.dtos.Mode; +import fr.insee.genesis.domain.dtos.SurveyUnitId; +import fr.insee.genesis.domain.dtos.SurveyUnitUpdateDto; +import fr.insee.genesis.domain.dtos.VariableDto; import fr.insee.genesis.domain.ports.api.SurveyUnitUpdateApiPort; import fr.insee.genesis.domain.service.SurveyUnitUpdateImpl; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -36,16 +41,16 @@ class ResponseControllerTest { static List surveyUnitIdList; @BeforeAll - static void init(){ + static void init() { surveyUnitUpdatePersistencePortStub = new SurveyUnitUpdatePersistencePortStub(); SurveyUnitUpdateApiPort surveyUnitUpdateApiPort = new SurveyUnitUpdateImpl(surveyUnitUpdatePersistencePortStub); FileUtils fileUtils = new FileUtils(new ConfigStub()); responseControllerStatic = new ResponseController( surveyUnitUpdateApiPort - ,new SurveyUnitQualityService() - ,fileUtils - ,new ControllerUtils(fileUtils) + , new SurveyUnitQualityService() + , fileUtils + , new ControllerUtils(fileUtils) ); surveyUnitIdList = new ArrayList<>(); @@ -62,7 +67,7 @@ void reset() throws IOException { externalVariableDtoList.add(variableDto); List collectedVariableDtoList = new ArrayList<>(); - CollectedVariableDto collectedVariableDto = new CollectedVariableDto("TESTIDVAR", List.of(new String[]{"V1", "V2"}),"TESTIDLOOP","TESTIDPARENT"); + CollectedVariableDto collectedVariableDto = new CollectedVariableDto("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); collectedVariableDtoList.add(collectedVariableDto); surveyUnitUpdatePersistencePortStub.getMongoStub().add(SurveyUnitUpdateDto.builder() .idCampaign("TESTIDCAMPAIGN") @@ -70,49 +75,101 @@ void reset() throws IOException { .idUE("TESTIDUE") .idQuest("TESTIDQUESTIONNAIRE") .state(DataState.COLLECTED) - .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) - .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) + .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0, 0)) + .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) .externalVariables(externalVariableDtoList) .collectedVariables(collectedVariableDtoList) .build()); - //Test file management - if(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY).resolve("DONE").toFile().exists()) + //Clean DONE folder + if (Path.of(TestConstants.TEST_RESOURCES_DIRECTORY).resolve("DONE").toFile().exists()) Files.walk(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY).resolve("DONE")) .sorted(Comparator.reverseOrder()) .map(Path::toFile) .forEach(File::delete); - if(!Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + //Recreate data files + //SAMPLETEST-PARADATA-v1 + if (!Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) .resolve("IN") .resolve("WEB") .resolve("SAMPLETEST-PARADATA-v1") .resolve("data.complete.validated.STPDv1.20231122164209.xml") .toFile().exists() - ) + ){ + Files.copy( + Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("reponse-platine") + .resolve("data.complete.partial.STPDv1.20231122164209.xml") + , Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("data.complete.partial.STPDv1.20231122164209.xml") + ); + Files.copy( + Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("reponse-platine") + .resolve("data.complete.validated.STPDv1.20231122164209.xml") + , Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v1") + .resolve("data.complete.validated.STPDv1.20231122164209.xml") + ); + } + //SAMPLETEST-PARADATA-v2 + if (!Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("data.complete.validated.STPDv2.20231122164209.xml") + .toFile().exists() + ){ Files.copy( - Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("reponse-platine") - .resolve("data.complete.validated.STPDv1.20231122164209.xml") - ,Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) - .resolve("IN") - .resolve("WEB") - .resolve("SAMPLETEST-PARADATA-v1") - .resolve("data.complete.validated.STPDv1.20231122164209.xml") + Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("reponse-platine") + .resolve("data.complete.partial.STPDv2.20231122164209.xml") + , Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("data.complete.partial.STPDv2.20231122164209.xml") ); + Files.copy( + Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("reponse-platine") + .resolve("data.complete.validated.STPDv2.20231122164209.xml") + , Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve("IN") + .resolve("WEB") + .resolve("SAMPLETEST-PARADATA-v2") + .resolve("data.complete.validated.STPDv2.20231122164209.xml") + ); + } } + //When + Then @Test void saveResponseFromXMLFileTest() throws Exception { responseControllerStatic.saveResponsesFromXmlFile( - Path.of(TestConstants.TEST_RESOURCES_DIRECTORY,"IN/WEB/SAMPLETEST-PARADATA-v1/reponse-platine/data.complete.validated.STPDv1.20231122164209.xml").toString() - ,Path.of(TestConstants.TEST_RESOURCES_DIRECTORY,"IN/specs/SAMPLETEST-PARADATA-v1/ddi-SAMPLETEST-PARADATA-v1.xml").toString() - ,Mode.WEB + Path.of(TestConstants.TEST_RESOURCES_DIRECTORY, "IN/WEB/SAMPLETEST-PARADATA-v1/reponse-platine/data.complete.validated.STPDv1.20231122164209.xml").toString() + , Path.of(TestConstants.TEST_RESOURCES_DIRECTORY, "specs/SAMPLETEST-PARADATA-v1/ddi-SAMPLETEST-PARADATA-v1.xml").toString() + , Mode.WEB ); Assertions.assertThat(surveyUnitUpdatePersistencePortStub.getMongoStub()).isNotEmpty(); @@ -122,7 +179,7 @@ void saveResponseFromXMLFileTest() throws Exception { void saveResponsesFromXmlCampaignFolderTest() throws Exception { responseControllerStatic.saveResponsesFromXmlCampaignFolder( "SAMPLETEST-PARADATA-v1" - ,Mode.WEB + , Mode.WEB ); Assertions.assertThat(surveyUnitUpdatePersistencePortStub.getMongoStub()).isNotEmpty(); @@ -134,15 +191,23 @@ void saveResponsesFromXmlCampaignFolderTest_noData() throws Exception { responseControllerStatic.saveResponsesFromXmlCampaignFolder( "TESTNODATA" - ,Mode.WEB + , Mode.WEB ); Assertions.assertThat(surveyUnitUpdatePersistencePortStub.getMongoStub()).isEmpty(); } @Test - void findResponsesByUEAndQuestionnaireTest(){ - ResponseEntity> response = responseControllerStatic.findResponsesByUEAndQuestionnaire("TESTIDUE","TESTIDQUESTIONNAIRE"); + void saveResponsesFromAllCampaignFoldersTests(){ + surveyUnitUpdatePersistencePortStub.getMongoStub().clear(); + responseControllerStatic.saveResponsesFromAllCampaignFolders(); + + Assertions.assertThat(surveyUnitUpdatePersistencePortStub.getMongoStub()).isNotEmpty(); + } + + @Test + void findResponsesByUEAndQuestionnaireTest() { + ResponseEntity> response = responseControllerStatic.findResponsesByUEAndQuestionnaire("TESTIDUE", "TESTIDQUESTIONNAIRE"); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); @@ -151,8 +216,8 @@ void findResponsesByUEAndQuestionnaireTest(){ } @Test - void findAllResponsesByQuestionnaireTest(){ - Path path = Path.of(TestConstants.TEST_RESOURCES_DIRECTORY,"OUT", "TESTIDQUESTIONNAIRE"); + void findAllResponsesByQuestionnaireTest() { + Path path = Path.of(TestConstants.TEST_RESOURCES_DIRECTORY, "OUT", "TESTIDQUESTIONNAIRE"); File dir = new File(String.valueOf(path)); FileSystemUtils.deleteRecursively(dir); @@ -173,13 +238,13 @@ void getAllResponsesByQuestionnaireTestSequential() throws IOException { //Given surveyUnitUpdatePersistencePortStub.getMongoStub().clear(); - for(int i = 0; i < Constants.BATCH_SIZE + 2; i++){ + for (int i = 0; i < Constants.BATCH_SIZE + 2; i++) { List externalVariableDtoList = new ArrayList<>(); VariableDto variableDto = VariableDto.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); externalVariableDtoList.add(variableDto); List collectedVariableDtoList = new ArrayList<>(); - CollectedVariableDto collectedVariableDto = new CollectedVariableDto("TESTIDVAR", List.of(new String[]{"V1", "V2"}),"TESTIDLOOP","TESTIDPARENT"); + CollectedVariableDto collectedVariableDto = new CollectedVariableDto("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); collectedVariableDtoList.add(collectedVariableDto); surveyUnitUpdatePersistencePortStub.getMongoStub().add(SurveyUnitUpdateDto.builder() @@ -188,8 +253,8 @@ void getAllResponsesByQuestionnaireTestSequential() throws IOException { .idUE("TESTIDUE" + i) .idQuest("TESTIDQUESTIONNAIRE") .state(DataState.COLLECTED) - .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) - .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) + .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0, 0)) + .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) .externalVariables(externalVariableDtoList) .collectedVariables(collectedVariableDtoList) .build()); @@ -210,10 +275,10 @@ void getAllResponsesByQuestionnaireTestSequential() throws IOException { } @Test - void getLatestByUETest(){ + void getLatestByUETest() { addAdditionnalDtoToMongoStub(); - ResponseEntity> response = responseControllerStatic.getLatestByUE("TESTIDUE","TESTIDQUESTIONNAIRE"); + ResponseEntity> response = responseControllerStatic.getLatestByUE("TESTIDUE", "TESTIDQUESTIONNAIRE"); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); @@ -223,8 +288,8 @@ void getLatestByUETest(){ } @Test - void getLatestByUEOneObjectTest(){ - ResponseEntity response = responseControllerStatic.getLatestByUEOneObject("TESTIDUE","TESTIDQUESTIONNAIRE", Mode.WEB); + void getLatestByUEOneObjectTest() { + ResponseEntity response = responseControllerStatic.getLatestByUEOneObject("TESTIDUE", "TESTIDQUESTIONNAIRE", Mode.WEB); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull(); @@ -233,8 +298,8 @@ void getLatestByUEOneObjectTest(){ } @Test - void getLatestForUEListTest(){ - ResponseEntity> response = responseControllerStatic.getLatestForUEList("TESTIDQUESTIONNAIRE",surveyUnitIdList); + void getLatestForUEListTest() { + ResponseEntity> response = responseControllerStatic.getLatestForUEList("TESTIDQUESTIONNAIRE", surveyUnitIdList); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); @@ -242,7 +307,7 @@ void getLatestForUEListTest(){ } @Test - void getAllIdUEsByQuestionnaireTest(){ + void getAllIdUEsByQuestionnaireTest() { ResponseEntity> response = responseControllerStatic.getAllIdUEsByQuestionnaire("TESTIDQUESTIONNAIRE"); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); @@ -251,7 +316,7 @@ void getAllIdUEsByQuestionnaireTest(){ } @Test - void getModesByQuestionnaireTest(){ + void getModesByQuestionnaireTest() { ResponseEntity> response = responseControllerStatic.getModesByQuestionnaire("TESTIDQUESTIONNAIRE"); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); @@ -259,13 +324,14 @@ void getModesByQuestionnaireTest(){ Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB); } - private void addAdditionnalDtoToMongoStub(){ + // Utilities + private void addAdditionnalDtoToMongoStub() { List externalVariableDtoList = new ArrayList<>(); VariableDto variableDto = VariableDto.builder().idVar("TESTIDVAR").values(List.of(new String[]{"V1", "V2"})).build(); externalVariableDtoList.add(variableDto); List collectedVariableDtoList = new ArrayList<>(); - CollectedVariableDto collectedVariableDto = new CollectedVariableDto("TESTIDVAR", List.of(new String[]{"V1", "V2"}),"TESTIDLOOP","TESTIDPARENT"); + CollectedVariableDto collectedVariableDto = new CollectedVariableDto("TESTIDVAR", List.of(new String[]{"V1", "V2"}), "TESTIDLOOP", "TESTIDPARENT"); collectedVariableDtoList.add(collectedVariableDto); SurveyUnitUpdateDto recentDTO = SurveyUnitUpdateDto.builder() @@ -274,8 +340,8 @@ private void addAdditionnalDtoToMongoStub(){ .idUE("TESTIDUE") .idQuest("TESTIDQUESTIONNAIRE") .state(DataState.COLLECTED) - .fileDate(LocalDateTime.of(2023,2,2,0,0,0)) - .recordDate(LocalDateTime.of(2024,2,2,0,0,0)) + .fileDate(LocalDateTime.of(2023, 2, 2, 0, 0, 0)) + .recordDate(LocalDateTime.of(2024, 2, 2, 0, 0, 0)) .externalVariables(externalVariableDtoList) .collectedVariables(collectedVariableDtoList) .build(); diff --git a/src/test/java/fr/insee/genesis/infrastructure/utils/FileUtilsTest.java b/src/test/java/fr/insee/genesis/infrastructure/utils/FileUtilsTest.java index b443ad28..ae43e05d 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/utils/FileUtilsTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/utils/FileUtilsTest.java @@ -33,7 +33,9 @@ void getDoneFolderTest() { @Test void getSpecFolderTest(){ - assertEquals(TestConstants.TEST_RESOURCES_DIRECTORY + "/IN/specs/TEST", + assertEquals(TestConstants.TEST_RESOURCES_DIRECTORY + "/specs", + fileUtils.getSpecFolder()); + assertEquals(TestConstants.TEST_RESOURCES_DIRECTORY + "/specs/TEST", fileUtils.getSpecFolder("TEST")); } diff --git a/src/test/java/fr/insee/genesis/stubs/ConfigStub.java b/src/test/java/fr/insee/genesis/stubs/ConfigStub.java index 416add5d..c934d35a 100644 --- a/src/test/java/fr/insee/genesis/stubs/ConfigStub.java +++ b/src/test/java/fr/insee/genesis/stubs/ConfigStub.java @@ -7,7 +7,7 @@ public class ConfigStub extends Config { private static final String DATA_SOURCE = TestConstants.TEST_RESOURCES_DIRECTORY; - private static final String SPEC_SOURCE = TestConstants.TEST_RESOURCES_DIRECTORY + "/IN"; + private static final String SPEC_SOURCE = TestConstants.TEST_RESOURCES_DIRECTORY; diff --git a/src/test/java/fr/insee/genesis/stubs/ScheduleApiPortStub.java b/src/test/java/fr/insee/genesis/stubs/ScheduleApiPortStub.java index 1ec38652..8ac99069 100644 --- a/src/test/java/fr/insee/genesis/stubs/ScheduleApiPortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/ScheduleApiPortStub.java @@ -95,4 +95,9 @@ public void updateLastExecutionName(String surveyName) throws NotFoundException storedSurveySchedule.setLastExecution(LocalDateTime.now()); }else throw new NotFoundException(); } + + @Override + public long countSchedules() { + return mongoStub.size(); + } } diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitUpdatePersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitUpdatePersistencePortStub.java index 7fe22a06..dd9cd57f 100644 --- a/src/test/java/fr/insee/genesis/stubs/SurveyUnitUpdatePersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitUpdatePersistencePortStub.java @@ -81,4 +81,11 @@ public List findIdUEsByIdQuestionnaire(String idQuestionnaire) { public Long deleteByIdQuestionnaire(String idQuestionnaire) { return null; } + + @Override + public long count() { + return mongoStub.size(); + } + + } diff --git a/src/test/resources/IN/WEB/SAMPLETEST-PARADATA-v1/data.complete.partial.STPDv1.20231122164209.xml b/src/test/resources/IN/WEB/SAMPLETEST-PARADATA-v1/data.complete.partial.STPDv1.20231122164209.xml new file mode 100644 index 00000000..3b05ae3d --- /dev/null +++ b/src/test/resources/IN/WEB/SAMPLETEST-PARADATA-v1/data.complete.partial.STPDv1.20231122164209.xml @@ -0,0 +1,6 @@ + + + SAMPLETEST-PARADATA-v1 + diff --git a/src/test/resources/IN/specs/SAMPLETEST-PARADATA-v1/WEB/ddi-SAMPLETEST-PARADATA-v1.xml b/src/test/resources/specs/SAMPLETEST-PARADATA-v1/WEB/ddi-SAMPLETEST-PARADATA-v1.xml similarity index 100% rename from src/test/resources/IN/specs/SAMPLETEST-PARADATA-v1/WEB/ddi-SAMPLETEST-PARADATA-v1.xml rename to src/test/resources/specs/SAMPLETEST-PARADATA-v1/WEB/ddi-SAMPLETEST-PARADATA-v1.xml diff --git a/src/test/resources/IN/specs/SAMPLETEST-PARADATA-v1/ddi-SAMPLETEST-PARADATA-v1.xml b/src/test/resources/specs/SAMPLETEST-PARADATA-v1/ddi-SAMPLETEST-PARADATA-v1.xml similarity index 100% rename from src/test/resources/IN/specs/SAMPLETEST-PARADATA-v1/ddi-SAMPLETEST-PARADATA-v1.xml rename to src/test/resources/specs/SAMPLETEST-PARADATA-v1/ddi-SAMPLETEST-PARADATA-v1.xml diff --git a/src/test/resources/IN/specs/SAMPLETEST-PARADATA-v2/WEB/ddi-SAMPLETEST-PARADATA-v2.xml b/src/test/resources/specs/SAMPLETEST-PARADATA-v2/WEB/ddi-SAMPLETEST-PARADATA-v2.xml similarity index 100% rename from src/test/resources/IN/specs/SAMPLETEST-PARADATA-v2/WEB/ddi-SAMPLETEST-PARADATA-v2.xml rename to src/test/resources/specs/SAMPLETEST-PARADATA-v2/WEB/ddi-SAMPLETEST-PARADATA-v2.xml diff --git a/src/test/resources/IN/specs/SAMPLETEST-PARADATA-v2/ddi-SAMPLETEST-PARADATA-v2.xml b/src/test/resources/specs/SAMPLETEST-PARADATA-v2/ddi-SAMPLETEST-PARADATA-v2.xml similarity index 100% rename from src/test/resources/IN/specs/SAMPLETEST-PARADATA-v2/ddi-SAMPLETEST-PARADATA-v2.xml rename to src/test/resources/specs/SAMPLETEST-PARADATA-v2/ddi-SAMPLETEST-PARADATA-v2.xml