Add reading without DDI (lunatic only)

src/main/java/fr/insee/genesis/Constants.java

@@ -11,6 +11,7 @@ public class Constants {
     public static final String FILTER_RESULT_PREFIX = "FILTER_RESULT_";
     public static final String MISSING_SUFFIX = "_MISSING";
     private static final String[] ENO_VARIABLES = {"COMMENT_QE","COMMENT_UE","HEURE_REMPL","MIN_REMPL"};
+    public static final String LOOP_NAME_PREFIX = "BOUCLE";
     public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
     public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries";
     public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY";

src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java

@@ -1,6 +1,6 @@
 package fr.insee.genesis.controller.adapter;
 
-import fr.insee.genesis.controller.sources.ddi.VariablesMap;
+import fr.insee.genesis.controller.sources.metadata.VariablesMap;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlCollectedData;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit;
 import fr.insee.genesis.controller.sources.xml.ValueType;

src/main/java/fr/insee/genesis/controller/rest/ResponseController.java

@@ -4,9 +4,10 @@
 import fr.insee.genesis.controller.adapter.LunaticXmlAdapter;
 import fr.insee.genesis.controller.responses.SurveyUnitUpdateSimplified;
 import fr.insee.genesis.controller.service.SurveyUnitQualityService;
+import fr.insee.genesis.controller.sources.metadata.ddi.DDIReader;
+import fr.insee.genesis.controller.sources.metadata.VariablesMap;
+import fr.insee.genesis.controller.sources.metadata.lunatic.LunaticReader;
 import fr.insee.genesis.controller.service.VolumetryLogService;
-import fr.insee.genesis.controller.sources.ddi.DDIReader;
-import fr.insee.genesis.controller.sources.ddi.VariablesMap;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlDataParser;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlDataSequentialParser;
@@ -29,7 +30,6 @@
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
-import org.springframework.http.HttpStatusCode;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.DeleteMapping;
@@ -76,20 +76,34 @@ public ResponseController(SurveyUnitUpdateApiPort surveyUnitService, SurveyUnitQ
         this.controllerUtils = controllerUtils;
     }
 
+    //SAVE
     @Operation(summary = "Save one file of responses in Genesis Database with its path")
     @PutMapping(path = "/save/lunatic-xml/one-file")
     public ResponseEntity<Object> saveResponsesFromXmlFile(@RequestParam("pathLunaticXml") String xmlFile,
-                                                           @RequestParam("pathDDI") String ddiFile,
-                                                           @RequestParam(value = "mode") Mode modeSpecified)
-            throws Exception {
-
-        log.info(String.format("Try to read DDI file : %s", ddiFile));
-        Path ddiFilePath = Paths.get(ddiFile);
+                                                           @RequestParam(value = "pathSpecFile") String metadataFilePath,
+                                                           @RequestParam(value = "mode") Mode modeSpecified,
+                                                           @RequestParam(value = "withDDI", defaultValue = "true") boolean withDDI
+    )throws Exception {
         VariablesMap variablesMap;
-        try {
-            variablesMap = DDIReader.getVariablesFromDDI(ddiFilePath.toFile().toURI().toURL());
-        } catch (GenesisException e) {
-            return ResponseEntity.status(e.getStatus()).body(e.getMessage());
+        if(withDDI) {
+            //Parse DDI
+            log.info(String.format("Try to read DDI file : %s", metadataFilePath));
+            Path ddiFilePath = Paths.get(metadataFilePath);
+            try {
+                variablesMap = DDIReader.getVariablesFromDDI(ddiFilePath.toFile().toURI().toURL());
+            } catch (GenesisException e) {
+                return ResponseEntity.status(e.getStatus()).body(e.getMessage());
+            }
+        }else{
+            //Parse Lunatic
+            log.info(String.format("Try to read lunatic file : %s", metadataFilePath));
+            Path lunaticFilePath = Paths.get(metadataFilePath);
+
+            try {
+                variablesMap = LunaticReader.getVariablesFromLunaticJson(lunaticFilePath);
+            } catch (GenesisException e) {
+                return ResponseEntity.status(e.getStatus()).body(e.getMessage());
+            }
         }
 
         log.info(String.format("Try to read Xml file : %s", xmlFile));
@@ -105,23 +119,25 @@ public ResponseEntity<Object> saveResponsesFromXmlFile(@RequestParam("pathLunati
     @Operation(summary = "Save multiples files in Genesis Database from campaign root folder")
     @PutMapping(path = "/save/lunatic-xml")
     public ResponseEntity<Object> saveResponsesFromXmlCampaignFolder(@RequestParam("campaignName") String campaignName,
-                                                                     @RequestParam(value = "mode", required = false) Mode modeSpecified)
-            throws Exception {
+                                                                     @RequestParam(value = "mode", required = false) Mode modeSpecified,
+                                                                     @RequestParam(value = "withDDI", defaultValue = "true") boolean withDDI
+    )throws Exception {
         List<GenesisError> errors = new ArrayList<>();
 
         log.info("Try to import XML data for campaign : {}", campaignName);
 
         try {
             List<Mode> modesList = controllerUtils.getModesList(campaignName, modeSpecified);
             for (Mode currentMode : modesList) {
-                treatCampaignWithMode(campaignName, currentMode, errors, null);
+                treatCampaignWithMode(campaignName, currentMode, errors, null, withDDI);
             }
         } catch (GenesisException e) {
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
         return ResponseEntity.ok("Data saved");
     }
 
+    //SAVE ALL
     @Operation(summary = "Save all files in Genesis Database (differential data folder only)")
     @PutMapping(path = "/save/lunatic-xml/all-campaigns")
     public ResponseEntity<Object> saveResponsesFromAllCampaignFolders(){
@@ -160,7 +176,8 @@ public ResponseEntity<Object> saveVolumetry() throws IOException {
         volumetryLogService.cleanOldFiles();
         return ResponseEntity.ok("Volumetry saved");
     }
-
+
+    //DELETE
     @Operation(summary = "Delete all responses of one questionnaire")
     @DeleteMapping(path = "/delete-responses/by-questionnaire")
     public ResponseEntity<Object> deleteAllResponsesByQuestionnaire(@RequestParam("idQuestionnaire") String idQuestionnaire) {
@@ -170,6 +187,7 @@ public ResponseEntity<Object> deleteAllResponsesByQuestionnaire(@RequestParam("i
         return ResponseEntity.ok(String.format("%d responses deleted", ndDocuments));
     }
 
+    //GET
     @Operation(summary = "Retrieve responses with IdUE and IdQuestionnaire from Genesis Database")
     @GetMapping(path = "/get-responses/by-ue-and-questionnaire")
     public ResponseEntity<List<SurveyUnitUpdateDto>> findResponsesByUEAndQuestionnaire(@RequestParam("idUE") String idUE,
@@ -322,8 +340,44 @@ public ResponseEntity<List<CampaignWithQuestionnaire>> getCampaignsWithQuestionn
 
     //Utilities
 
+    /**
+     * Checks if DDI is present for a campaign and mode or not and treats it accordingly
+     * @param campaignName name of campaign
+     * @param currentMode mode of collected data
+     * @param errors error list to fill
+     */
+    private void treatCampaignWithMode(String campaignName, Mode currentMode, List<GenesisError> errors, String rootDataFolder) throws GenesisException,
+            SAXException, XMLStreamException {
+        try {
+            fileUtils.findFile(String.format("%s/%s", fileUtils.getSpecFolder(campaignName),currentMode), "ddi[\\w," +
+                    "\\s-]+\\.xml");
+            //DDI if DDI file found
+            treatCampaignWithMode(campaignName, currentMode, errors, rootDataFolder, true);
+        }catch (RuntimeException re){
+            //Lunatic if no DDI
+            log.info("No DDI File found for {}, {} mode. Will try to use Lunatic...", campaignName,
+                    currentMode.getModeName());
+            try{
+                treatCampaignWithMode(campaignName, currentMode, errors, rootDataFolder, false);
+            } catch (IOException | ParserConfigurationException e) {
+                throw new GenesisException(500, e.toString());
+            }
+        }catch (IOException | ParserConfigurationException e){
+            log.error(e.toString());
+            throw new GenesisException(500, e.toString());
+        }
+    }
+
+
+    /**
+     * Treat a campaign with a specific mode
+     * @param campaignName name of campaign
+     * @param mode mode of collected data
+     * @param errors error list to fill
+     * @param withDDI true if it uses DDI, false if Lunatic
+     */
     private void treatCampaignWithMode(String campaignName, Mode mode, List<GenesisError> errors,
-                                       String rootDataFolder) throws IOException, ParserConfigurationException,
+                                       String rootDataFolder, boolean withDDI) throws IOException, ParserConfigurationException,
             SAXException, XMLStreamException {
         log.info("Try to import data for mode : {}", mode.getModeName());
         String dataFolder = rootDataFolder == null ?
@@ -336,16 +390,10 @@ private void treatCampaignWithMode(String campaignName, Mode mode, List<GenesisE
             log.warn("No data file found in folder {}", dataFolder);
             return;
         }
-        //Read DDI
-        VariablesMap variablesMap;
-        try {
-            Path ddiFilePath = fileUtils.findDDIFile(campaignName, mode.getModeName());
-            variablesMap = DDIReader.getVariablesFromDDI(ddiFilePath.toUri().toURL());
-        } catch (Exception e) {
-            log.error(e.toString());
-            errors.add(new GenesisError(e.toString()));
-            return;
-        }
+
+        VariablesMap variablesMap = readMetadatas(campaignName, mode, errors, withDDI);
+        if (variablesMap == null) return;
+
         //For each XML data file
         for (String fileName : dataFiles.stream().filter(s -> s.endsWith(".xml")).toList()) {
             String filepathString = String.format("%s/%s", dataFolder, fileName);
@@ -373,6 +421,34 @@ private void treatCampaignWithMode(String campaignName, Mode mode, List<GenesisE
         }
     }
 
+    private VariablesMap readMetadatas(String campaignName, Mode mode, List<GenesisError> errors, boolean withDDI) {
+        VariablesMap variablesMap;
+        if(withDDI){
+            //Read DDI
+            try {
+                Path ddiFilePath = fileUtils.findFile(String.format("%s/%s", fileUtils.getSpecFolder(campaignName),
+                            mode.getModeName()), "ddi[\\w," + "\\s-]+\\.xml");
+                variablesMap = DDIReader.getVariablesFromDDI(ddiFilePath.toUri().toURL());
+            } catch (Exception e) {
+                log.error(e.toString());
+                errors.add(new GenesisError(e.toString()));
+                return null;
+            }
+        }else{
+            //Read Lunatic
+            try {
+                Path lunaticFilePath = fileUtils.findFile(String.format("%s/%s", fileUtils.getSpecFolder(campaignName),
+                        mode.getModeName()), "lunatic[\\w," + "\\s-]+\\.json");
+                variablesMap = LunaticReader.getVariablesFromLunaticJson(lunaticFilePath);
+            } catch (Exception e) {
+                log.error(e.toString());
+                errors.add(new GenesisError(e.toString()));
+                return null;
+            }
+        }
+        return variablesMap;
+    }
+
     private boolean isDataFileInDoneFolder(Path filepath, String campaignName, String modeFolder) {
         return Path.of(fileUtils.getDoneFolder(campaignName, modeFolder)).resolve(filepath.getFileName()).toFile().exists();
     }

src/main/java/fr/insee/genesis/controller/service/SurveyUnitQualityService.java

@@ -1,6 +1,6 @@
 package fr.insee.genesis.controller.service;
 
-import fr.insee.genesis.controller.sources.ddi.VariablesMap;
+import fr.insee.genesis.controller.sources.metadata.VariablesMap;
 import fr.insee.genesis.controller.utils.DataVerifier;
 import fr.insee.genesis.domain.dtos.SurveyUnitUpdateDto;
 import org.springframework.stereotype.Service;

src/main/java/fr/insee/genesis/controller/sources/ddi/Group.java → src/main/java/fr/insee/genesis/controller/sources/metadata/Group.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.controller.sources.ddi;
+package fr.insee.genesis.controller.sources.metadata;
 
 import lombok.Getter;
 import lombok.NonNull;

src/main/java/fr/insee/genesis/controller/sources/ddi/McqVariable.java → src/main/java/fr/insee/genesis/controller/sources/metadata/McqVariable.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.controller.sources.ddi;
+package fr.insee.genesis.controller.sources.metadata;
 
 import lombok.Builder;
 import lombok.Getter;

src/main/java/fr/insee/genesis/controller/sources/ddi/UcqModality.java → src/main/java/fr/insee/genesis/controller/sources/metadata/UcqModality.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.controller.sources.ddi;
+package fr.insee.genesis.controller.sources.metadata;
 
 import lombok.Getter;
 import lombok.Setter;

src/main/java/fr/insee/genesis/controller/sources/ddi/UcqVariable.java → src/main/java/fr/insee/genesis/controller/sources/metadata/UcqVariable.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.controller.sources.ddi;
+package fr.insee.genesis.controller.sources.metadata;
 
 import lombok.Getter;
 import lombok.Setter;

src/main/java/fr/insee/genesis/controller/sources/ddi/Variable.java → src/main/java/fr/insee/genesis/controller/sources/metadata/Variable.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.controller.sources.ddi;
+package fr.insee.genesis.controller.sources.metadata;
 
 import lombok.Getter;
 import lombok.Setter;

src/main/java/fr/insee/genesis/controller/sources/ddi/VariableType.java → src/main/java/fr/insee/genesis/controller/sources/metadata/VariableType.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.controller.sources.ddi;
+package fr.insee.genesis.controller.sources.metadata;
 
 import lombok.Getter;
 import lombok.extern.log4j.Log4j2;

src/main/java/fr/insee/genesis/controller/sources/ddi/VariablesMap.java → src/main/java/fr/insee/genesis/controller/sources/metadata/VariablesMap.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.controller.sources.ddi;
+package fr.insee.genesis.controller.sources.metadata;
 
 
 import fr.insee.genesis.Constants;

src/main/java/fr/insee/genesis/controller/sources/ddi/DDIReader.java → src/main/java/fr/insee/genesis/controller/sources/metadata/ddi/DDIReader.java

@@ -1,6 +1,12 @@
-package fr.insee.genesis.controller.sources.ddi;
+package fr.insee.genesis.controller.sources.metadata.ddi;
 
 import fr.insee.genesis.Constants;
+import fr.insee.genesis.controller.sources.metadata.Group;
+import fr.insee.genesis.controller.sources.metadata.McqVariable;
+import fr.insee.genesis.controller.sources.metadata.UcqVariable;
+import fr.insee.genesis.controller.sources.metadata.Variable;
+import fr.insee.genesis.controller.sources.metadata.VariableType;
+import fr.insee.genesis.controller.sources.metadata.VariablesMap;
 import fr.insee.genesis.controller.utils.SaxonTransformer;
 import fr.insee.genesis.exceptions.GenesisException;
 import lombok.extern.slf4j.Slf4j;

src/main/java/fr/insee/genesis/controller/sources/metadata/lunatic/ComponentLunatic.java

@@ -0,0 +1,40 @@
+package fr.insee.genesis.controller.sources.metadata.lunatic;
+
+import fr.insee.genesis.controller.sources.metadata.VariableType;
+import lombok.Getter;
+
+import java.util.Arrays;
+
+@Getter
+public enum ComponentLunatic {
+
+    DATE_PICKER("Datepicker", VariableType.DATE),
+    CHECKBOX_BOOLEAN("CheckboxBoolean", VariableType.BOOLEAN),
+    INPUT_NUMBER("InputNumber", null),
+    INPUT("Input", VariableType.STRING),
+    TEXT_AREA("Textarea", VariableType.STRING),
+    RADIO("Radio", VariableType.STRING),
+    CHECKBOX_ONE("CheckboxOne", VariableType.STRING),
+    DROPDOWN("Dropdown", VariableType.STRING),
+    CHECKBOX_GROUP("CheckboxGroup", VariableType.BOOLEAN),
+    SUGGESTER("Suggester", VariableType.STRING),
+    PAIRWISE_LINKS("PairwiseLinks", null),
+    TABLE("Table", null);
+
+    private String jsonName;
+    // Represents the type of the variable expected with this component type
+    // If null, the type is not unique
+    private VariableType type;
+
+    ComponentLunatic(String jsonName, VariableType type) {
+        this.jsonName=jsonName;
+        this.type = type;
+    }
+
+    public static ComponentLunatic fromJsonName(String jsonName) {
+        return Arrays.stream(ComponentLunatic.values())
+                .filter(component -> component.getJsonName().equals(jsonName))
+                .findFirst()
+                .orElse(null);
+    }
+}