Merge branch 'main' of https://github.com/InseeFr/Genesis-API

InseeFr · Oct 7, 2024 · d2dfc56 · d2dfc56
2 parents af1d6fe + b8fcb20
commit d2dfc56
Show file tree

Hide file tree

Showing 14 changed files with 430 additions and 8 deletions.
diff --git a/.gitignore b/.gitignore
@@ -46,3 +46,4 @@ hs_err_pid*
 # test generated files
 src/test/resources/OUT/*
 /src/test/resources/DONE/*
+/src/test/resources/genesis_deleted_schedules/*
diff --git a/pom.xml b/pom.xml
@@ -18,7 +18,7 @@
         <java.version>21</java.version>
         <springdoc.version>2.6.0</springdoc.version>
         <mapstruct.version>1.6.2</mapstruct.version>
-        <cucumber.version>7.19.0</cucumber.version>
+        <cucumber.version>7.20.0</cucumber.version>
         <!-- Proprietes sonar -->
         <jacoco.version>0.8.12</jacoco.version>
         <sonar.core.codeCoveragePlugin>jacoco</sonar.core.codeCoveragePlugin>
@@ -98,6 +98,11 @@
             <artifactId>jackson-databind</artifactId>
             <version>${jackson.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+            <version>${jackson.version}</version>
+        </dependency>
 
         <!-- generate implementation auto -->
         <dependency>

diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java
@@ -11,13 +11,17 @@ public class Constants {
     public static final String FILTER_RESULT_PREFIX = "FILTER_RESULT_";
     public static final String MISSING_SUFFIX = "_MISSING";
     private static final String[] ENO_VARIABLES = {"COMMENT_QE","COMMENT_UE","HEURE_REMPL","MIN_REMPL"};
+
+    public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules";
     public static final String LOOP_NAME_PREFIX = "BOUCLE";
     public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
     public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries";
     public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY";
     public static final String VOLUMETRY_FILE_DATE_FORMAT = "yyyy_MM_dd";
     public static final int VOLUMETRY_FILE_EXPIRATION_DAYS = 30;
 
+    public static final String SCHEDULE_ARCHIVE_FOLDER_NAME = "genesis_deleted_schedules";
+
 
    // XML sequential reading parameters
     public static final int MAX_FILE_SIZE_UNTIL_SEQUENTIAL = 200; // In megabytes

diff --git a/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java b/src/main/java/fr/insee/genesis/controller/rest/ScheduleController.java
@@ -1,6 +1,9 @@
 package fr.insee.genesis.controller.rest;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import fr.insee.genesis.Constants;
+import fr.insee.genesis.domain.model.schedule.KraftwerkExecutionSchedule;
 import fr.insee.genesis.domain.model.schedule.ScheduleModel;
 import fr.insee.genesis.domain.model.schedule.ServiceToCall;
 import fr.insee.genesis.domain.model.schedule.TrustParameters;
@@ -22,8 +25,14 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.time.LocalDateTime;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 @RequestMapping(path = "/schedule")
 @Controller
@@ -75,11 +84,19 @@ public ResponseEntity<Object> addSchedule(
                         useSignature
                 );
                 log.info("New schedule request for survey {} with encryption", surveyName);
-                scheduleApiPort.addSchedule(surveyName, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate,
+                scheduleApiPort.addSchedule(surveyName,
+                        serviceToCall == null ? ServiceToCall.MAIN : serviceToCall,
+                        frequency,
+                        scheduleBeginDate,
+                        scheduleEndDate,
                         trustParameters);
             }else{
                 log.info("New schedule request for survey {}", surveyName);
-                scheduleApiPort.addSchedule(surveyName, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate,
+                scheduleApiPort.addSchedule(surveyName,
+                        serviceToCall == null ? ServiceToCall.MAIN : serviceToCall,
+                        frequency,
+                        scheduleBeginDate,
+                        scheduleEndDate,
                         null);
             }
 
@@ -124,4 +141,36 @@ public ResponseEntity<Object> setSurveyLastExecution(
         }
         return ResponseEntity.ok().build();
     }
+
+    @Operation(summary = "Delete expired schedules")
+    @DeleteMapping(path = "/delete/expired-schedules")
+    public ResponseEntity<Object> deleteExpiredSchedules() throws NotFoundException, IOException {
+        Set<String> storedSurveySchedulesNames = new HashSet<>();
+        for(ScheduleModel scheduleModel : scheduleApiPort.getAllSchedules()){
+            storedSurveySchedulesNames.add(scheduleModel.getSurveyName());
+        }
+        for (String surveyScheduleName : storedSurveySchedulesNames) {
+            List<KraftwerkExecutionSchedule> deletedKraftwerkExecutionSchedules = scheduleApiPort.deleteExpiredSchedules(surveyScheduleName);
+            //Save in JSON log
+            if(!deletedKraftwerkExecutionSchedules.isEmpty()) {
+                Path jsonLogPath = Path.of(fileUtils.getLogFolder(), Constants.SCHEDULE_ARCHIVE_FOLDER_NAME,
+                        surveyScheduleName + ".json");
+                ObjectMapper objectMapper = new ObjectMapper();
+                objectMapper.registerModule(new JavaTimeModule());
+                String jsonToWrite = objectMapper.writeValueAsString(deletedKraftwerkExecutionSchedules);
+                if(Files.exists(jsonLogPath)){
+                    //Remove last ] and append survey
+                    StringBuilder content = new StringBuilder(Files.readString(jsonLogPath));
+                    content.setCharAt(content.length()-1, ',');
+                    content.append(jsonToWrite, 1, jsonToWrite.length()-1);
+                    content.append(']');
+                    Files.write(jsonLogPath, content.toString().getBytes(), StandardOpenOption.TRUNCATE_EXISTING);
+                }else {
+                    Files.createDirectories(jsonLogPath.getParent());
+                    Files.write(jsonLogPath, jsonToWrite.getBytes());
+                }
+            }
+        }
+        return ResponseEntity.ok().build();
+    }
 }
diff --git a/src/main/java/fr/insee/genesis/domain/model/schedule/KraftwerkExecutionSchedule.java b/src/main/java/fr/insee/genesis/domain/model/schedule/KraftwerkExecutionSchedule.java
@@ -1,5 +1,6 @@
 package fr.insee.genesis.domain.model.schedule;
 
+import com.fasterxml.jackson.annotation.JsonFormat;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 
@@ -12,7 +13,10 @@ public class KraftwerkExecutionSchedule {
 
     private ServiceToCall serviceToCall;
 
+    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss")
     private LocalDateTime scheduleBeginDate;
+
+    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss")
     private LocalDateTime scheduleEndDate;
 
     private TrustParameters trustParameters;

diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ScheduleApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ScheduleApiPort.java
@@ -1,5 +1,6 @@
 package fr.insee.genesis.domain.ports.api;
 
+import fr.insee.genesis.domain.model.schedule.KraftwerkExecutionSchedule;
 import fr.insee.genesis.domain.model.schedule.ScheduleModel;
 import fr.insee.genesis.domain.model.schedule.ServiceToCall;
 import fr.insee.genesis.domain.model.schedule.TrustParameters;
@@ -22,6 +23,8 @@ void addSchedule(String surveyName,
 
     void deleteSchedule(String surveyName) throws NotFoundException;
 
+    List<KraftwerkExecutionSchedule> deleteExpiredSchedules(String surveyName) throws NotFoundException;
+
     void updateLastExecutionName(String surveyName, LocalDateTime newDate) throws NotFoundException;
 
     long countSchedules();

diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SchedulePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SchedulePersistencePort.java
@@ -1,5 +1,6 @@
 package fr.insee.genesis.domain.ports.spi;
 
+import fr.insee.genesis.domain.model.schedule.KraftwerkExecutionSchedule;
 import fr.insee.genesis.domain.model.schedule.ScheduleModel;
 
 import java.util.List;
@@ -14,4 +15,6 @@ public interface SchedulePersistencePort {
     void deleteBySurveyName(String surveyName);
 
     long countSchedules();
+
+    public List<KraftwerkExecutionSchedule> removeExpiredSchedules(ScheduleModel scheduleModel);
 }
diff --git a/src/main/java/fr/insee/genesis/domain/service/schedule/ScheduleService.java b/src/main/java/fr/insee/genesis/domain/service/schedule/ScheduleService.java
@@ -79,6 +79,30 @@ public void deleteSchedule(String surveyName) throws NotFoundException {
         schedulePersistencePort.deleteBySurveyName(surveyName);
     }
 
+    @Override
+    public List<KraftwerkExecutionSchedule> deleteExpiredSchedules(String surveyName) throws NotFoundException {
+        List<ScheduleModel> scheduleModels = schedulePersistencePort.findBySurveyName(surveyName);
+        if (scheduleModels.isEmpty()) {
+            throw new NotFoundException();
+        }
+        List<KraftwerkExecutionSchedule> deletedKraftwerkExecutionSchedules = new ArrayList<>();
+        for (ScheduleModel scheduleModel : scheduleModels) {
+            deletedKraftwerkExecutionSchedules.addAll(schedulePersistencePort.removeExpiredSchedules(scheduleModel));
+            //Delete schedule if empty kraftwerkExecutionScheduleList
+            schedulePersistencePort.findBySurveyName(surveyName)
+                    .stream()
+                    .filter(storedSurveySchedule -> storedSurveySchedule.getKraftwerkExecutionScheduleList().isEmpty())
+                    .forEach(storedSurveySchedule -> {
+                        try {
+                            deleteSchedule(surveyName);
+                        } catch (NotFoundException e) {
+                            log.error("Tried to delete schedule for {} but wasn't found !", surveyName);
+                        }
+                    });
+        }
+        return deletedKraftwerkExecutionSchedules;
+    }
+
     @Override
     public void updateLastExecutionName(String surveyName, LocalDateTime newDate) throws NotFoundException {
         List<ScheduleModel> scheduleModels = schedulePersistencePort.findBySurveyName(surveyName);

diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/ScheduleMongoAdapter.java
@@ -1,21 +1,34 @@
 package fr.insee.genesis.infrastructure.adapter;
 
+import fr.insee.genesis.Constants;
+import fr.insee.genesis.domain.model.schedule.KraftwerkExecutionSchedule;
 import fr.insee.genesis.domain.model.schedule.ScheduleModel;
 import fr.insee.genesis.domain.ports.spi.SchedulePersistencePort;
 import fr.insee.genesis.infrastructure.mappers.ScheduleDocumentMapper;
 import fr.insee.genesis.infrastructure.repository.ScheduleMongoDBRepository;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
 import org.springframework.stereotype.Service;
 
+import java.time.LocalDateTime;
+import java.util.ArrayList;
 import java.util.List;
 
 @Service
+@Slf4j
 public class ScheduleMongoAdapter implements SchedulePersistencePort {
     private final ScheduleMongoDBRepository scheduleMongoDBRepository;
+    private final MongoTemplate mongoTemplate;
+
 
     @Autowired
-    public ScheduleMongoAdapter(ScheduleMongoDBRepository scheduleMongoDBRepository) {
+    public ScheduleMongoAdapter(ScheduleMongoDBRepository scheduleMongoDBRepository, MongoTemplate mongoTemplate) {
         this.scheduleMongoDBRepository = scheduleMongoDBRepository;
+        this.mongoTemplate = mongoTemplate;
     }
 
     @Override
@@ -42,4 +55,23 @@ public void deleteBySurveyName(String surveyName) {
     public long countSchedules() {
         return scheduleMongoDBRepository.count();
     }
+
+    @Override
+    public List<KraftwerkExecutionSchedule> removeExpiredSchedules(ScheduleModel scheduleModel) {
+        List<KraftwerkExecutionSchedule> deletedKraftwerkExecutionSchedules = new ArrayList<>();
+        for (KraftwerkExecutionSchedule kraftwerkExecutionScheduleToRemove :
+                scheduleModel.getKraftwerkExecutionScheduleList().stream().filter(
+                        kraftwerkExecutionSchedule -> kraftwerkExecutionSchedule.getScheduleEndDate().isBefore(LocalDateTime.now())
+                ).toList()) {
+            deletedKraftwerkExecutionSchedules.add(kraftwerkExecutionScheduleToRemove);
+            Query query =
+                    Query.query(Criteria.where("scheduleEndDate").is(kraftwerkExecutionScheduleToRemove.getScheduleEndDate()));
+            mongoTemplate.updateMulti(Query.query(Criteria.where("surveyName").is(scheduleModel.getSurveyName())), new Update().pull(
+                            "kraftwerkExecutionScheduleList", query),
+                    Constants.MONGODB_SCHEDULE_COLLECTION_NAME);
+            log.info("Removed kraftwerk execution schedule on {} because it is expired since {}", scheduleModel.getSurveyName(),
+                    kraftwerkExecutionScheduleToRemove.getScheduleEndDate());
+        }
+        return deletedKraftwerkExecutionSchedules;
+    }
 }
diff --git a/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java b/src/main/java/fr/insee/genesis/infrastructure/utils/FileUtils.java
@@ -29,9 +29,12 @@ public class FileUtils {
 
 	private final String specFolderSource;
 
+	private final String logFolderSource;
+
 	public FileUtils(Config config) {
 		this.dataFolderSource = config.getDataFolderSource();
 		this.specFolderSource = config.getSpecFolderSource();
+		this.logFolderSource = config.getLogFolder();
 	}
 
 	/**
@@ -188,6 +191,14 @@ public String getKraftwerkOutFolder(String campaign) {
 		return  String.format("%s/%s/%s", dataFolderSource, "out", campaign);
 	}
 
+	/**
+	 * Get the path of the folder where the log files are stored
+	 * @return Path of the output folder
+	 */
+	public String getLogFolder() {
+		return logFolderSource;
+	}
+
 	/**
 	 * Write a text file.
 	 * @param filePath Path to the file.