Hotfix/quick fix minio bugs

pom.xml

@@ -13,7 +13,7 @@
     <groupId>fr.insee.rmes</groupId>
     <artifactId>Bauhaus-BO</artifactId>
     <packaging>jar</packaging>
-    <version>4.1.6</version>
+    <version>4.1.7-beta4</version>
     <name>Bauhaus-Back-Office</name>
     <description>Back-office services for Bauhaus</description>
     <url>https://github.com/InseeFr/Bauhaus-Back-Office</url>
@@ -87,6 +87,7 @@
         <zt.version>1.17</zt.version>
         <saxon.version>12.4</saxon.version>
         <minio.version>8.5.11</minio.version>
+        <testcontainers-minio.version>1.20.4</testcontainers-minio.version>
 
 
         <!-- SONAR -->
@@ -100,6 +101,7 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
+
         <dependency>
             <groupId>io.minio</groupId>
             <artifactId>minio</artifactId>
@@ -242,15 +244,30 @@
             <version>${flexmark.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>minio</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>
+            <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>

src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java

@@ -18,45 +18,51 @@ public class FileSystemOperation implements FilesOperations {
     protected Config config;
 
     @Override
-    public void delete(String path) {
+    public void delete(Path absolutePath) {
         try {
-            Files.delete(Paths.get(path));
+            Files.delete(absolutePath);
         } catch (IOException e) {
-            throw new RmesFileException("Failed to delete file: " + path, e);
+            throw new RmesFileException(absolutePath.getFileName().toString(), "Failed to delete file: " + absolutePath, e);
         }
     }
 
     @Override
-    public InputStream read(String fileName) {
+    public InputStream readInDirectoryGestion(String fileName) {
         try {
             return Files.newInputStream(Paths.get(config.getDocumentsStorageGestion()).resolve(fileName));
         } catch (IOException e) {
-            throw new RmesFileException("Failed to read file: " + fileName, e);
+            throw new RmesFileException(fileName, "Failed to read file: " + fileName, e);
         }
     }
 
     @Override
-    public void write(InputStream content, Path destPath) {
+    public boolean existsInStorageGestion(String filename) {
+        return Files.exists(Paths.get(config.getDocumentsStorageGestion()).resolve(filename));
+    }
+
+    @Override
+    public void writeToDirectoryGestion(InputStream content, Path destPath) {
         try {
             Files.copy(content, destPath, StandardCopyOption.REPLACE_EXISTING);
         } catch (IOException e) {
-            throw new RmesFileException("Failed to write file: " + destPath, e);
+            throw new RmesFileException(destPath.toString(),"Failed to write file: " + destPath, e);
         }
     }
 
     @Override
-    public void copy(String srcPath, String destPath)  {
+    public void copyFromGestionToPublication(String srcPath, String destPath)  {
         Path file = Paths.get(srcPath);
         Path targetPath = Paths.get(destPath);
         try {
             Files.copy(file, targetPath.resolve(file.getFileName()), StandardCopyOption.REPLACE_EXISTING);
         } catch (IOException e) {
-            throw new RmesFileException("Failed to copy file : " + srcPath + " to " + destPath, e);
+            throw new RmesFileException(srcPath, "Failed to copy file : " + srcPath + " to " + destPath, e);
         }
     }
 
     @Override
     public boolean dirExists(Path gestionStorageFolder) {
         return Files.isDirectory(requireNonNull(gestionStorageFolder));
     }
+
 }

src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java

@@ -4,10 +4,14 @@
 import java.nio.file.Path;
 
 public interface FilesOperations {
-    void delete(String path);
-    InputStream read(String path);
-    void write(InputStream content, Path destPath);
-    void copy(String srcPath, String destPath);
+    default void delete(Path absolutePath){
+        throw new UnsupportedOperationException("Not implemented yet.");
+    }
+    InputStream readInDirectoryGestion(String filename);
+    void writeToDirectoryGestion(InputStream content, Path destPath);
+    void copyFromGestionToPublication(String srcPath, String destPath);
 
     boolean dirExists(Path gestionStorageFolder);
+
+    boolean existsInStorageGestion(String filename);
 }

src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java

@@ -12,21 +12,25 @@
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 
+import static java.util.Objects.requireNonNull;
+
 public record MinioFilesOperation(MinioClient minioClient, String bucketName, String directoryGestion, String directoryPublication) implements FilesOperations {
 
-    private static final Logger logger = LoggerFactory.getLogger(MinioFilesOperation.class);
+    static final Logger logger = LoggerFactory.getLogger(MinioFilesOperation.class);
 
     @Override
-    public InputStream read(String pathFile){
-        String objectName= extractFileName(pathFile);
+    public InputStream readInDirectoryGestion(String filename){
+        String objectName = directoryGestion + "/" + filename;
+
+        logger.debug("Reading file with name {} from path {} as object {} in bucket {}", filename, filename, objectName, bucketName);
 
         try {
             return minioClient.getObject(GetObjectArgs.builder()
                     .bucket(bucketName)
-                    .object(directoryGestion +"/"+ objectName)
+                    .object(objectName)
                     .build());
         } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
-            throw new RmesFileException("Error reading file: " + objectName, e);
+            throw new RmesFileException(filename, "Error reading file: " + filename+" as object `"+objectName+"` in bucket "+bucketName, e);
         }
     }
     private static String extractFileName(String filePath) {
@@ -36,38 +40,59 @@ private static String extractFileName(String filePath) {
         return Path.of(filePath).getFileName().toString();
     }
 
+    @Override
+    public boolean existsInStorageGestion(String filename) {
+        String objectName = directoryGestion + "/" + filename;
+        logger.debug("Check existence of file with name {} as object {} in bucket {}", filename, objectName, bucketName);
+        try {
+            return minioClient.statObject(StatObjectArgs.builder()
+                    .bucket(bucketName)
+                    .object(objectName)
+                    .build()).size() > 0;
+        } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
+            return false;
+        }
+    }
 
     @Override
-    public void write(InputStream content, Path objectName) {
+    public void writeToDirectoryGestion(InputStream content, Path filePath) {
+        String filename = filePath.getFileName().toString();
+        String objectName = directoryGestion + "/" + filename;
+        logger.debug("Writing to file with name {} from path {} as object {} in bucket {}", filename, filePath, objectName, bucketName);
         try {
             minioClient.putObject(PutObjectArgs.builder()
                     .bucket(bucketName)
-                    .object(directoryGestion +"/"+ objectName.getFileName().toString())
+                    .object(objectName)
                     .stream(content, content.available(), -1)
                     .build());
         } catch (IOException | ErrorResponseException | InsufficientDataException | InternalException |
                  InvalidKeyException | InvalidResponseException | NoSuchAlgorithmException | ServerException |
                  XmlParserException e) {
-            throw new RmesFileException("Error writing file: " + objectName, e);
+            throw new RmesFileException(filePath.toString(), "Error writing file: " + filename+ "as object `"+objectName+"` in bucket "+bucketName, e);
         }
     }
 
     @Override
-    public void copy(String srcObjectName, String destObjectName)  {
+    public void copyFromGestionToPublication(String srcObjectName, String destObjectName)  {
+
+        String srcObject = directoryGestion + "/" + extractFileName(srcObjectName);
+        String destObject = directoryPublication + "/" + extractFileName(srcObjectName);
+
+        logger.debug("Copy from source {} as object {} to destination {} as object {} in bucket {}", srcObjectName, srcObject, destObjectName, destObject, bucketName);
 
         try {
             CopySource source = CopySource.builder()
                     .bucket(bucketName)
-                    .object(directoryGestion +"/"+ extractFileName(srcObjectName))
+                    .object(srcObject)
                     .build();
 
             minioClient.copyObject(CopyObjectArgs.builder()
                     .bucket(bucketName)
-                    .object(directoryPublication +"/"+ extractFileName(srcObjectName))
+                    .object(destObject)
                     .source(source)
                     .build());
         } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
-            throw new RmesFileException("Error copying file from " + srcObjectName + " to " + destObjectName, e);
+            throw new RmesFileException(srcObjectName,"Error copying file from `" + srcObject + "` to `" + destObject+"` in bucket "+bucketName, e);
         }
     }
 
@@ -78,14 +103,18 @@ public boolean dirExists(Path gestionStorageFolder) {
     }
 
      @Override
-    public void delete(String objectName) {
+    public void delete(Path absolutePath) {
+        String objectName = absolutePath.getFileName().toString();
+
+        logger.debug("Delete file with path {} as object {} in bucket {}", absolutePath, objectName, bucketName);
+
         try {
             minioClient.removeObject(RemoveObjectArgs.builder()
                     .bucket(bucketName)
                     .object(objectName)
                     .build());
         } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
-            throw new RmesFileException("Error deleting file: " + objectName, e);
+            throw new RmesFileException(objectName,"Error deleting file: " + objectName+" in bucket "+bucketName, e);
         }
     }
 

src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExport.java

@@ -160,19 +160,19 @@ private Set<String> exportRubricsDocuments(JSONObject sims, Path directory) thro
 
 		for (int i = 0; i < documents.length(); i++) {
 			JSONObject document = documents.getJSONObject(i);
-			String url = document.getString("url").replace("file://", "");
+			String url = DocumentsUtils.getDocumentUrlFromDocument(document);
 			if(!history.contains(url)){
 				history.add(url);
 				logger.debug("Extracting document {}", url);
 
 
-				Path documentPath = Path.of(url);
+				String documentFilename = DocumentsUtils.getDocumentNameFromUrl(url);
 
-				if(!Files.exists(documentPath)){
+				if(!documentsUtils.existsInStorage(documentFilename)){
 					missingDocuments.add(document.getString("id"));
 				} else {
 					String documentFileName = FilesUtils.generateFinalFileNameWithExtension(UriUtils.getLastPartFromUri(url), maxLength);
-					try (InputStream inputStream = Files.newInputStream(documentPath)){
+					try (InputStream inputStream = documentsUtils.retrieveDocumentFromStorage(documentFilename)){
 						Path documentDirectory = Path.of(directory.toString(), "documents");
 						if (!Files.exists(documentDirectory)) {
 							logger.debug("Creating the documents folder");

src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java

@@ -44,12 +44,12 @@ public void publishAllDocumentsInSims(String idSims) throws RmesException {
 		JSONArray listDoc = docUtils.getListDocumentSims(idSims);
 
 		Map<Integer,String> mapIdUrls = new HashMap<>();
-		listDoc.forEach(doc -> mapIdUrls.put(docUtils.getIdFromJson((JSONObject) doc), docUtils.getDocumentUrlFromDocument((JSONObject) doc)));
+		listDoc.forEach(doc -> mapIdUrls.put(docUtils.getIdFromJson((JSONObject) doc), DocumentsUtils.getDocumentUrlFromDocument((JSONObject) doc)));
 
 		for (Map.Entry<Integer, String> doc : mapIdUrls.entrySet()) {
 			String docId = doc.getKey().toString();
 			String originalPath = doc.getValue();
-			String filename = docUtils.getDocumentNameFromUrl(originalPath);
+			String filename = DocumentsUtils.getDocumentNameFromUrl(originalPath);
 			// Publish the physical files
 			copyFileInPublicationFolders(originalPath);
 			// Change url in document (getModelToPublish) and publish the RDF
@@ -68,10 +68,8 @@ public void publishAllDocumentsInSims(String idSims) throws RmesException {
 	}
 
 	private void copyFileInPublicationFolders(String originalPath){
-        String documentsStoragePublicationInterne = config.getDocumentsStoragePublicationInterne();
         String documentsStoragePublicationExterne = config.getDocumentsStoragePublicationExterne();
-        filesOperations.copy(originalPath, documentsStoragePublicationInterne);
-        filesOperations.copy(originalPath, documentsStoragePublicationExterne);
+        filesOperations.copyFromGestionToPublication(originalPath, documentsStoragePublicationExterne);
 	}
 
 

src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java

@@ -41,7 +41,6 @@
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URI;
-import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -282,7 +281,7 @@ private void checkUrlDoesNotExist(String id, String url, int errorCode, String e
         JSONObject existingUriJson = repoGestion.getResponseAsObject(DocumentsQueries.getDocumentUriQuery(url));
         if (existingUriJson.length() > 0) {
             String uri = existingUriJson.getString(Constants.DOCUMENT);
-            String existingId = getIdFromUri(uri);
+            String existingId = getDocumentNameFromUrl(uri);
             if (!existingId.equals(id)) {
                 throw new RmesNotAcceptableException(errorCode, errorMessage, uri);
             }
@@ -497,7 +496,7 @@ private void uploadFile(InputStream documentFile, String documentName, String ur
             throw new RmesBadRequestException(ErrorCodes.DOCUMENT_CREATION_EXISTING_FILE,
                     "There is already a document with that name.", documentName);
         }
-        filesOperations.write(documentFile, path);
+        filesOperations.writeToDirectoryGestion(documentFile, path);
         // don't throw an error if a file already exists under this name
     }
 
@@ -556,21 +555,13 @@ private void changeDocumentsURL(String iri, String docUrl, String newUrl) throws
 
     private void deleteFile(String docUrl) {
         Path path = Paths.get(docUrl);
-        try {
-            Files.delete(path);
-        } catch (IOException e) {
-            logger.error(e.getMessage());
-        }
+        filesOperations.delete(path);
     }
 
-    public String getDocumentNameFromUrl(String docUrl) {
+    public static String getDocumentNameFromUrl(String docUrl) {
         return UriUtils.getLastPartFromUri(docUrl);
     }
 
-    private String getIdFromUri(String uri) {
-        return UriUtils.getLastPartFromUri(uri);
-    }
-
     private String createFileUrl(String name) throws RmesException {
         Path gestionStorageFolder=Path.of(config.getDocumentsStorageGestion());
         if (!filesOperations.dirExists(gestionStorageFolder)){
@@ -603,7 +594,7 @@ private IRI getDocumentUri(IRI url) throws RmesException {
         return RdfUtils.toURI(uri.getString(Constants.DOCUMENT));
     }
 
-    public String getDocumentUrlFromDocument(JSONObject jsonDoc) {
+    public static String getDocumentUrlFromDocument(JSONObject jsonDoc) {
         return jsonDoc.getString(Constants.URL).replace(SCHEME_FILE, "");
     }
 
@@ -655,7 +646,7 @@ public Document buildDocumentFromJson(JSONObject jsonDoc) {
         return doc;
     }
 
-    private String getDocumentFilename(String id) throws RmesException {
+    protected String getDocumentFilename(String id) throws RmesException {
         JSONObject jsonDoc = getDocument(id, false);
         String url = getDocumentUrlFromDocument(jsonDoc);
         return getDocumentNameFromUrl(url);
@@ -671,7 +662,7 @@ private String getDocumentFilename(String id) throws RmesException {
     public ResponseEntity<org.springframework.core.io.Resource> downloadDocumentFile(String id) throws RmesException {
         String filePath = getDocumentFilename(id);
 
-        try (InputStream inputStream = filesOperations.read(filePath)) { // Lire via l'abstraction et utiliser try-with-resources
+        try (InputStream inputStream = filesOperations.readInDirectoryGestion(filePath)) { // Lire via l'abstraction et utiliser try-with-resources
             byte[] data = StreamUtils.copyToByteArray(inputStream); // Convertir InputStream en byte[]
 
             HttpHeaders headers = new HttpHeaders();
@@ -696,5 +687,13 @@ private String getFileName(String path) {
         // Extraire juste le nom de fichier du chemin
         return Paths.get(path).getFileName().toString();
     }
+
+    public InputStream retrieveDocumentFromStorage(String filename) {
+        return filesOperations.readInDirectoryGestion(filename);
+    }
+
+    public boolean existsInStorage(String filename) {
+        return filesOperations.existsInStorageGestion(filename);
+    }
 }