diff --git a/pom.xml b/pom.xml index 02054a40b..8f0eec112 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ fr.insee.rmes Bauhaus-BO jar - 4.1.6 + 4.1.7-beta4 Bauhaus-Back-Office Back-office services for Bauhaus https://github.com/InseeFr/Bauhaus-Back-Office @@ -87,6 +87,7 @@ 1.17 12.4 8.5.11 + 1.20.4 @@ -100,6 +101,7 @@ org.springframework.boot spring-boot-starter-web + io.minio minio @@ -242,15 +244,30 @@ ${flexmark.version} + + org.testcontainers + minio + test + + org.testcontainers testcontainers + test org.testcontainers junit-jupiter + test + + + org.slf4j + slf4j-simple + test + + diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java b/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java index 1be00cb9a..a8769180e 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java @@ -18,40 +18,45 @@ public class FileSystemOperation implements FilesOperations { protected Config config; @Override - public void delete(String path) { + public void delete(Path absolutePath) { try { - Files.delete(Paths.get(path)); + Files.delete(absolutePath); } catch (IOException e) { - throw new RmesFileException("Failed to delete file: " + path, e); + throw new RmesFileException(absolutePath.getFileName().toString(), "Failed to delete file: " + absolutePath, e); } } @Override - public InputStream read(String fileName) { + public InputStream readInDirectoryGestion(String fileName) { try { return Files.newInputStream(Paths.get(config.getDocumentsStorageGestion()).resolve(fileName)); } catch (IOException e) { - throw new RmesFileException("Failed to read file: " + fileName, e); + throw new RmesFileException(fileName, "Failed to read file: " + fileName, e); } } @Override - public void write(InputStream content, Path destPath) { + public boolean existsInStorageGestion(String filename) { + return Files.exists(Paths.get(config.getDocumentsStorageGestion()).resolve(filename)); + } + + @Override + public void writeToDirectoryGestion(InputStream content, Path destPath) { try { Files.copy(content, destPath, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { - throw new RmesFileException("Failed to write file: " + destPath, e); + throw new RmesFileException(destPath.toString(),"Failed to write file: " + destPath, e); } } @Override - public void copy(String srcPath, String destPath) { + public void copyFromGestionToPublication(String srcPath, String destPath) { Path file = Paths.get(srcPath); Path targetPath = Paths.get(destPath); try { Files.copy(file, targetPath.resolve(file.getFileName()), StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { - throw new RmesFileException("Failed to copy file : " + srcPath + " to " + destPath, e); + throw new RmesFileException(srcPath, "Failed to copy file : " + srcPath + " to " + destPath, e); } } @@ -59,4 +64,5 @@ public void copy(String srcPath, String destPath) { public boolean dirExists(Path gestionStorageFolder) { return Files.isDirectory(requireNonNull(gestionStorageFolder)); } + } diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java b/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java index 7f7f6453f..d067c126c 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java @@ -4,10 +4,14 @@ import java.nio.file.Path; public interface FilesOperations { - void delete(String path); - InputStream read(String path); - void write(InputStream content, Path destPath); - void copy(String srcPath, String destPath); + default void delete(Path absolutePath){ + throw new UnsupportedOperationException("Not implemented yet."); + } + InputStream readInDirectoryGestion(String filename); + void writeToDirectoryGestion(InputStream content, Path destPath); + void copyFromGestionToPublication(String srcPath, String destPath); boolean dirExists(Path gestionStorageFolder); + + boolean existsInStorageGestion(String filename); } diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java b/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java index 0b794034e..577389403 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java @@ -12,21 +12,25 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; +import static java.util.Objects.requireNonNull; + public record MinioFilesOperation(MinioClient minioClient, String bucketName, String directoryGestion, String directoryPublication) implements FilesOperations { - private static final Logger logger = LoggerFactory.getLogger(MinioFilesOperation.class); + static final Logger logger = LoggerFactory.getLogger(MinioFilesOperation.class); @Override - public InputStream read(String pathFile){ - String objectName= extractFileName(pathFile); + public InputStream readInDirectoryGestion(String filename){ + String objectName = directoryGestion + "/" + filename; + + logger.debug("Reading file with name {} from path {} as object {} in bucket {}", filename, filename, objectName, bucketName); try { return minioClient.getObject(GetObjectArgs.builder() .bucket(bucketName) - .object(directoryGestion +"/"+ objectName) + .object(objectName) .build()); } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) { - throw new RmesFileException("Error reading file: " + objectName, e); + throw new RmesFileException(filename, "Error reading file: " + filename+" as object `"+objectName+"` in bucket "+bucketName, e); } } private static String extractFileName(String filePath) { @@ -36,38 +40,59 @@ private static String extractFileName(String filePath) { return Path.of(filePath).getFileName().toString(); } + @Override + public boolean existsInStorageGestion(String filename) { + String objectName = directoryGestion + "/" + filename; + logger.debug("Check existence of file with name {} as object {} in bucket {}", filename, objectName, bucketName); + try { + return minioClient.statObject(StatObjectArgs.builder() + .bucket(bucketName) + .object(objectName) + .build()).size() > 0; + } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) { + return false; + } + } @Override - public void write(InputStream content, Path objectName) { + public void writeToDirectoryGestion(InputStream content, Path filePath) { + String filename = filePath.getFileName().toString(); + String objectName = directoryGestion + "/" + filename; + logger.debug("Writing to file with name {} from path {} as object {} in bucket {}", filename, filePath, objectName, bucketName); try { minioClient.putObject(PutObjectArgs.builder() .bucket(bucketName) - .object(directoryGestion +"/"+ objectName.getFileName().toString()) + .object(objectName) .stream(content, content.available(), -1) .build()); } catch (IOException | ErrorResponseException | InsufficientDataException | InternalException | InvalidKeyException | InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException e) { - throw new RmesFileException("Error writing file: " + objectName, e); + throw new RmesFileException(filePath.toString(), "Error writing file: " + filename+ "as object `"+objectName+"` in bucket "+bucketName, e); } } @Override - public void copy(String srcObjectName, String destObjectName) { + public void copyFromGestionToPublication(String srcObjectName, String destObjectName) { + + String srcObject = directoryGestion + "/" + extractFileName(srcObjectName); + String destObject = directoryPublication + "/" + extractFileName(srcObjectName); + + logger.debug("Copy from source {} as object {} to destination {} as object {} in bucket {}", srcObjectName, srcObject, destObjectName, destObject, bucketName); try { CopySource source = CopySource.builder() .bucket(bucketName) - .object(directoryGestion +"/"+ extractFileName(srcObjectName)) + .object(srcObject) .build(); minioClient.copyObject(CopyObjectArgs.builder() .bucket(bucketName) - .object(directoryPublication +"/"+ extractFileName(srcObjectName)) + .object(destObject) .source(source) .build()); } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) { - throw new RmesFileException("Error copying file from " + srcObjectName + " to " + destObjectName, e); + throw new RmesFileException(srcObjectName,"Error copying file from `" + srcObject + "` to `" + destObject+"` in bucket "+bucketName, e); } } @@ -78,14 +103,18 @@ public boolean dirExists(Path gestionStorageFolder) { } @Override - public void delete(String objectName) { + public void delete(Path absolutePath) { + String objectName = absolutePath.getFileName().toString(); + + logger.debug("Delete file with path {} as object {} in bucket {}", absolutePath, objectName, bucketName); + try { minioClient.removeObject(RemoveObjectArgs.builder() .bucket(bucketName) .object(objectName) .build()); } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) { - throw new RmesFileException("Error deleting file: " + objectName, e); + throw new RmesFileException(objectName,"Error deleting file: " + objectName+" in bucket "+bucketName, e); } } diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExport.java b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExport.java index 46eef73dd..e8dc43ccb 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExport.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExport.java @@ -160,19 +160,19 @@ private Set exportRubricsDocuments(JSONObject sims, Path directory) thro for (int i = 0; i < documents.length(); i++) { JSONObject document = documents.getJSONObject(i); - String url = document.getString("url").replace("file://", ""); + String url = DocumentsUtils.getDocumentUrlFromDocument(document); if(!history.contains(url)){ history.add(url); logger.debug("Extracting document {}", url); - Path documentPath = Path.of(url); + String documentFilename = DocumentsUtils.getDocumentNameFromUrl(url); - if(!Files.exists(documentPath)){ + if(!documentsUtils.existsInStorage(documentFilename)){ missingDocuments.add(document.getString("id")); } else { String documentFileName = FilesUtils.generateFinalFileNameWithExtension(UriUtils.getLastPartFromUri(url), maxLength); - try (InputStream inputStream = Files.newInputStream(documentPath)){ + try (InputStream inputStream = documentsUtils.retrieveDocumentFromStorage(documentFilename)){ Path documentDirectory = Path.of(directory.toString(), "documents"); if (!Files.exists(documentDirectory)) { logger.debug("Creating the documents folder"); diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java index 51b6a327d..83aec577f 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java @@ -44,12 +44,12 @@ public void publishAllDocumentsInSims(String idSims) throws RmesException { JSONArray listDoc = docUtils.getListDocumentSims(idSims); Map mapIdUrls = new HashMap<>(); - listDoc.forEach(doc -> mapIdUrls.put(docUtils.getIdFromJson((JSONObject) doc), docUtils.getDocumentUrlFromDocument((JSONObject) doc))); + listDoc.forEach(doc -> mapIdUrls.put(docUtils.getIdFromJson((JSONObject) doc), DocumentsUtils.getDocumentUrlFromDocument((JSONObject) doc))); for (Map.Entry doc : mapIdUrls.entrySet()) { String docId = doc.getKey().toString(); String originalPath = doc.getValue(); - String filename = docUtils.getDocumentNameFromUrl(originalPath); + String filename = DocumentsUtils.getDocumentNameFromUrl(originalPath); // Publish the physical files copyFileInPublicationFolders(originalPath); // Change url in document (getModelToPublish) and publish the RDF @@ -68,10 +68,8 @@ public void publishAllDocumentsInSims(String idSims) throws RmesException { } private void copyFileInPublicationFolders(String originalPath){ - String documentsStoragePublicationInterne = config.getDocumentsStoragePublicationInterne(); String documentsStoragePublicationExterne = config.getDocumentsStoragePublicationExterne(); - filesOperations.copy(originalPath, documentsStoragePublicationInterne); - filesOperations.copy(originalPath, documentsStoragePublicationExterne); + filesOperations.copyFromGestionToPublication(originalPath, documentsStoragePublicationExterne); } diff --git a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java index 98656e17f..8f2979606 100644 --- a/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java +++ b/src/main/java/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java @@ -41,7 +41,6 @@ import java.io.InputStream; import java.net.MalformedURLException; import java.net.URI; -import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; @@ -282,7 +281,7 @@ private void checkUrlDoesNotExist(String id, String url, int errorCode, String e JSONObject existingUriJson = repoGestion.getResponseAsObject(DocumentsQueries.getDocumentUriQuery(url)); if (existingUriJson.length() > 0) { String uri = existingUriJson.getString(Constants.DOCUMENT); - String existingId = getIdFromUri(uri); + String existingId = getDocumentNameFromUrl(uri); if (!existingId.equals(id)) { throw new RmesNotAcceptableException(errorCode, errorMessage, uri); } @@ -497,7 +496,7 @@ private void uploadFile(InputStream documentFile, String documentName, String ur throw new RmesBadRequestException(ErrorCodes.DOCUMENT_CREATION_EXISTING_FILE, "There is already a document with that name.", documentName); } - filesOperations.write(documentFile, path); + filesOperations.writeToDirectoryGestion(documentFile, path); // don't throw an error if a file already exists under this name } @@ -556,21 +555,13 @@ private void changeDocumentsURL(String iri, String docUrl, String newUrl) throws private void deleteFile(String docUrl) { Path path = Paths.get(docUrl); - try { - Files.delete(path); - } catch (IOException e) { - logger.error(e.getMessage()); - } + filesOperations.delete(path); } - public String getDocumentNameFromUrl(String docUrl) { + public static String getDocumentNameFromUrl(String docUrl) { return UriUtils.getLastPartFromUri(docUrl); } - private String getIdFromUri(String uri) { - return UriUtils.getLastPartFromUri(uri); - } - private String createFileUrl(String name) throws RmesException { Path gestionStorageFolder=Path.of(config.getDocumentsStorageGestion()); if (!filesOperations.dirExists(gestionStorageFolder)){ @@ -603,7 +594,7 @@ private IRI getDocumentUri(IRI url) throws RmesException { return RdfUtils.toURI(uri.getString(Constants.DOCUMENT)); } - public String getDocumentUrlFromDocument(JSONObject jsonDoc) { + public static String getDocumentUrlFromDocument(JSONObject jsonDoc) { return jsonDoc.getString(Constants.URL).replace(SCHEME_FILE, ""); } @@ -655,7 +646,7 @@ public Document buildDocumentFromJson(JSONObject jsonDoc) { return doc; } - private String getDocumentFilename(String id) throws RmesException { + protected String getDocumentFilename(String id) throws RmesException { JSONObject jsonDoc = getDocument(id, false); String url = getDocumentUrlFromDocument(jsonDoc); return getDocumentNameFromUrl(url); @@ -671,7 +662,7 @@ private String getDocumentFilename(String id) throws RmesException { public ResponseEntity downloadDocumentFile(String id) throws RmesException { String filePath = getDocumentFilename(id); - try (InputStream inputStream = filesOperations.read(filePath)) { // Lire via l'abstraction et utiliser try-with-resources + try (InputStream inputStream = filesOperations.readInDirectoryGestion(filePath)) { // Lire via l'abstraction et utiliser try-with-resources byte[] data = StreamUtils.copyToByteArray(inputStream); // Convertir InputStream en byte[] HttpHeaders headers = new HttpHeaders(); @@ -696,5 +687,13 @@ private String getFileName(String path) { // Extraire juste le nom de fichier du chemin return Paths.get(path).getFileName().toString(); } + + public InputStream retrieveDocumentFromStorage(String filename) { + return filesOperations.readInDirectoryGestion(filename); + } + + public boolean existsInStorage(String filename) { + return filesOperations.existsInStorageGestion(filename); + } } diff --git a/src/main/java/fr/insee/rmes/config/PropertiesLogger.java b/src/main/java/fr/insee/rmes/config/PropertiesLogger.java index 93b717a56..6ee0ef291 100644 --- a/src/main/java/fr/insee/rmes/config/PropertiesLogger.java +++ b/src/main/java/fr/insee/rmes/config/PropertiesLogger.java @@ -21,7 +21,7 @@ public class PropertiesLogger implements ApplicationListener baseMotsCaches = Set.of("password", "pwd", "jeton", "token", "secret", "credential", "pw"); - private static final Set prefixesAffichesParDefaut= Set.of("fr.insee","logging","keycloak","spring","application","server","springdoc","management"); + private static final Set prefixesAffichesParDefaut= Set.of("fr.insee","logging","keycloak","spring","application","server","springdoc","management","minio"); private static final Set propertySourcesIgnoreesParDefaut = Set.of("systemProperties", "systemEnvironment"); private static final PropertySelectorEnum PROPERTY_SELECTOR_PAR_DEFAUT = PropertySelectorEnum.PREFIX; diff --git a/src/main/java/fr/insee/rmes/exceptions/RmesExceptionHandler.java b/src/main/java/fr/insee/rmes/exceptions/RmesExceptionHandler.java index aade896a6..fb757b73e 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RmesExceptionHandler.java +++ b/src/main/java/fr/insee/rmes/exceptions/RmesExceptionHandler.java @@ -32,6 +32,12 @@ public final ResponseEntity handleSubclassesOfRmesException(RmesExceptio return ResponseEntity.status(exception.getStatus()).body(exception.getDetails()); } + @ExceptionHandler(RmesFileException.class) + public final ResponseEntity handleRmesFileException(RmesFileException exception){ + logger.error(exception.getMessage(), exception); + return ResponseEntity.internalServerError().body(exception.toString()); + } + @ExceptionHandler(RmesException.class) public final ResponseEntity handleRmesException(RmesException exception){ logger.error(exception.getMessageAndDetails(), exception); diff --git a/src/main/java/fr/insee/rmes/exceptions/RmesFileException.java b/src/main/java/fr/insee/rmes/exceptions/RmesFileException.java index e36357ecb..c59d6ecca 100644 --- a/src/main/java/fr/insee/rmes/exceptions/RmesFileException.java +++ b/src/main/java/fr/insee/rmes/exceptions/RmesFileException.java @@ -1,7 +1,22 @@ package fr.insee.rmes.exceptions; public class RmesFileException extends RuntimeException { - public RmesFileException(String message, Throwable cause) { + + private final String fileName; + + public RmesFileException(String filename, String message, Throwable cause) { super(message, cause); + this.fileName = filename; + } + + public String getFileName() { + return fileName; + } + + @Override + public String toString() { + return "RmesFileException{" + + "fileName='" + fileName + '\'' + + '}'; } } diff --git a/src/main/java/fr/insee/rmes/utils/RestTemplateUtils.java b/src/main/java/fr/insee/rmes/utils/RestTemplateUtils.java deleted file mode 100644 index b80894d6c..000000000 --- a/src/main/java/fr/insee/rmes/utils/RestTemplateUtils.java +++ /dev/null @@ -1,121 +0,0 @@ -package fr.insee.rmes.utils; - -import fr.insee.rmes.exceptions.RmesException; -import org.apache.commons.codec.binary.Base64; -import org.springframework.core.io.FileSystemResource; -import org.springframework.core.io.Resource; -import org.springframework.http.*; -import org.springframework.stereotype.Service; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.client.RestTemplate; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Service -public class RestTemplateUtils { - - RestTemplate restTemplate = new RestTemplate(); - - public HttpHeaders getHeadersWithBasicAuth(String username, String password) { - // HttpHeaders - HttpHeaders headers = new HttpHeaders(); - - // Authentication - String auth = username + ":" + password; - byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.US_ASCII)); - String authHeader = "Basic " + new String(encodedAuth); - headers.set("Authorization", authHeader); - - return headers; - } - - public HttpHeaders addJsonContentToHeader(HttpHeaders headers) { - addAcceptJsonToHeader(headers); - headers.setContentType(MediaType.APPLICATION_JSON ); - return headers; - } - - - public String getResponseAsString(String target, HttpHeaders headers, Map params) { - if (params == null) return getResponseAsString(target, headers); - // HttpEntity: To get result as String. - HttpEntity entity = new HttpEntity<>(headers); - - // Send request with GET method, and Headers, and Params (can't be null). - ResponseEntity response = restTemplate.exchange(target, HttpMethod.GET, entity, String.class, params); - - return response.getBody(); - } - - public String getResponseAsString(String target, HttpHeaders headers) { - // HttpEntity: To get result as String. - HttpEntity entity = new HttpEntity<>(headers); - - // Send request with GET method, and Headers. - ResponseEntity response = restTemplate.exchange(target, HttpMethod.GET, entity, String.class); - - return response.getBody(); - } - - public void addAcceptJsonToHeader(HttpHeaders headers) { - List acceptHeaders =new ArrayList<>( headers.getAccept()); - acceptHeaders.add(MediaType.APPLICATION_JSON ); - headers.setAccept(acceptHeaders); - } - - private static Resource getFileAsResource(InputStream fileIs, String filename) throws RmesException { - try { - String tempDir = System.getProperty("java.io.tmpdir"); - Path tempFile = Paths.get(tempDir ,filename); - Files.write(tempFile, fileIs.readAllBytes()); - File file = tempFile.toFile(); - file.deleteOnExit(); - return new FileSystemResource(file); - } catch (IOException e) { - throw new RmesException(HttpStatus.INTERNAL_SERVER_ERROR, "Can't convert file to resource IOException", e.getMessage()); - } - - } - - public String postForEntity(String target, MultiValueMap body, HttpHeaders headers) throws RmesException { - - // HttpEntity>: To get result as String. - HttpEntity requestEntity = new HttpEntity<>(body, headers); - - // Send request with POST method, and Headers. - try { - ResponseEntity response = restTemplate.postForEntity(target, requestEntity, String.class); - return response.getBody(); - }catch(Exception e) { - throw new RmesException(HttpStatus.FAILED_DEPENDENCY, "SPOC error, "+ e.getClass(), e.getMessage()); - } - } - - public MultiValueMap buildBodyAsMap(String request, InputStream fileResource, String filename) throws RmesException { - MultiValueMap body = new LinkedMultiValueMap<>(); - - //add file to join to body - HttpHeaders fileheaders = new HttpHeaders(); - fileheaders.setContentType(MediaType.APPLICATION_OCTET_STREAM); - body.add("attachments", new HttpEntity<>(getFileAsResource(fileResource, filename), fileheaders)); - - //add xml content to body - HttpHeaders contentheaders = new HttpHeaders(); - contentheaders.setContentType(MediaType.APPLICATION_XML); - body.add("request", new HttpEntity<>(request, contentheaders)); - - return body; - } - - -} diff --git a/src/test/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExportTest.java b/src/test/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExportTest.java index 3a5171cbc..0661536f7 100644 --- a/src/test/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExportTest.java +++ b/src/test/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExportTest.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.Map; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -71,6 +72,7 @@ void testExportAsZip_success() throws Exception { document.put("id", "1"); when(documentsUtils.getDocumentsUriAndUrlForSims("sims123")).thenReturn(new JSONArray().put(document)); + when(documentsUtils.existsInStorage(any())).thenReturn(false); var sims = new JSONObject(); sims.put("id", "sims123"); sims.put("labelLg1", "simsLabel"); @@ -93,8 +95,8 @@ void testExportAsZip_success() throws Exception { assertEquals(HttpStatus.OK, response.getStatusCode()); assertNotNull(response.getBody()); - assertEquals(response.getStatusCode(), HttpStatus.OK); - assertEquals(response.getHeaders().get("X-Missing-Documents").get(0), "1"); + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); + assertThat(response.getHeaders().get("X-Missing-Documents").getFirst()).isEqualTo("1"); } @Test diff --git a/src/test/java/fr/insee/rmes/integration/TestDocumentsResourcesWithFilesOperation.java b/src/test/java/fr/insee/rmes/integration/TestDocumentsResourcesWithFilesOperation.java new file mode 100644 index 000000000..5db3be9e7 --- /dev/null +++ b/src/test/java/fr/insee/rmes/integration/TestDocumentsResourcesWithFilesOperation.java @@ -0,0 +1,124 @@ +package fr.insee.rmes.integration; + +import fr.insee.rmes.bauhaus_services.FilesOperations; +import fr.insee.rmes.bauhaus_services.operations.documentations.documents.DocumentsImpl; +import fr.insee.rmes.bauhaus_services.operations.documentations.documents.DocumentsUtils; +import fr.insee.rmes.bauhaus_services.rdf_utils.PublicationUtils; +import fr.insee.rmes.bauhaus_services.rdf_utils.RepositoryGestion; +import fr.insee.rmes.bauhaus_services.rdf_utils.RepositoryPublication; +import fr.insee.rmes.bauhaus_services.stamps.StampsRestrictionServiceImpl; +import fr.insee.rmes.config.BaseConfigForMvcTests; +import fr.insee.rmes.config.Config; +import fr.insee.rmes.exceptions.RmesFileException; +import fr.insee.rmes.utils.IdGenerator; +import fr.insee.rmes.webservice.operations.DocumentsResources; +import io.minio.errors.MinioException; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import java.io.InputStream; +import java.nio.file.Path; + +import static fr.insee.rmes.utils.ConsoleCapture.startCapturingConsole; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@WebMvcTest(DocumentsResources.class) +@Import({BaseConfigForMvcTests.class, DocumentsImpl.class, TestDocumentsResourcesWithFilesOperation.ConfigurationForTest.class}) +class TestDocumentsResourcesWithFilesOperation { + + @Autowired + private MockMvc mockMvc; + + @MockBean + private RepositoryPublication repositoryPublication; + @MockBean + private StampsRestrictionServiceImpl stampsRestrictionService; + @MockBean + private IdGenerator idGenerator; + @MockBean + private PublicationUtils publicationUtils; + @MockBean + private Config config; + @MockBean + private RepositoryGestion repositoryGestion; + + private final String fichierId="ID"; + + static final String nomFichier = "nomFichier"; + static final String objectName = "directoryGestion/"+nomFichier; + static final String bucketName = "metadata_bucket"; + + @Test + void shouldLogAndReturnInternalException_WhenErrorOccursInMinio() throws Exception { + var capture=startCapturingConsole(); + mockMvc.perform(MockMvcRequestBuilders.get("/documents/document/" + fichierId + "/file")) + .andExpect(status().isInternalServerError()) + .andExpect(content().string(containsString("fileName='" + nomFichier + "'"))); + assertThat(capture.standardOut()).asString().contains("Error reading file: "+nomFichier+" as object `"+objectName+"` in bucket "+bucketName); + assertThat(capture.standardOut()).asString().contains("at fr.insee.rmes.bauhaus_services.operations.documentations.documents.DocumentsUtils.downloadDocumentFile"); + assertThat(capture.standardOut()).asString().contains("at fr.insee.rmes.bauhaus_services.operations.documentations.documents.DocumentsImpl.downloadDocument"); + capture.stop(); + } + + @TestConfiguration + static class ConfigurationForTest{ + + @Bean + public DocumentsUtils documentsUtils() { + return new DocumentsUtils(null, new FilesOperationStub()){ + @Override + protected String getDocumentFilename(String id){ + return nomFichier; + } + }; + } + + } + + static class FilesOperationStub implements FilesOperations{ + + @Override + public void delete(Path absolutePath) { + + } + + @Override + public InputStream readInDirectoryGestion(String path) { + throw new RmesFileException(nomFichier, "Error reading file: " + nomFichier+ + " as object `"+objectName+"` in bucket "+bucketName, new MinioException()); + } + + @Override + public void writeToDirectoryGestion(InputStream content, Path destPath) { + + } + + @Override + public void copyFromGestionToPublication(String srcPath, String destPath) { + + } + + @Override + public boolean dirExists(Path gestionStorageFolder) { + return false; + } + + @Override + public boolean existsInStorageGestion(String filename) { + return false; + } + } + +} \ No newline at end of file diff --git a/src/test/java/fr/insee/rmes/testcontainers/minio/TestMinioFilesOperation.java b/src/test/java/fr/insee/rmes/testcontainers/minio/TestMinioFilesOperation.java new file mode 100644 index 000000000..dff9b1a27 --- /dev/null +++ b/src/test/java/fr/insee/rmes/testcontainers/minio/TestMinioFilesOperation.java @@ -0,0 +1,74 @@ +package fr.insee.rmes.testcontainers.minio; + +import fr.insee.rmes.bauhaus_services.MinioFilesOperation; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.StatObjectArgs; +import io.minio.errors.*; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MinIOContainer; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import static org.assertj.core.api.Assertions.assertThat; + +class TestMinioFilesOperation { + + MinIOContainer container = new MinIOContainer("minio/minio:RELEASE.2024-11-07T00-52-20Z"); + + @BeforeAll + public static void configureSlf4j() { + System.setProperty("org.slf4j.simpleLogger.log."+MinioFilesOperation.class.getName(), "debug"); + System.setProperty("slf4j.provider", "org.slf4j.simple.SimpleServiceProvider"); + } + + @Test + void testWritingThenCheckExistThenCopyThenRead_shouldBeOK() throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException { + container.start(); + var nomFichier = "test.txt"; + MinioClient minioClient = MinioClient + .builder() + .endpoint(container.getS3URL()) + .credentials(container.getUserName(), container.getPassword()) + .build(); + MinioFilesOperation minioFilesOperation = new MinioFilesOperation(minioClient,"metadata", "gestion", "publication"); + createBucket(minioFilesOperation.bucketName(), minioClient); + + Path absolutePathInGestion = Path.of("/mnt/applishare/rmes/data/storage/documents").resolve(nomFichier); + String contenuFichier = "Test"; + minioFilesOperation.writeToDirectoryGestion(new ByteArrayInputStream(contenuFichier.getBytes()), absolutePathInGestion); + assertThat(minioFilesOperation.dirExists(Path.of(minioFilesOperation.directoryGestion()))).isTrue(); + assertThat(minioFilesOperation.existsInStorageGestion(nomFichier)).isTrue(); + + minioFilesOperation.copyFromGestionToPublication(String.valueOf(absolutePathInGestion), "/mnt/applishare/rmes/data/storage/documents/tempPub1"); + assertThat(minioFilesOperation.dirExists(Path.of(minioFilesOperation.directoryPublication()))).isTrue(); + assertThat(fileExistsInPublication(minioClient, minioFilesOperation, nomFichier) + ).isTrue(); + + assertThat(new String(minioFilesOperation.readInDirectoryGestion(nomFichier).readAllBytes())).isEqualTo(contenuFichier); + } + + @AfterEach + void tearDown() { + container.stop(); + } + + private static boolean fileExistsInPublication(MinioClient minioClient, MinioFilesOperation minioFilesOperation, String nomFichier) throws ErrorResponseException, InsufficientDataException, InternalException, InvalidKeyException, InvalidResponseException, IOException, NoSuchAlgorithmException, ServerException, XmlParserException { + return minioClient.statObject( + StatObjectArgs.builder() + .bucket(minioFilesOperation.bucketName()) + .object(minioFilesOperation.directoryPublication() + "/" + nomFichier).build() + ).size() > 0; + } + + private void createBucket(String bucketName, MinioClient minioClient) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException { + minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build()); + } + +} diff --git a/src/test/java/fr/insee/rmes/utils/ConsoleCapture.java b/src/test/java/fr/insee/rmes/utils/ConsoleCapture.java new file mode 100644 index 000000000..9a4cc23cf --- /dev/null +++ b/src/test/java/fr/insee/rmes/utils/ConsoleCapture.java @@ -0,0 +1,41 @@ +package fr.insee.rmes.utils; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +public class ConsoleCapture{ + + private final ByteArrayOutputStream standardOut; + private final ByteArrayOutputStream standardErr; + private boolean capturing; + private PrintStream lastErr; + private PrintStream lastOut; + + private ConsoleCapture(ByteArrayOutputStream standardOut, ByteArrayOutputStream standardErr) { + this.standardOut = standardOut; + this.standardErr = standardErr; + this.capturing=true; + } + + public static ConsoleCapture startCapturingConsole() { + ConsoleCapture consoleCapture = new ConsoleCapture(new ByteArrayOutputStream(), new ByteArrayOutputStream()); + consoleCapture.lastErr=System.err; + consoleCapture.lastOut=System.out; + System.setOut(new PrintStream(consoleCapture.standardOut)); + System.setErr(new PrintStream(consoleCapture.standardErr)); + return consoleCapture; + } + + public void stop() { + System.setErr(this.lastErr); + System.setOut(this.lastOut); + capturing=false; + } + + public String standardOut() { + if(! capturing) { + throw new IllegalStateException("No capturing any more"); + } + return standardOut.toString(); + } +} \ No newline at end of file diff --git a/src/test/java/fr/insee/rmes/webservice/DocumentsResourcesTest.java b/src/test/java/fr/insee/rmes/webservice/DocumentsResourcesTest.java index 22ef10045..8204cd645 100644 --- a/src/test/java/fr/insee/rmes/webservice/DocumentsResourcesTest.java +++ b/src/test/java/fr/insee/rmes/webservice/DocumentsResourcesTest.java @@ -15,7 +15,6 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.not; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; @@ -40,13 +39,4 @@ void shouldReturnNotFoundException() throws Exception { .andExpect(status().isNotFound()) .andExpect(content().string(containsString("id not found"))); } - - @Test - void shouldReturnInternalException() throws Exception { - when(documentsService.downloadDocument(anyString())).thenThrow(new RmesException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "I/O error", "Error downloading file")); - - mockMvc.perform(MockMvcRequestBuilders.get("/documents/document/nomFichier/file")) - .andExpect(status().isInternalServerError()) - .andExpect(content().string(not(containsString("nomFichier")))); - } }