Skip to content

Commit

Permalink
Hotfix/quick fix minio bugs (#839)
Browse files Browse the repository at this point in the history 
* fix(hotfix no information about minio objects reported in error) WIP

- Modify exception message to log more information
- Adapt exception handler to process these exceptions
- Start writing test

* tests(hotfix always use FileOperations to process documents)

* fix (print debug message with miniofiles)

* pom (new beta version)

* refactor (optimize import in MinioFilesOperation)

* fix (bug with fr.insee.rmes.bauhaus_services.MinioFilesOperation.existsInStorageGestion)

---------

Co-authored-by: Fabrice B <[email protected]>
  • Loading branch information
@FBibonne
FBibonne and Fabrice B authored Dec 20, 2024
1 parent 3769883 commit bd54ea0
Show file tree
Hide file tree
Showing 16 changed files with 372 additions and 188 deletions.
19 changes: 18 additions & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
<groupId>fr.insee.rmes</groupId>
<artifactId>Bauhaus-BO</artifactId>
<packaging>jar</packaging>
<version>4.1.6</version>
<version>4.1.7-beta4</version>
<name>Bauhaus-Back-Office</name>
<description>Back-office services for Bauhaus</description>
<url>https://github.com/InseeFr/Bauhaus-Back-Office</url>
Expand Down Expand Up @@ -87,6 +87,7 @@
<zt.version>1.17</zt.version>
<saxon.version>12.4</saxon.version>
<minio.version>8.5.11</minio.version>
<testcontainers-minio.version>1.20.4</testcontainers-minio.version>


<!-- SONAR -->
Expand All @@ -100,6 +101,7 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>

<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
Expand Down Expand Up @@ -242,15 +244,30 @@
<version>${flexmark.version}</version>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>minio</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>

</dependencies>

<build>
Expand Down
24 changes: 15 additions & 9 deletions src/main/java/fr/insee/rmes/bauhaus_services/FileSystemOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,45 +18,51 @@ public class FileSystemOperation implements FilesOperations {
protected Config config;

@Override
public void delete(String path) {
public void delete(Path absolutePath) {
try {
Files.delete(Paths.get(path));
Files.delete(absolutePath);
} catch (IOException e) {
throw new RmesFileException("Failed to delete file: " + path, e);
throw new RmesFileException(absolutePath.getFileName().toString(), "Failed to delete file: " + absolutePath, e);
}
}

@Override
public InputStream read(String fileName) {
public InputStream readInDirectoryGestion(String fileName) {
try {
return Files.newInputStream(Paths.get(config.getDocumentsStorageGestion()).resolve(fileName));
} catch (IOException e) {
throw new RmesFileException("Failed to read file: " + fileName, e);
throw new RmesFileException(fileName, "Failed to read file: " + fileName, e);
}
}

@Override
public void write(InputStream content, Path destPath) {
public boolean existsInStorageGestion(String filename) {
return Files.exists(Paths.get(config.getDocumentsStorageGestion()).resolve(filename));
}

@Override
public void writeToDirectoryGestion(InputStream content, Path destPath) {
try {
Files.copy(content, destPath, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
throw new RmesFileException("Failed to write file: " + destPath, e);
throw new RmesFileException(destPath.toString(),"Failed to write file: " + destPath, e);
}
}

@Override
public void copy(String srcPath, String destPath) {
public void copyFromGestionToPublication(String srcPath, String destPath) {
Path file = Paths.get(srcPath);
Path targetPath = Paths.get(destPath);
try {
Files.copy(file, targetPath.resolve(file.getFileName()), StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
throw new RmesFileException("Failed to copy file : " + srcPath + " to " + destPath, e);
throw new RmesFileException(srcPath, "Failed to copy file : " + srcPath + " to " + destPath, e);
}
}

@Override
public boolean dirExists(Path gestionStorageFolder) {
return Files.isDirectory(requireNonNull(gestionStorageFolder));
}

}
12 changes: 8 additions & 4 deletions src/main/java/fr/insee/rmes/bauhaus_services/FilesOperations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@
import java.nio.file.Path;

public interface FilesOperations {
void delete(String path);
InputStream read(String path);
void write(InputStream content, Path destPath);
void copy(String srcPath, String destPath);
default void delete(Path absolutePath){
throw new UnsupportedOperationException("Not implemented yet.");
}
InputStream readInDirectoryGestion(String filename);
void writeToDirectoryGestion(InputStream content, Path destPath);
void copyFromGestionToPublication(String srcPath, String destPath);

boolean dirExists(Path gestionStorageFolder);

boolean existsInStorageGestion(String filename);
}
57 changes: 43 additions & 14 deletions src/main/java/fr/insee/rmes/bauhaus_services/MinioFilesOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,25 @@
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;

import static java.util.Objects.requireNonNull;

public record MinioFilesOperation(MinioClient minioClient, String bucketName, String directoryGestion, String directoryPublication) implements FilesOperations {

private static final Logger logger = LoggerFactory.getLogger(MinioFilesOperation.class);
static final Logger logger = LoggerFactory.getLogger(MinioFilesOperation.class);

@Override
public InputStream read(String pathFile){
String objectName= extractFileName(pathFile);
public InputStream readInDirectoryGestion(String filename){
String objectName = directoryGestion + "/" + filename;

logger.debug("Reading file with name {} from path {} as object {} in bucket {}", filename, filename, objectName, bucketName);

try {
return minioClient.getObject(GetObjectArgs.builder()
.bucket(bucketName)
.object(directoryGestion +"/"+ objectName)
.object(objectName)
.build());
} catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
throw new RmesFileException("Error reading file: " + objectName, e);
throw new RmesFileException(filename, "Error reading file: " + filename+" as object `"+objectName+"` in bucket "+bucketName, e);
}
}
private static String extractFileName(String filePath) {
Expand All @@ -36,38 +40,59 @@ private static String extractFileName(String filePath) {
return Path.of(filePath).getFileName().toString();
}

@Override
public boolean existsInStorageGestion(String filename) {
String objectName = directoryGestion + "/" + filename;
logger.debug("Check existence of file with name {} as object {} in bucket {}", filename, objectName, bucketName);
try {
return minioClient.statObject(StatObjectArgs.builder()
.bucket(bucketName)
.object(objectName)
.build()).size() > 0;
} catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
return false;
}
}

@Override
public void write(InputStream content, Path objectName) {
public void writeToDirectoryGestion(InputStream content, Path filePath) {
String filename = filePath.getFileName().toString();
String objectName = directoryGestion + "/" + filename;
logger.debug("Writing to file with name {} from path {} as object {} in bucket {}", filename, filePath, objectName, bucketName);
try {
minioClient.putObject(PutObjectArgs.builder()
.bucket(bucketName)
.object(directoryGestion +"/"+ objectName.getFileName().toString())
.object(objectName)
.stream(content, content.available(), -1)
.build());
} catch (IOException | ErrorResponseException | InsufficientDataException | InternalException |
InvalidKeyException | InvalidResponseException | NoSuchAlgorithmException | ServerException |
XmlParserException e) {
throw new RmesFileException("Error writing file: " + objectName, e);
throw new RmesFileException(filePath.toString(), "Error writing file: " + filename+ "as object `"+objectName+"` in bucket "+bucketName, e);
}
}

@Override
public void copy(String srcObjectName, String destObjectName) {
public void copyFromGestionToPublication(String srcObjectName, String destObjectName) {

String srcObject = directoryGestion + "/" + extractFileName(srcObjectName);
String destObject = directoryPublication + "/" + extractFileName(srcObjectName);

logger.debug("Copy from source {} as object {} to destination {} as object {} in bucket {}", srcObjectName, srcObject, destObjectName, destObject, bucketName);

try {
CopySource source = CopySource.builder()
.bucket(bucketName)
.object(directoryGestion +"/"+ extractFileName(srcObjectName))
.object(srcObject)
.build();

minioClient.copyObject(CopyObjectArgs.builder()
.bucket(bucketName)
.object(directoryPublication +"/"+ extractFileName(srcObjectName))
.object(destObject)
.source(source)
.build());
} catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
throw new RmesFileException("Error copying file from " + srcObjectName + " to " + destObjectName, e);
throw new RmesFileException(srcObjectName,"Error copying file from `" + srcObject + "` to `" + destObject+"` in bucket "+bucketName, e);
}
}

Expand All @@ -78,14 +103,18 @@ public boolean dirExists(Path gestionStorageFolder) {
}

@Override
public void delete(String objectName) {
public void delete(Path absolutePath) {
String objectName = absolutePath.getFileName().toString();

logger.debug("Delete file with path {} as object {} in bucket {}", absolutePath, objectName, bucketName);

try {
minioClient.removeObject(RemoveObjectArgs.builder()
.bucket(bucketName)
.object(objectName)
.build());
} catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) {
throw new RmesFileException("Error deleting file: " + objectName, e);
throw new RmesFileException(objectName,"Error deleting file: " + objectName+" in bucket "+bucketName, e);
}
}

Expand Down
8 changes: 4 additions & 4 deletions ...in/java/fr/insee/rmes/bauhaus_services/operations/documentations/DocumentationExport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,19 +160,19 @@ private Set<String> exportRubricsDocuments(JSONObject sims, Path directory) thro

for (int i = 0; i < documents.length(); i++) {
JSONObject document = documents.getJSONObject(i);
String url = document.getString("url").replace("file://", "");
String url = DocumentsUtils.getDocumentUrlFromDocument(document);
if(!history.contains(url)){
history.add(url);
logger.debug("Extracting document {}", url);


Path documentPath = Path.of(url);
String documentFilename = DocumentsUtils.getDocumentNameFromUrl(url);

if(!Files.exists(documentPath)){
if(!documentsUtils.existsInStorage(documentFilename)){
missingDocuments.add(document.getString("id"));
} else {
String documentFileName = FilesUtils.generateFinalFileNameWithExtension(UriUtils.getLastPartFromUri(url), maxLength);
try (InputStream inputStream = Files.newInputStream(documentPath)){
try (InputStream inputStream = documentsUtils.retrieveDocumentFromStorage(documentFilename)){
Path documentDirectory = Path.of(directory.toString(), "documents");
if (!Files.exists(documentDirectory)) {
logger.debug("Creating the documents folder");
Expand Down
8 changes: 3 additions & 5 deletions ...insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsPublication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,12 @@ public void publishAllDocumentsInSims(String idSims) throws RmesException {
JSONArray listDoc = docUtils.getListDocumentSims(idSims);

Map<Integer,String> mapIdUrls = new HashMap<>();
listDoc.forEach(doc -> mapIdUrls.put(docUtils.getIdFromJson((JSONObject) doc), docUtils.getDocumentUrlFromDocument((JSONObject) doc)));
listDoc.forEach(doc -> mapIdUrls.put(docUtils.getIdFromJson((JSONObject) doc), DocumentsUtils.getDocumentUrlFromDocument((JSONObject) doc)));

for (Map.Entry<Integer, String> doc : mapIdUrls.entrySet()) {
String docId = doc.getKey().toString();
String originalPath = doc.getValue();
String filename = docUtils.getDocumentNameFromUrl(originalPath);
String filename = DocumentsUtils.getDocumentNameFromUrl(originalPath);
// Publish the physical files
copyFileInPublicationFolders(originalPath);
// Change url in document (getModelToPublish) and publish the RDF
Expand All @@ -68,10 +68,8 @@ public void publishAllDocumentsInSims(String idSims) throws RmesException {
}

private void copyFileInPublicationFolders(String originalPath){
String documentsStoragePublicationInterne = config.getDocumentsStoragePublicationInterne();
String documentsStoragePublicationExterne = config.getDocumentsStoragePublicationExterne();
filesOperations.copy(originalPath, documentsStoragePublicationInterne);
filesOperations.copy(originalPath, documentsStoragePublicationExterne);
filesOperations.copyFromGestionToPublication(originalPath, documentsStoragePublicationExterne);
}


Expand Down
31 changes: 15 additions & 16 deletions ...va/fr/insee/rmes/bauhaus_services/operations/documentations/documents/DocumentsUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
Expand Down Expand Up @@ -282,7 +281,7 @@ private void checkUrlDoesNotExist(String id, String url, int errorCode, String e
JSONObject existingUriJson = repoGestion.getResponseAsObject(DocumentsQueries.getDocumentUriQuery(url));
if (existingUriJson.length() > 0) {
String uri = existingUriJson.getString(Constants.DOCUMENT);
String existingId = getIdFromUri(uri);
String existingId = getDocumentNameFromUrl(uri);
if (!existingId.equals(id)) {
throw new RmesNotAcceptableException(errorCode, errorMessage, uri);
}
Expand Down Expand Up @@ -497,7 +496,7 @@ private void uploadFile(InputStream documentFile, String documentName, String ur
throw new RmesBadRequestException(ErrorCodes.DOCUMENT_CREATION_EXISTING_FILE,
"There is already a document with that name.", documentName);
}
filesOperations.write(documentFile, path);
filesOperations.writeToDirectoryGestion(documentFile, path);
// don't throw an error if a file already exists under this name
}

Expand Down Expand Up @@ -556,21 +555,13 @@ private void changeDocumentsURL(String iri, String docUrl, String newUrl) throws

private void deleteFile(String docUrl) {
Path path = Paths.get(docUrl);
try {
Files.delete(path);
} catch (IOException e) {
logger.error(e.getMessage());
}
filesOperations.delete(path);
}

public String getDocumentNameFromUrl(String docUrl) {
public static String getDocumentNameFromUrl(String docUrl) {
return UriUtils.getLastPartFromUri(docUrl);
}

private String getIdFromUri(String uri) {
return UriUtils.getLastPartFromUri(uri);
}

private String createFileUrl(String name) throws RmesException {
Path gestionStorageFolder=Path.of(config.getDocumentsStorageGestion());
if (!filesOperations.dirExists(gestionStorageFolder)){
Expand Down Expand Up @@ -603,7 +594,7 @@ private IRI getDocumentUri(IRI url) throws RmesException {
return RdfUtils.toURI(uri.getString(Constants.DOCUMENT));
}

public String getDocumentUrlFromDocument(JSONObject jsonDoc) {
public static String getDocumentUrlFromDocument(JSONObject jsonDoc) {
return jsonDoc.getString(Constants.URL).replace(SCHEME_FILE, "");
}

Expand Down Expand Up @@ -655,7 +646,7 @@ public Document buildDocumentFromJson(JSONObject jsonDoc) {
return doc;
}

private String getDocumentFilename(String id) throws RmesException {
protected String getDocumentFilename(String id) throws RmesException {
JSONObject jsonDoc = getDocument(id, false);
String url = getDocumentUrlFromDocument(jsonDoc);
return getDocumentNameFromUrl(url);
Expand All @@ -671,7 +662,7 @@ private String getDocumentFilename(String id) throws RmesException {
public ResponseEntity<org.springframework.core.io.Resource> downloadDocumentFile(String id) throws RmesException {
String filePath = getDocumentFilename(id);

try (InputStream inputStream = filesOperations.read(filePath)) { // Lire via l'abstraction et utiliser try-with-resources
try (InputStream inputStream = filesOperations.readInDirectoryGestion(filePath)) { // Lire via l'abstraction et utiliser try-with-resources
byte[] data = StreamUtils.copyToByteArray(inputStream); // Convertir InputStream en byte[]

HttpHeaders headers = new HttpHeaders();
Expand All @@ -696,5 +687,13 @@ private String getFileName(String path) {
// Extraire juste le nom de fichier du chemin
return Paths.get(path).getFileName().toString();
}

public InputStream retrieveDocumentFromStorage(String filename) {
return filesOperations.readInDirectoryGestion(filename);
}

public boolean existsInStorage(String filename) {
return filesOperations.existsInStorageGestion(filename);
}
}

Loading

0 comments on commit bd54ea0

Please sign in to comment.