From 878c72c7b92019cb63042e9501f738461675cb7e Mon Sep 17 00:00:00 2001 From: Marcel Hekman Date: Thu, 27 Jun 2019 11:49:08 +0200 Subject: [PATCH 1/3] Implement copying directories. --- .../storage/s3fs/S3FileSystemProvider.java | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java index 3f81b341..9a0f5bb3 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java @@ -855,35 +855,42 @@ public void copy(Path source, S3Path s3Source = toS3Path(source); S3Path s3Target = toS3Path(target); - // TODO: implements support for copying directories - - Preconditions.checkArgument(!Files.isDirectory(source), "copying directories is not yet supported: %s", source); - Preconditions.checkArgument(!Files.isDirectory(target), "copying directories is not yet supported: %s", target); ImmutableSet actualOptions = ImmutableSet.copyOf(options); verifySupportedOptions(EnumSet.of(StandardCopyOption.REPLACE_EXISTING), actualOptions); - if (exists(s3Target) && !actualOptions.contains(StandardCopyOption.REPLACE_EXISTING)) + if (exists(s3Target)) { - throw new FileAlreadyExistsException(format("target already exists: %s", target)); - } - - String bucketNameOrigin = s3Source.getFileStore().name(); - String keySource = s3Source.getKey(); - String bucketNameTarget = s3Target.getFileStore().name(); - String keyTarget = s3Target.getKey(); - final S3Client client = s3Source.getFileSystem().getClient(); + if (!actualOptions.contains(StandardCopyOption.REPLACE_EXISTING)) + throw new FileAlreadyExistsException(format("target already exists: %s", target)); - final String encodedUrl = encodeUrl(bucketNameOrigin, keySource); - - final CopyObjectRequest request = CopyObjectRequest.builder() - .copySource(encodedUrl) - .cacheControl(s3Target.getFileSystem().getRequestHeaderCacheControlProperty()) - .destinationBucket(bucketNameTarget) - .destinationKey(keyTarget) - .build(); + if (Files.isDirectory(source)) + delete(s3Target); + } - client.copyObject(request); + if (Files.isDirectory(source)) + { + createDirectory(s3Target); + } + else + { + String bucketNameOrigin = s3Source.getFileStore().name(); + String keySource = s3Source.getKey(); + String bucketNameTarget = s3Target.getFileStore().name(); + String keyTarget = s3Target.getKey(); + final S3Client client = s3Source.getFileSystem().getClient(); + + final String encodedUrl = encodeUrl(bucketNameOrigin, keySource); + + final CopyObjectRequest request = CopyObjectRequest.builder() + .copySource(encodedUrl) + .cacheControl(s3Target.getFileSystem().getRequestHeaderCacheControlProperty()) + .destinationBucket(bucketNameTarget) + .destinationKey(keyTarget) + .build(); + + client.copyObject(request); + } } private String encodeUrl(final String bucketNameOrigin, From 498e809c4c00939ca89421f6eb82c64a15782093 Mon Sep 17 00:00:00 2001 From: Enno Ruijters Date: Wed, 29 Sep 2021 11:00:34 +0200 Subject: [PATCH 2/3] Add unit test --- .../s3fs/fileSystemProvider/CopyTest.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CopyTest.java b/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CopyTest.java index fa6520f2..b78d5d64 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CopyTest.java +++ b/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CopyTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -126,6 +127,37 @@ void copyAlreadyExists() assertNotNull(exception); } + @Test + public void copyDirectory() throws IOException + { + final String content = "sample-content"; + + // fixtures + S3ClientMock client = S3MockFactory.getS3ClientMock(); + client.bucket("bucketA").dir("dir1").file("dir1/file", content.getBytes()); + FileSystem fs = createNewS3FileSystem(); + Path dir1 = fs.getPath("/bucketA", "dir1"); + Path file1 = fs.getPath("/bucketA", "dir1", "file"); + Path dir2 = fs.getPath("/bucketA", "dir2"); + Path file2 = fs.getPath("/bucketA", "dir2", "file"); + + // assert + assertTrue(Files.exists(dir1)); + assertTrue(Files.exists(file1)); + assertTrue(Files.isDirectory(dir1)); + assertTrue(Files.isRegularFile(file1)); + assertFalse(Files.exists(dir2)); + assertFalse(Files.exists(file2)); + + // act + s3fsProvider.copy(dir1, dir2); + + // assertions + assertTrue(Files.exists(dir2)); + assertTrue(Files.isDirectory(dir2)); + assertFalse(Files.exists(file2)); + } + /** * create a new file system for s3 scheme with fake credentials * and global endpoint From 5d928725f455f8b4f71e0f3c912abedea1a81b2a Mon Sep 17 00:00:00 2001 From: Martin Todorov Date: Mon, 4 Oct 2021 23:54:09 +0100 Subject: [PATCH 3/3] Code style fixes --- .../carlspring/cloud/storage/s3fs/S3FileSystemProvider.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java index 9a0f5bb3..481fc79f 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java @@ -862,10 +862,14 @@ public void copy(Path source, if (exists(s3Target)) { if (!actualOptions.contains(StandardCopyOption.REPLACE_EXISTING)) + { throw new FileAlreadyExistsException(format("target already exists: %s", target)); + } if (Files.isDirectory(source)) + { delete(s3Target); + } } if (Files.isDirectory(source)) @@ -906,6 +910,7 @@ private String encodeUrl(final String bucketNameOrigin, { throw new UnsupportedEncodingException("URL could not be encoded: " + e.getMessage()); } + return encodedUrl; }