Set constant directory modification times

RIPE-NCC · Dec 8, 2023 · aa3785f · aa3785f
1 parent a8ea871
commit aa3785f
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 14 deletions.
diff --git a/src/main/java/net/ripe/rpki/rsyncit/rsync/RsyncWriter.java b/src/main/java/net/ripe/rpki/rsyncit/rsync/RsyncWriter.java
@@ -20,7 +20,9 @@
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ForkJoinPool;
+import java.util.function.Consumer;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -101,10 +103,7 @@ private Path writeObjectToNewDirectory(List<RpkiObject> objects, Instant now) th
                         .forEach(dir -> {
                             try {
                                 Files.createDirectories(dir);
-                                Files.setPosixFilePermissions(dir, DIRECTORY_PERMISSIONS);
-                                Files.setLastModifiedTime(dir, INTERNAL_DIRECTORY_LAST_MODIFIED_TIME);
                             } catch (IOException e) {
-                                log.error("Could not create directory {}", dir, e);
                                 throw new UncheckedIOException(e);
                             }
                         })
@@ -121,9 +120,27 @@ private Path writeObjectToNewDirectory(List<RpkiObject> objects, Instant now) th
                     }
                 })).join();
 
-                log.info("Wrote {} directories ({} ms) and {} files ({} ms) for host {}",
-                        targetDirectories.size(), t1 - t0,
-                        writableContent.size(), System.currentTimeMillis() - t1,
+                var t2 = System.currentTimeMillis();
+                // Set permissions and modification time on directories
+                //
+                // directory modification times are updated when files are written to them (!)
+                // so update at the end.
+                try (Stream<Path> paths = Files.walk(hostDirectory)) {
+                    fileWriterPool.submit(() -> paths.parallel().filter(Files::isDirectory).forEach(dir -> {
+                        try {
+                            Files.setPosixFilePermissions(dir, DIRECTORY_PERMISSIONS);
+                            Files.setLastModifiedTime(dir, INTERNAL_DIRECTORY_LAST_MODIFIED_TIME);
+                        } catch (IOException e) {
+                            throw new UncheckedIOException(e);
+                        }
+                    })).join();
+                } catch (IOException e) {
+                    log.error("Could not walk directory {}", hostDirectory, e);
+                    throw new UncheckedIOException(e);
+                }
+
+                log.info("Wrote {} directories ({} ms, mtime/chmod {}ms) and {} files ({} ms) for host {}",
+                        targetDirectories.size(), t1 - t0, t2 - t1, writableContent.size(), System.currentTimeMillis() - t2,
                         hostName);
             });
 
@@ -198,7 +215,7 @@ void cleanupOldTargetDirectories(Instant now, Path baseDirectory) throws IOExcep
                 .filter(Files::isDirectory)
                 .sorted(Comparator.comparing(this::getLastModifiedTime).reversed())
                 .skip(config.targetDirectoryRetentionCopiesCount())
-                .filter((directory) -> getLastModifiedTime(directory).toMillis() < cutoff)
+                .filter(directory -> getLastModifiedTime(directory).toMillis() < cutoff)
                 .filter(dir -> {
                     try {
                         return !dir.toRealPath().equals(actualPublishedDir);
@@ -218,6 +235,18 @@ void cleanupOldTargetDirectories(Instant now, Path baseDirectory) throws IOExcep
         }
     }
 
+    public interface IOExceptionThrowingCallable<T> {
+        T call() throws IOException;
+    }
+
+    static <T> T withUncheckedIOException(IOExceptionThrowingCallable<T> callable) {
+        try {
+            return callable.call();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
     private FileTime getLastModifiedTime(Path path) {
         try {
             return Files.getLastModifiedTime(path);

diff --git a/src/test/java/net/ripe/rpki/rsyncit/rsync/RsyncWriterTest.java b/src/test/java/net/ripe/rpki/rsyncit/rsync/RsyncWriterTest.java
@@ -1,16 +1,15 @@
 package net.ripe.rpki.rsyncit.rsync;
 
+import lombok.extern.slf4j.Slf4j;
 import net.ripe.rpki.rsyncit.config.Config;
 import net.ripe.rpki.rsyncit.rrdp.RpkiObject;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.nio.file.attribute.FileTime;
 import java.time.Instant;
 import java.time.ZoneId;
@@ -19,17 +18,15 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Random;
-import java.util.function.BiFunction;
-import java.util.function.Consumer;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static net.ripe.rpki.TestDefaults.defaultConfig;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.assertj.core.api.Assertions.*;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+@Slf4j
 class RsyncWriterTest {
 
     @Test
@@ -86,6 +83,28 @@ public void testWriteMultipleObjects(@TempDir Path tmpPath) throws Exception {
         });
     }
 
+    @Test
+    public void testWrite_set_time_and_permissions_on_empty_intermediate_paths(@TempDir Path tmpPath) throws Exception {
+        withRsyncWriter(tmpPath, rsyncWriter -> {
+            var o1 = new RpkiObject(URI.create("rsync://bla.net/path1/a.cer"), someBytes(), Instant.now());
+            var o2 = new RpkiObject(URI.create("rsync://bla.net/path1/nested/empty/dir/tree/c.cer"), someBytes(), Instant.now());
+            var targetDir = rsyncWriter.writeObjects(Arrays.asList(o1, o2), Instant.now());
+
+            var dirCount = new AtomicInteger();
+
+            Files.walk(targetDir).filter(path -> !path.equals(targetDir) && path.toFile().isDirectory()).forEach(path -> {
+                dirCount.incrementAndGet();
+                assertThatCode(() -> {
+                    assertThat(Files.getLastModifiedTime(path)).isEqualTo(RsyncWriter.INTERNAL_DIRECTORY_LAST_MODIFIED_TIME);
+                    assertThat(Files.getPosixFilePermissions(path)).isEqualTo(RsyncWriter.DIRECTORY_PERMISSIONS);
+                })
+                        .doesNotThrowAnyException();
+            });
+
+            assertThat(dirCount.get()).isGreaterThan(5);
+        });
+    }
+
     @Test
     public void testIgnoreBadUrls(@TempDir Path tmpPath) throws Exception {
         withRsyncWriter(tmpPath, rsyncWriter -> {