From e065d2c78ff14ce5b4824a69454fe37ba762dfa0 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Thu, 21 Jul 2022 17:32:09 -0500 Subject: [PATCH 01/12] Summarize the cache in a simple text file --- .../j2cl/build/DefaultDiskCache.java | 29 +++--------- .../com/vertispan/j2cl/build/DiskCache.java | 47 +++++++++++++++++-- .../java/com/vertispan/j2cl/build/Input.java | 9 ++++ .../j2cl/build/PropertyTrackingConfig.java | 2 +- 4 files changed, 60 insertions(+), 27 deletions(-) diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java index 20ffb948..f310371d 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java @@ -29,28 +29,8 @@ public DefaultDiskCache(File cacheDir, Executor executor) throws IOException { } @Override - protected Path taskDir(CollectedTaskInputs inputs) { - String projectName = inputs.getProject().getKey(); - Murmur3F hash = new Murmur3F(); - - hash.update(inputs.getTaskFactory().getClass().toString().getBytes(StandardCharsets.UTF_8)); - hash.update(inputs.getTaskFactory().getTaskName().getBytes(StandardCharsets.UTF_8)); - hash.update(inputs.getTaskFactory().getVersion().getBytes(StandardCharsets.UTF_8)); - - for (Input input : inputs.getInputs()) { - input.updateHash(hash); - } - - for (Map.Entry entry : inputs.getUsedConfigs().entrySet()) { - hash.update(entry.getKey().getBytes(StandardCharsets.UTF_8)); - if (entry.getValue() == null) { - hash.update(0); - } else { - hash.update(entry.getValue().getBytes(StandardCharsets.UTF_8)); - } - } - - return cacheDir.toPath().resolve(projectName.replaceAll("[^\\-_a-zA-Z0-9.]", "-")).resolve(hash.getValueHexString() + "-" + inputs.getTaskFactory().getOutputType()); + protected Path taskDir(String projectName, String hashString, String outputType) { + return cacheDir.toPath().resolve(projectName.replaceAll("[^\\-_a-zA-Z0-9.]", "-")).resolve(hashString + "-" + outputType); } @Override @@ -72,4 +52,9 @@ protected Path logFile(Path taskDir) { protected Path outputDir(Path taskDir) { return taskDir.resolve("results"); } + + @Override + protected Path cacheSummary(Path taskDir) { + return taskDir.resolve("cacheSummary"); + } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java index f60cc46b..c8bd9975 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java @@ -5,19 +5,19 @@ import io.methvin.watcher.PathUtils; import io.methvin.watcher.hashing.FileHash; import io.methvin.watcher.hashing.FileHasher; +import io.methvin.watcher.hashing.Murmur3F; import io.methvin.watchservice.MacOSXListeningWatchService; import io.methvin.watchservice.WatchablePath; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.nio.file.WatchService; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; import java.time.Instant; -import java.time.temporal.TemporalAmount; -import java.time.temporal.TemporalUnit; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -340,11 +340,42 @@ private void deleteRecursively(Path path) throws IOException { } } - protected abstract Path taskDir(CollectedTaskInputs inputs); + protected String taskSummaryContents(CollectedTaskInputs inputs) { + StringBuilder sb = new StringBuilder(); + sb.append("Cache summary for "); + String projectName = inputs.getProject().getKey(); + sb.append(projectName); + + sb.append("\nTask name ").append(inputs.getTaskFactory().getTaskName()); + sb.append("\nImplemented by ").append(inputs.getTaskFactory().getClass().toString()); + sb.append(" version ").append(inputs.getTaskFactory().getVersion()); + + sb.append("\n\nInputs:"); + for (Input input : inputs.getInputs()) { + input.updateHashSummary(sb); + } + + + sb.append("\n\nConfigs:"); + for (Map.Entry entry : inputs.getUsedConfigs().entrySet()) { + sb.append("\n\t").append(entry.getKey()).append(" = "); + if (entry.getValue() == null) { + sb.append("null"); + } else { + sb.append("\"").append(entry.getValue()).append("\""); + } + } + + return sb.toString(); + } + + protected abstract Path taskDir(String projectName, String hashString, String outputType); + protected abstract Path successMarker(Path taskDir); protected abstract Path failureMarker(Path taskDir); protected abstract Path logFile(Path taskDir); protected abstract Path outputDir(Path taskDir); + protected abstract Path cacheSummary(Path taskDir); interface Listener { void onReady(CacheResult result); @@ -431,7 +462,14 @@ private synchronized void failure() { */ public void waitForTask(CollectedTaskInputs taskDetails, Listener listener) { assert taskDetails.getInputs().stream().allMatch(Input::hasContents); - final Path taskDir = taskDir(taskDetails); + + Murmur3F murmur3F = new Murmur3F(); + byte[] taskSummaryContents = taskSummaryContents(taskDetails).getBytes(StandardCharsets.UTF_8); + murmur3F.update(taskSummaryContents); + String hashString = murmur3F.getValueHexString(); + + final Path taskDir = taskDir(taskDetails.getProject().getKey(), hashString, taskDetails.getTaskFactory().getOutputType()); + PendingCacheResult cancelable = new PendingCacheResult(taskDir, listener); taskFutures.computeIfAbsent(taskDir, ignore -> Collections.newSetFromMap(new ConcurrentHashMap<>())).add(cancelable); try { @@ -449,6 +487,7 @@ public void waitForTask(CollectedTaskInputs taskDetails, Listener listener) { // caller can begin work right away Files.createDirectory(outputDir); Files.createFile(logFile(taskDir)); + Files.write(cacheSummary(taskDir), taskSummaryContents); cancelable.ready(); return; } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java index 94e67eb9..ab4921b0 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java @@ -114,6 +114,15 @@ public void updateHash(Murmur3F hash) { } } + public void updateHashSummary(StringBuilder stringBuilder) { + stringBuilder.append("\n\t").append(getProject().getKey()).append(" ").append(getOutputType()); + for (DiskCache.CacheEntry fileAndHash : getFilesAndHashes()) { + stringBuilder.append("\n\t\t").append(fileAndHash.getSourcePath().toString()); + stringBuilder.append(" with hash ").append(fileAndHash.getHash().asString()); + } + } + + @Override public Project getProject() { return project; diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java b/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java index 7f9b061c..3f25034c 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java @@ -72,7 +72,7 @@ private File useFileConfig(ConfigValueProvider.ConfigNode node) { } catch (IOException e) { throw new UncheckedIOException("Failed to hash file contents " + value, e); } - useKey(node.getPath(), hash); + useKey(node.getPath(), "File with hash " + hash); return value; } private void useKey(String path, String value) { From 8d771195a35a65778675af886bebd673d0a49946 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 16 Nov 2022 11:10:28 -0600 Subject: [PATCH 02/12] suffix for the summary file --- .../main/java/com/vertispan/j2cl/build/DefaultDiskCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java index f310371d..070470a8 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java @@ -55,6 +55,6 @@ protected Path outputDir(Path taskDir) { @Override protected Path cacheSummary(Path taskDir) { - return taskDir.resolve("cacheSummary"); + return taskDir.resolve("cacheSummary.txt"); } } From 323f67e4610f4ec29a1396a80dcbe257c5031520 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 21 Dec 2022 15:10:13 -0600 Subject: [PATCH 03/12] Apparently-working impl of incremental task tracking Includes an incremental implementation of gwtincompatible stripping and closure bundling. --- build-caching/pom.xml | 7 + .../j2cl/build/ChangedCachedPath.java | 34 ++ .../j2cl/build/DefaultDiskCache.java | 2 +- .../com/vertispan/j2cl/build/DiskCache.java | 86 +++-- .../java/com/vertispan/j2cl/build/Input.java | 70 +++- .../j2cl/build/LocalProjectBuildCache.java | 62 ++++ .../j2cl/build/PropertyTrackingConfig.java | 7 + .../vertispan/j2cl/build/TaskScheduler.java | 83 ++++- .../j2cl/build/TaskSummaryDiskFormat.java | 93 +++++ .../com/vertispan/j2cl/build/task/Config.java | 13 + .../com/vertispan/j2cl/build/task/Input.java | 16 + .../j2cl/build/task/TaskContext.java | 15 +- .../j2cl/mojo/AbstractBuildMojo.java | 3 + .../j2cl/mojo/AbstractCacheMojo.java | 3 + .../com/vertispan/j2cl/mojo/BuildMojo.java | 3 +- .../com/vertispan/j2cl/mojo/TestMojo.java | 3 +- .../com/vertispan/j2cl/mojo/WatchMojo.java | 3 +- .../build/provided/ClosureBundleTask.java | 336 ++++++++++++++++-- .../j2cl/build/provided/J2clTask.java | 3 + .../j2cl/build/provided/StripSourcesTask.java | 46 ++- 20 files changed, 800 insertions(+), 88 deletions(-) create mode 100644 build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java create mode 100644 build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java create mode 100644 build-caching/src/main/java/com/vertispan/j2cl/build/TaskSummaryDiskFormat.java diff --git a/build-caching/pom.xml b/build-caching/pom.xml index a6dc4bcc..3ea147dc 100644 --- a/build-caching/pom.xml +++ b/build-caching/pom.xml @@ -23,6 +23,13 @@ 0.15.0 + + + com.google.code.gson + gson + 2.8.6 + + junit diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java b/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java new file mode 100644 index 00000000..7047b064 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java @@ -0,0 +1,34 @@ +package com.vertispan.j2cl.build; + +import com.vertispan.j2cl.build.task.CachedPath; +import com.vertispan.j2cl.build.task.Input; + +import java.nio.file.Path; +import java.util.Optional; + +public class ChangedCachedPath implements Input.ChangedCachedPath { + private final ChangeType type; + private final Path sourcePath; + private final Optional newIfAny; + + public ChangedCachedPath(ChangeType type, Path sourcePath, CachedPath newPath) { + this.type = type; + this.sourcePath = sourcePath; + this.newIfAny = Optional.ofNullable(newPath); + } + + @Override + public ChangeType changeType() { + return type; + } + + @Override + public Path getSourcePath() { + return sourcePath; + } + + @Override + public Optional getNewAbsolutePath() { + return newIfAny.map(CachedPath::getAbsolutePath); + } +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java index 070470a8..8afd270b 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java @@ -55,6 +55,6 @@ protected Path outputDir(Path taskDir) { @Override protected Path cacheSummary(Path taskDir) { - return taskDir.resolve("cacheSummary.txt"); + return taskDir.resolve("cacheSummary.json"); } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java index c8bd9975..fd0c855f 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java @@ -1,5 +1,7 @@ package com.vertispan.j2cl.build; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.vertispan.j2cl.build.impl.CollectedTaskInputs; import com.vertispan.j2cl.build.task.CachedPath; import io.methvin.watcher.PathUtils; @@ -22,9 +24,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; +import java.util.stream.Collectors; /** - * Manages the cached task inputs and outputs. + * Manages the cached task inputs and outputs, without direct knowledge of the project or task apis. */ public abstract class DiskCache { private static final boolean IS_MAC = System.getProperty("os.name").toLowerCase().contains("mac"); @@ -38,6 +41,10 @@ public CacheResult(Path taskDir) { this.taskDir = taskDir; } + public Path taskDir() { + return taskDir; + } + public Path logFile() { //TODO finish building a logger that will write to this return DiskCache.this.logFile(taskDir); @@ -54,6 +61,10 @@ public TaskOutput output() { return taskOutput; } + public Path cachedSummary() { + return DiskCache.this.cacheSummary(taskDir); + } + public void markSuccess() { markFinished(this); runningTasks.remove(taskDir); @@ -340,33 +351,39 @@ private void deleteRecursively(Path path) throws IOException { } } - protected String taskSummaryContents(CollectedTaskInputs inputs) { - StringBuilder sb = new StringBuilder(); - sb.append("Cache summary for "); - String projectName = inputs.getProject().getKey(); - sb.append(projectName); - - sb.append("\nTask name ").append(inputs.getTaskFactory().getTaskName()); - sb.append("\nImplemented by ").append(inputs.getTaskFactory().getClass().toString()); - sb.append(" version ").append(inputs.getTaskFactory().getVersion()); - - sb.append("\n\nInputs:"); - for (Input input : inputs.getInputs()) { - input.updateHashSummary(sb); - } - - - sb.append("\n\nConfigs:"); - for (Map.Entry entry : inputs.getUsedConfigs().entrySet()) { - sb.append("\n\t").append(entry.getKey()).append(" = "); - if (entry.getValue() == null) { - sb.append("null"); - } else { - sb.append("\"").append(entry.getValue()).append("\""); - } - } - - return sb.toString(); + private String taskSummaryContents(CollectedTaskInputs inputs) { + TaskSummaryDiskFormat src = new TaskSummaryDiskFormat(); + src.setProjectKey(inputs.getProject().getKey()); + src.setOutputType(inputs.getTaskFactory().getOutputType()); + src.setTaskImpl(inputs.getTaskFactory().getClass().getName()); + src.setTaskImplVersion(inputs.getTaskFactory().getVersion()); + + src.setInputs(inputs.getInputs().stream() + .map(Input::makeDiskFormat) + .collect(Collectors.groupingBy(i -> i.getProjectKey() + "-" + i.getOutputType())) + .values().stream() + .map(list -> { + TaskSummaryDiskFormat.InputDiskFormat result = new TaskSummaryDiskFormat.InputDiskFormat(); + result.setProjectKey(list.get(0).getProjectKey()); + result.setOutputType(list.get(0).getOutputType()); + + result.setFileHashes( + list.stream().flatMap(i -> i.getFileHashes().entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (left, right) -> { + if (left.equals(right)) { + return left; + } + throw new IllegalStateException("Two hashes for one file! " + left + " vs " + right); + })) + ); + + return result; + }) + .collect(Collectors.toList())); + + src.setConfigs(inputs.getUsedConfigs()); + + return new GsonBuilder().serializeNulls().setPrettyPrinting().create().toJson(src); } protected abstract Path taskDir(String projectName, String hashString, String outputType); @@ -378,9 +395,13 @@ protected String taskSummaryContents(CollectedTaskInputs inputs) { protected abstract Path cacheSummary(Path taskDir); interface Listener { + /** Ready for the current listener to do the work */ void onReady(CacheResult result); + /** Someone else did it, but failed for some reason, not re-runnable */ void onFailure(CacheResult result); + /** Someone else tried to do it, but ran into an error, possibly recoverable if we try again */ void onError(Throwable throwable); + /** Someone else finished it, successfully, notify listeners */ void onSuccess(CacheResult result); } public class PendingCacheResult implements Cancelable { @@ -585,4 +606,13 @@ public void markFailed(CacheResult failedResult) { } } + public Optional getCacheResult(Path taskDir) { + if (Files.exists(taskDir) || Files.exists(successMarker(taskDir))) { + CacheResult result = new CacheResult(taskDir); + knownOutputs.computeIfAbsent(taskDir, this::makeOutput); + return Optional.of(result); + } + return Optional.empty(); + } + } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java index ab4921b0..2dc195d9 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java @@ -18,10 +18,29 @@ * interested in, and take the hash of the hashes to represent */ public class Input implements com.vertispan.j2cl.build.task.Input { + public interface BuildSpecificChanges { + List compute(); + static BuildSpecificChanges memoize(BuildSpecificChanges changes) { + return new BuildSpecificChanges() { + private List results; + @Override + public List compute() { + if (results == null) { + List computed = changes.compute(); + assert computed != null; + results = new ArrayList<>(computed); + } + + return results; + } + }; + } + } private final Project project; private final String outputType; private TaskOutput contents; + private BuildSpecificChanges buildSpecificChanges; public Input(Project project, String outputType) { this.project = project; @@ -66,6 +85,13 @@ public Collection getFilesAndHashes() { .collect(Collectors.toList()); } + @Override + public Collection getChanges() { + return wrapped.getChanges().stream() + .filter(entry -> Arrays.stream(filters).anyMatch(f -> f.matches(entry.getSourcePath()))) + .collect(Collectors.toList()); + } + @Override public com.vertispan.j2cl.build.task.Project getProject() { return wrapped.getProject(); @@ -101,28 +127,34 @@ public void setCurrentContents(TaskOutput contents) { } /** - * Internal API. + * Internal API * - * Updates the given hash object with the filtered file inputs - their paths and their - * hashes, so that if files are moved or changed we change the hash value, but we don't - * re-hash each file every time we ask. + * Once a task is finished, we can let it be used by other tasks as an input. In + * order for those tasks to execute incrementally, each particular Input must + * only have changed relative to the last time that its owner task ran successfully. + * Instead of being set all at once, this is re-assigned before each task actually + * executes. */ - public void updateHash(Murmur3F hash) { - for (DiskCache.CacheEntry fileAndHash : getFilesAndHashes()) { - hash.update(fileAndHash.getSourcePath().toString().getBytes(StandardCharsets.UTF_8)); - hash.update(fileAndHash.getHash().asBytes()); - } + public void setBuildSpecificChanges(BuildSpecificChanges buildSpecificChanges) { + this.buildSpecificChanges = buildSpecificChanges; } - public void updateHashSummary(StringBuilder stringBuilder) { - stringBuilder.append("\n\t").append(getProject().getKey()).append(" ").append(getOutputType()); - for (DiskCache.CacheEntry fileAndHash : getFilesAndHashes()) { - stringBuilder.append("\n\t\t").append(fileAndHash.getSourcePath().toString()); - stringBuilder.append(" with hash ").append(fileAndHash.getHash().asString()); - } + public TaskSummaryDiskFormat.InputDiskFormat makeDiskFormat() { + TaskSummaryDiskFormat.InputDiskFormat out = new TaskSummaryDiskFormat.InputDiskFormat(); + + out.setProjectKey(getProject().getKey()); + out.setOutputType(getOutputType()); + + out.setFileHashes(getFilesAndHashes().stream().collect(Collectors.toMap( + e -> e.getSourcePath().toString(), + e -> e.getHash().asString() + ))); + + return out; } + @Override public Project getProject() { return project; @@ -148,6 +180,14 @@ public Collection getFilesAndHashes() { return contents.filesAndHashes(); } + @Override + public Collection getChanges() { + if (buildSpecificChanges == null) { + throw new NullPointerException("Changes not yet provided " + this); + } + return buildSpecificChanges.compute(); + } + @Override public String toString() { return "Input{" + diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java new file mode 100644 index 00000000..ba06c899 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java @@ -0,0 +1,62 @@ +package com.vertispan.j2cl.build; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Optional; + +public class LocalProjectBuildCache { + private final File cacheDir; + private final DiskCache cache; + + + public LocalProjectBuildCache(File cacheDir, DiskCache cache) { + this.cacheDir = cacheDir; + this.cache = cache; + } + + public void markLocalSuccess(Project project, String task, Path taskDir) { + String timestamp = String.valueOf(System.currentTimeMillis()); + try { + Path localTaskDir = cacheDir.toPath().resolve(project.getKey()).resolve(task); + Files.createDirectories(localTaskDir); + Files.write(localTaskDir.resolve(timestamp), Collections.singleton(taskDir.toString())); + + Files.list(localTaskDir).sorted().skip(5).forEach(old -> { + try { + Files.delete(old); + } catch (IOException e) { + // we don't care at all about failure, as long as it sometimes works. there could be races + // to delete, or someone just cleaned, etc. + e.printStackTrace(); + } + }); + } catch (IOException e) { + // ignore, cache just won't work for now + //TODO log this, the user will be irritated + e.printStackTrace(); + } + } + + public Optional getLatestResult(Project project, String task) { + try { + Path taskDir = cacheDir.toPath().resolve(project.getKey()).resolve(task); + Optional latest = Files.list(taskDir).sorted().findFirst(); + if (latest.isPresent()) { + String path = Files.readAllLines(latest.get()).get(0); + + return cache.getCacheResult(Paths.get(path)); + } else { + return Optional.empty(); + } + } catch (IOException e) { + // Probably file doesn't exist. This seems terrible and dirty... but on the other hand, just about any + // failure to read could mean "look just go do it from scratch, since that is always possible". + return Optional.empty(); + } + } + +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java b/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java index 3f25034c..e618a460 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java @@ -214,4 +214,11 @@ public Path getWebappDirectory() { } return Paths.get(s); } + + @Override + public boolean isIncrementalEnabled() { + // TODO once we have awesome tests for this, consider skipping the cache. Could be dangerous, + // in the case of externally provided buggy tasks. + return getString("incrementalEnabled").equalsIgnoreCase("true"); + } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java index f12411cf..70449aa0 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java @@ -1,18 +1,24 @@ package com.vertispan.j2cl.build; +import com.google.gson.Gson; import com.vertispan.j2cl.build.impl.CollectedTaskInputs; import com.vertispan.j2cl.build.task.BuildLog; +import com.vertispan.j2cl.build.task.CachedPath; import com.vertispan.j2cl.build.task.OutputTypes; import com.vertispan.j2cl.build.task.TaskFactory; import com.vertispan.j2cl.build.task.TaskContext; import java.io.FileNotFoundException; -import java.nio.file.Path; +import java.io.FileReader; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; @@ -21,6 +27,10 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static com.vertispan.j2cl.build.task.Input.ChangedCachedPath.ChangeType.ADDED; +import static com.vertispan.j2cl.build.task.Input.ChangedCachedPath.ChangeType.MODIFIED; +import static com.vertispan.j2cl.build.task.Input.ChangedCachedPath.ChangeType.REMOVED; + /** * Decides how much work to do, and when. Naive implementation just has a threadpool and does as much work * as possible at a time, depth first from the tree of tasks. A smarter impl could try to do work with many @@ -32,6 +42,7 @@ public class TaskScheduler { private final Executor executor; private final DiskCache diskCache; + private final LocalProjectBuildCache buildCache; private final BuildLog buildLog; // This is technically incorrect, but covers the current use cases, and should fail loudly if this assumption @@ -50,14 +61,15 @@ public class TaskScheduler { * * Caller is responsible for shutting down the executor service - canceling ongoing work * should be supported, but presently isn't. - * - * @param executor executor to submit work to, to be performed off thread + * @param executor executor to submit work to, to be performed off thread * @param diskCache cache to read results from, and save new results to + * @param buildCache * @param buildLog log to write details to about work being performed */ - public TaskScheduler(Executor executor, DiskCache diskCache, BuildLog buildLog) { + public TaskScheduler(Executor executor, DiskCache diskCache, LocalProjectBuildCache buildCache, BuildLog buildLog) { this.executor = executor; this.diskCache = diskCache; + this.buildCache = buildCache; this.buildLog = buildLog; } @@ -211,7 +223,27 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult } try { long start = System.currentTimeMillis(); - taskDetails.getTask().execute(new TaskContext(result.outputDir(), log)); + + Optional latestResult = buildCache.getLatestResult(taskDetails.getProject(), taskDetails.getTaskFactory().getOutputType()); + final TaskSummaryDiskFormat taskSummaryDiskFormat = latestResult.map(TaskScheduler.this::getTaskSummary).orElse(null); + // deserialize its inputs list and use its outputs below + for (Input input : taskDetails.getInputs()) { + input.setBuildSpecificChanges(() -> { + if (latestResult.isPresent() && taskSummaryDiskFormat != null) { + Optional prevInput = taskSummaryDiskFormat.getInputs().stream() + .filter(i -> i.getProjectKey().equals(input.getProject().getKey())) + .filter(i -> i.getOutputType().equals(input.getOutputType())) + .findAny(); + if (prevInput.isPresent()) { + return diff(input.getFilesAndHashes().stream().collect(Collectors.toMap(e -> e.getSourcePath().toString(), Function.identity())), prevInput.get().getFileHashes()); + } + } + + return input.getFilesAndHashes().stream() + .map(entry -> new ChangedCachedPath(ADDED, entry.getSourcePath(), entry)).collect(Collectors.toList()); + }); + } + taskDetails.getTask().execute(new TaskContext(result.outputDir(), log, latestResult.map(DiskCache.CacheResult::outputDir).orElse(null))); if (Thread.currentThread().isInterrupted()) { // Tried and failed to be canceled, so even though we were successful, some files might // have been deleted. Continue deleting contents @@ -222,6 +254,7 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult if (elapsedMillis > 5) { buildLog.info("Finished " + taskDetails.getDebugName() + " in " + elapsedMillis + "ms"); } + buildCache.markLocalSuccess(taskDetails.getProject(), taskDetails.getTaskFactory().getOutputType(), result.taskDir()); result.markSuccess(); } catch (Throwable exception) { @@ -324,9 +357,10 @@ private void scheduleMoreWork(DiskCache.CacheResult cacheResult) { // When something finishes, remove it from the various dependency lists and see if we can run the loop again with more work. // Presently this could be called multiple times, so we check if already removed if (tasks.complete(taskDetails)) { + TaskOutput output = cacheResult.output(); for (Input input : allInputs.computeIfAbsent(taskDetails.getAsInput(), ignore -> Collections.emptyList())) { // since we don't support running more than one thing at a time, this will not change data out from under a running task - input.setCurrentContents(cacheResult.output()); + input.setCurrentContents(output); //TODO This maybe can race with the check on line 84, and we break since the input isn't ready yet } @@ -349,7 +383,9 @@ private boolean executeFinalTask(CollectedTaskInputs taskDetails, DiskCache.Cach try { //TODO Make sure that we want to write this to _only_ the current log, and not also to any file //TODO Also be sure to write a prefix automatically - ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog)); + + // TODO also consider if lastSuccessfulPath should be null for final tasks + ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog, null)); buildLog.info("Finished final task " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms"); } catch (Throwable t) { buildLog.error("FAILED " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms",t); @@ -366,4 +402,37 @@ private boolean executeFinalTask(CollectedTaskInputs taskDetails, DiskCache.Cach }); }); } + + private List diff(Map currentFiles, Map previousFiles) { + List changes = new ArrayList<>(); + Set added = new HashSet<>(currentFiles.keySet()); + added.removeAll(previousFiles.keySet()); + added.forEach(newPath -> { + changes.add(new ChangedCachedPath(ADDED, Paths.get(newPath), currentFiles.get(newPath))); + }); + + Set removed = new HashSet<>(previousFiles.keySet()); + removed.removeAll(currentFiles.keySet()); + removed.forEach(removedPath -> { + changes.add(new ChangedCachedPath(REMOVED, Paths.get(removedPath), null)); + }); + + Map changed = new HashMap<>(currentFiles); + changed.keySet().removeAll(added); + changed.forEach((possiblyModifiedPath, entry) -> { + if (!entry.getHash().asString().equals(previousFiles.get(possiblyModifiedPath))) { + changes.add(new ChangedCachedPath(MODIFIED, Paths.get(possiblyModifiedPath), entry)); + } + }); + + return changes; + } + + private TaskSummaryDiskFormat getTaskSummary(DiskCache.CacheResult latestResult) { + try { + return new Gson().fromJson(new FileReader(latestResult.cachedSummary().toFile()), TaskSummaryDiskFormat.class); + } catch (FileNotFoundException ex) { + return null; + } + } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskSummaryDiskFormat.java b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskSummaryDiskFormat.java new file mode 100644 index 00000000..fe18e924 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskSummaryDiskFormat.java @@ -0,0 +1,93 @@ +package com.vertispan.j2cl.build; + +import java.util.List; +import java.util.Map; + +public class TaskSummaryDiskFormat { + private String projectKey; + private String outputType; + private String taskImpl; + private String taskImplVersion; + + private List inputs; + + private Map configs; + + public String getProjectKey() { + return projectKey; + } + + public void setProjectKey(String projectKey) { + this.projectKey = projectKey; + } + + public String getOutputType() { + return outputType; + } + + public void setOutputType(String outputType) { + this.outputType = outputType; + } + + public String getTaskImpl() { + return taskImpl; + } + + public void setTaskImpl(String taskImpl) { + this.taskImpl = taskImpl; + } + + public String getTaskImplVersion() { + return taskImplVersion; + } + + public void setTaskImplVersion(String taskImplVersion) { + this.taskImplVersion = taskImplVersion; + } + + public List getInputs() { + return inputs; + } + + public void setInputs(List inputs) { + this.inputs = inputs; + } + + public Map getConfigs() { + return configs; + } + + public void setConfigs(Map configs) { + this.configs = configs; + } + + public static class InputDiskFormat { + private String projectKey; + private String outputType; + private Map fileHashes; + + public String getProjectKey() { + return projectKey; + } + + public void setProjectKey(String projectKey) { + this.projectKey = projectKey; + } + + public String getOutputType() { + return outputType; + } + + public void setOutputType(String outputType) { + this.outputType = outputType; + } + + public Map getFileHashes() { + return fileHashes; + } + + public void setFileHashes(Map fileHashes) { + this.fileHashes = fileHashes; + } + } +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Config.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Config.java index 77c1efdb..9e40d98d 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Config.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Config.java @@ -48,4 +48,17 @@ public interface Config { */ Path getWebappDirectory(); + /** + * Allow tasks to know if they need to do the work to build incrementally. Tasks + * should decide for themselves what inputs they might need or what work they + * might do based on this, plus if sources of a given project are mapped or not. + *

+ * For example, if incremental is cheap, might as well always do it, if not, only + * do it if all markers suggest it is a good idea. However, if this flag is false, + * incremental should never be attempted (could be a bug in it, etc). + * + * @return true if incremental is enabled, false if it should be skipped + */ + boolean isIncrementalEnabled(); + } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java index 8d8db900..507ebc9e 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java @@ -3,6 +3,7 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.util.Collection; +import java.util.Optional; public interface Input { /** @@ -23,6 +24,21 @@ public interface Input { */ Collection getFilesAndHashes(); + Collection getChanges(); + + interface ChangedCachedPath { + enum ChangeType { + ADDED, + REMOVED, + MODIFIED; + } + ChangeType changeType(); + + Path getSourcePath(); + + Optional getNewAbsolutePath(); + } + /** * Public API for tasks. * diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java index 59575933..e9bb4fd1 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java @@ -1,14 +1,17 @@ package com.vertispan.j2cl.build.task; import java.nio.file.Path; +import java.util.Optional; public class TaskContext implements BuildLog { private final Path path; private final BuildLog log; + private final Path lastSuccessfulPath; - public TaskContext(Path path, BuildLog log) { + public TaskContext(Path path, BuildLog log, Path lastSuccessfulPath) { this.path = path; this.log = log; + this.lastSuccessfulPath = lastSuccessfulPath; } public Path outputPath() { @@ -19,6 +22,16 @@ public BuildLog log() { return log; } + /** + * Returns the output directory from the last time this task ran, to be used to copy other unchanged + * output files rather than regenerate them. + * + * @return empty if no previous build exists, otherwise a path to the last successful output + */ + public Optional lastSuccessfulOutput() { + return Optional.ofNullable(lastSuccessfulPath); + } + @Override public void debug(String msg) { log.debug(msg); diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java index bbc709b6..084f48ef 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java @@ -85,6 +85,9 @@ public abstract class AbstractBuildMojo extends AbstractCacheMojo { @Parameter(defaultValue = "AVOID_MAVEN") private AnnotationProcessorMode annotationProcessorMode; + @Parameter(defaultValue = "true") + private boolean incrementalEnabled; + private List defaultDependencyReplacements = Arrays.asList( new DependencyReplacement("com.google.jsinterop:base", "com.vertispan.jsinterop:base:" + Versions.VERTISPAN_JSINTEROP_BASE_VERSION), new DependencyReplacement("org.realityforge.com.google.jsinterop:base", "com.vertispan.jsinterop:base:" + Versions.VERTISPAN_JSINTEROP_BASE_VERSION), diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractCacheMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractCacheMojo.java index dc539aaf..d30cbe68 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractCacheMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractCacheMojo.java @@ -16,6 +16,9 @@ public abstract class AbstractCacheMojo extends AbstractMojo { @Parameter(defaultValue = "${project.build.directory}/gwt3BuildCache", required = true, property = "gwt3.cache.dir") private File gwt3BuildCacheDir; + @Parameter(defaultValue = "${project.build.directory}/j2cl-maven-plugin-local-cache", required = true) + protected File localBuildCache; + protected Path getCacheDir() { PluginDescriptor pluginDescriptor = (PluginDescriptor) getPluginContext().get("pluginDescriptor"); String pluginVersion = pluginDescriptor.getVersion(); diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/BuildMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/BuildMojo.java index b1eb448e..755a5a5d 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/BuildMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/BuildMojo.java @@ -4,6 +4,7 @@ import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.DefaultDiskCache; import com.vertispan.j2cl.build.DiskCache; +import com.vertispan.j2cl.build.LocalProjectBuildCache; import com.vertispan.j2cl.build.Project; import com.vertispan.j2cl.build.TaskRegistry; import com.vertispan.j2cl.build.TaskScheduler; @@ -231,7 +232,7 @@ public void execute() throws MojoExecutionException, MojoFailureException { addShutdownHook(executor, diskCache); MavenLog mavenLog = new MavenLog(getLog()); - TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, mavenLog); + TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, new LocalProjectBuildCache(localBuildCache, diskCache), mavenLog); TaskRegistry taskRegistry = createTaskRegistry(); diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/TestMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/TestMojo.java index b3923eba..64e7e5a9 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/TestMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/TestMojo.java @@ -9,6 +9,7 @@ import com.vertispan.j2cl.build.DefaultDiskCache; import com.vertispan.j2cl.build.Dependency; import com.vertispan.j2cl.build.DiskCache; +import com.vertispan.j2cl.build.LocalProjectBuildCache; import com.vertispan.j2cl.build.Project; import com.vertispan.j2cl.build.PropertyTrackingConfig; import com.vertispan.j2cl.build.TaskRegistry; @@ -332,7 +333,7 @@ public void execute() throws MojoExecutionException, MojoFailureException { addShutdownHook(executor, diskCache); MavenLog mavenLog = new MavenLog(getLog()); - TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, mavenLog); + TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, new LocalProjectBuildCache(localBuildCache, diskCache), mavenLog); TaskRegistry taskRegistry = createTaskRegistry(); // Given these, build the graph of work we need to complete to get the list of tests diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/WatchMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/WatchMojo.java index 5563318e..8ab89842 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/WatchMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/WatchMojo.java @@ -3,6 +3,7 @@ import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.DefaultDiskCache; import com.vertispan.j2cl.build.DiskCache; +import com.vertispan.j2cl.build.LocalProjectBuildCache; import com.vertispan.j2cl.build.Project; import com.vertispan.j2cl.build.TaskRegistry; import com.vertispan.j2cl.build.TaskScheduler; @@ -196,7 +197,7 @@ public void execute() throws MojoExecutionException, MojoFailureException { addShutdownHook(executor, diskCache); MavenLog mavenLog = new MavenLog(getLog()); - TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, mavenLog); + TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, new LocalProjectBuildCache(localBuildCache, diskCache), mavenLog); // TODO support individual task registries per execution TaskRegistry taskRegistry = createTaskRegistry(); diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java index 6e6bfa5d..779d8c94 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java @@ -1,24 +1,53 @@ package com.vertispan.j2cl.build.provided; import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.javascript.jscomp.CompilationLevel; +import com.google.javascript.jscomp.Compiler; +import com.google.javascript.jscomp.CompilerInput; import com.google.javascript.jscomp.CompilerOptions; import com.google.javascript.jscomp.DependencyOptions; +import com.google.javascript.jscomp.SourceFile; +import com.google.javascript.jscomp.deps.ClosureBundler; +import com.google.javascript.jscomp.deps.DependencyInfo; +import com.google.javascript.jscomp.deps.ModuleLoader; +import com.google.javascript.jscomp.deps.SortedDependencies; +import com.google.javascript.jscomp.parsing.parser.FeatureSet; +import com.google.javascript.jscomp.serialization.SourceFileProto; +import com.google.javascript.jscomp.transpile.BaseTranspiler; +import com.google.javascript.jscomp.transpile.Transpiler; +import com.google.protobuf.CodedInputStream; import com.vertispan.j2cl.build.task.*; import io.methvin.watcher.hashing.Murmur3F; import com.vertispan.j2cl.tools.Closure; import org.apache.commons.io.FileUtils; import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; +import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -61,6 +90,11 @@ public Task resolve(Project project, Config config) { .collect(Collectors.toList()); } + // Consider treating this always as true, since the build doesnt get more costly to be incremental + boolean incrementalEnabled = config.isIncrementalEnabled(); + + Gson gson = new GsonBuilder().setPrettyPrinting().serializeNulls().create(); + return context -> { assert Files.isDirectory(context.outputPath()); File closureOutputDir = context.outputPath().toFile(); @@ -77,8 +111,6 @@ public Task resolve(Project project, Config config) { return;// nothing to do } - Closure closureCompiler = new Closure(context); - // copy the sources locally so that we can create usable sourcemaps //TODO consider a soft link File sources = new File(closureOutputDir, Closure.SOURCES_DIRECTORY_NAME); @@ -86,35 +118,113 @@ public Task resolve(Project project, Config config) { FileUtils.copyDirectory(path.toFile(), sources); } - // create the JS bundle, only ordering these files - boolean success = closureCompiler.compile( - CompilationLevel.BUNDLE, - DependencyOptions.DependencyMode.SORT_ONLY, - CompilerOptions.LanguageMode.NO_TRANSPILE, - Collections.singletonMap( - sources.getAbsolutePath(), - js.stream() - .map(Input::getFilesAndHashes) + List dependencyInfos = new ArrayList<>(); + Compiler jsCompiler = new Compiler(System.err);//TODO before merge, write this to the log + + if (incrementalEnabled && context.lastSuccessfulOutput().isPresent()) { + // collect any dep info from disk for existing files + Map depInfoMap = new HashMap<>(); + Path lastOutput = context.lastSuccessfulOutput().get(); + try (InputStream inputStream = Files.newInputStream(lastOutput.resolve("depInfo.json"))) { + Type listType = new TypeToken>() { + }.getType(); + List deps = gson.fromJson(new BufferedReader(new InputStreamReader(inputStream)), listType); + depInfoMap.putAll( + deps.stream() + .map(info -> new DependencyInfoAndSource(info, () -> { + return Files.readString(lastOutput.resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(info.getName())); + })) + .collect(Collectors.toMap(DependencyInfo::getName, Function.identity())) + ); + } + + // create new dep info for any added/modified file + for (Input jsInput : js) { + for (Input.ChangedCachedPath change : jsInput.getChanges()) { + if (change.changeType() == Input.ChangedCachedPath.ChangeType.REMOVED) { + depInfoMap.remove(change.getSourcePath().toString()); + } else { + // ADD or MODIFY + CompilerInput input = new CompilerInput(SourceFile.builder() + .withPath(context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(change.getSourcePath())) + .withOriginalPath(change.getSourcePath().toString()) + .build()); + input.setCompiler(jsCompiler); + depInfoMap.put( + change.getSourcePath().toString(), + new DependencyInfoAndSource(input, input::getCode) + ); + } + } + } + + // no need to expand to include other files, since this is only computed locally + + // assign the dep info and sources we have + dependencyInfos.addAll(depInfoMap.values()); + } else { + //non-incremental, read everything + for (Input jsInput : js) { + for (CachedPath path : jsInput.getFilesAndHashes()) { + CompilerInput input = new CompilerInput(SourceFile.builder() + .withPath(context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(path.getSourcePath())) + .withOriginalPath(path.getSourcePath().toString()) + .build()); + input.setCompiler(jsCompiler); + + dependencyInfos.add(new DependencyInfoAndSource(input, input::getCode)); + } + } + } + + // re-sort that full collection + SortedDependencies sorter = new SortedDependencies<>(dependencyInfos); + + + // TODO optional/stretch-goal find first change in the list, so we can keep old prefix of bundle output + + + // rebundle all (optional: remaining) files using this already handled sort + ClosureBundler bundler = new ClosureBundler(Transpiler.NULL, new BaseTranspiler( + new BaseTranspiler.CompilerSupplier( + CompilerOptions.LanguageMode.ECMASCRIPT_NEXT.toFeatureSet().without(FeatureSet.Feature.MODULES), + ModuleLoader.ResolutionMode.BROWSER, + ImmutableList.copyOf(js.stream() + .map(Input::getParentPaths) .flatMap(Collection::stream) - .map(CachedPath::getSourcePath) .map(Path::toString) - .collect(Collectors.toList()) + .collect(Collectors.toList())), + ImmutableMap.of() ), - sources, - Collections.emptyList(), - Collections.emptyMap(), - Collections.emptyList(),//TODO actually pass these in when we can restrict and cache them sanely - Optional.empty(), - true,//TODO have this be passed in, - true,//default to true, will have no effect anyway - false, - false, - "CUSTOM", // doesn't matter, bundle won't check this - outputFile - ); - - if (!success) { - throw new IllegalStateException("Closure Compiler failed, check log for details"); + "" + )).useEval(true); + + try (OutputStream outputStream = Files.newOutputStream(Paths.get(outputFile)); + BufferedWriter bundleOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) { + for (DependencyInfoAndSource info : sorter.getSortedList()) { + String code = info.getSource(); + String name = info.getName(); + + //TODO do we actually need this? + if (Compiler.isFillFileName(name) && code.isEmpty()) { + continue; + } + + // append this file and a comment where it came from + bundleOut.append("//").append(name).append("\n"); + bundler.withPath(name).withSourceUrl(Closure.SOURCES_DIRECTORY_NAME + "/" + name).appendTo(bundleOut, info, code); + bundleOut.append("\n"); + + } + + } + // append dependency info to deserialize on some incremental rebuild + try (OutputStream outputStream = Files.newOutputStream(context.outputPath().resolve("depInfo.json")); + BufferedWriter jsonOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) { + List jsonList = sorter.getSortedList().stream() + .map(DependencyInfoFormat::new) + .collect(Collectors.toList()); + gson.toJson(jsonList, jsonOut); } // hash the file itself, rename to include that hash @@ -129,4 +239,174 @@ public Task resolve(Project project, Config config) { //TODO when back to keyboard rename sourcemap? is that a thing we need to do? }; } + + public interface SourceSupplier { + String get() throws IOException; + } + public static class DependencyInfoAndSource extends DependencyInfo.Base { + private final DependencyInfo delegate; + private final SourceSupplier sourceSupplier; + + public DependencyInfoAndSource(DependencyInfo delegate, SourceSupplier sourceSupplier) { + this.delegate = delegate; + this.sourceSupplier = sourceSupplier; + } + + public String getSource() throws IOException { + return sourceSupplier.get(); + } + + @Override + public String getName() { + return delegate.getName(); + } + + @Override + public String getPathRelativeToClosureBase() { + return delegate.getPathRelativeToClosureBase(); + } + + @Override + public ImmutableList getProvides() { + return delegate.getProvides(); + } + + @Override + public ImmutableList getRequires() { + return delegate.getRequires(); + } + + @Override + public ImmutableList getRequiredSymbols() { + //deliberately overriding the base impl + return delegate.getRequiredSymbols(); + } + + @Override + public ImmutableList getTypeRequires() { + return delegate.getTypeRequires(); + } + + @Override + public ImmutableMap getLoadFlags() { + return delegate.getLoadFlags(); + } + + @Override + public boolean isModule() { + return delegate.isModule(); + } + + @Override + public boolean isEs6Module() { + return delegate.isEs6Module(); + } + + @Override + public boolean isGoogModule() { + return delegate.isGoogModule(); + } + + @Override + public boolean getHasExternsAnnotation() { + return delegate.getHasExternsAnnotation(); + } + + @Override + public boolean getHasNoCompileAnnotation() { + return delegate.getHasNoCompileAnnotation(); + } + } + + public static class DependencyInfoFormat extends DependencyInfo.Base { + private String name; +// private String pathRelativeToClosureBase = name; + private List provides; +// private List requires; //skipping requires as it isnt used by the dep sorter + private List requiredSymbols; + private List typeRequires; + private Map loadFlags; + private boolean hasExternsAnnotation; + private boolean hasNoCompileAnnotation; + + public DependencyInfoFormat() { + + } + + public DependencyInfoFormat(DependencyInfo info) { + setName(info.getName()); + setHasExternsAnnotation(info.getHasExternsAnnotation()); + setHasNoCompileAnnotation(info.getHasExternsAnnotation()); + setProvides(info.getProvides()); + setLoadFlags(info.getLoadFlags()); + setTypeRequires(info.getTypeRequires()); + setRequiredSymbols(info.getRequiredSymbols()); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public String getPathRelativeToClosureBase() { + return getName(); + } + + public ImmutableList getProvides() { + return ImmutableList.copyOf(provides); + } + + public void setProvides(List provides) { + this.provides = provides; + } + + public ImmutableList getRequires() { + return ImmutableList.of(); + } + + @Override + public ImmutableList getRequiredSymbols() { + return ImmutableList.copyOf(requiredSymbols); + } + + public void setRequiredSymbols(List requiredSymbols) { + this.requiredSymbols = requiredSymbols; + } + + public ImmutableList getTypeRequires() { + return ImmutableList.copyOf(typeRequires); + } + + public void setTypeRequires(List typeRequires) { + this.typeRequires = typeRequires; + } + + public ImmutableMap getLoadFlags() { + return ImmutableMap.copyOf(loadFlags); + } + + public void setLoadFlags(Map loadFlags) { + this.loadFlags = loadFlags; + } + + public boolean getHasExternsAnnotation() { + return hasExternsAnnotation; + } + + public void setHasExternsAnnotation(boolean hasExternsAnnotation) { + this.hasExternsAnnotation = hasExternsAnnotation; + } + + public boolean getHasNoCompileAnnotation() { + return hasNoCompileAnnotation; + } + + public void setHasNoCompileAnnotation(boolean hasNoCompileAnnotation) { + this.hasNoCompileAnnotation = hasNoCompileAnnotation; + } + } } diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java index a5e94858..6e138dba 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java @@ -6,8 +6,11 @@ import com.vertispan.j2cl.tools.J2cl; import java.io.File; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java index 0ae10076..9d1a15bd 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java @@ -5,7 +5,13 @@ import com.vertispan.j2cl.build.task.*; import com.vertispan.j2cl.tools.GwtIncompatiblePreprocessor; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.PathMatcher; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -37,12 +43,42 @@ public Task resolve(Project project, Config config) { if (inputSources.getFilesAndHashes().isEmpty()) { return;// nothing to do } + List filesToProcess = new ArrayList<>(); + if (context.lastSuccessfulOutput().isPresent()) { + Map unmodified = inputSources.getFilesAndHashes().stream().collect(Collectors.toMap( + CachedPath::getSourcePath, + Function.identity() + )); + //only process changed files, copy unchanged ones + for (Input.ChangedCachedPath change : inputSources.getChanges()) { + // remove the file, since it was changed in some way + unmodified.remove(change.getSourcePath()); + + if (change.changeType() != Input.ChangedCachedPath.ChangeType.REMOVED) { + // track the files we actually need to process + filesToProcess.add(makeFileInfo(change)); + } + } + for (CachedPath path : unmodified.values()) { + Files.createDirectories(context.outputPath().resolve(path.getSourcePath()).getParent()); + Files.copy(context.lastSuccessfulOutput().get().resolve(path.getSourcePath()), context.outputPath().resolve(path.getSourcePath())); + } + } else { + for (CachedPath path : inputSources.getFilesAndHashes()) { + filesToProcess.add(makeFileInfo(path)); + } + } GwtIncompatiblePreprocessor preprocessor = new GwtIncompatiblePreprocessor(context.outputPath().toFile(), context); - preprocessor.preprocess( - inputSources.getFilesAndHashes().stream() - .map(p -> SourceUtils.FileInfo.create(p.getAbsolutePath().toString(), p.getSourcePath().toString())) - .collect(Collectors.toList()) - ); + preprocessor.preprocess(filesToProcess); }; } + + private SourceUtils.FileInfo makeFileInfo(Input.ChangedCachedPath change) { + assert change.getNewAbsolutePath().isPresent() : "Can't make a FileInfo if it no longer exists"; + return SourceUtils.FileInfo.create(change.getNewAbsolutePath().get().toString(), change.getSourcePath().toString()); + } + + private SourceUtils.FileInfo makeFileInfo(CachedPath path) { + return SourceUtils.FileInfo.create(path.getAbsolutePath().toString(), path.getSourcePath().toString()); + } } From 1df406d2109f63876812e33c2c848580280a02cc Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 21 Dec 2022 15:12:27 -0600 Subject: [PATCH 04/12] Hacky demo attempt of j2cl incremental work --- .../j2cl/build/provided/J2clTask.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java index 6e138dba..48052c86 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java @@ -52,12 +52,80 @@ public Task resolve(Project project, Config config) { .map(input -> input.filter(JAVA_BYTECODE)) .collect(Collectors.toList()); + Input ownStrippedBytecode = null; + //TODO api question, do we not allow incremental builds of external deps? + boolean incrementalEnabled = config.isIncrementalEnabled(); + if (incrementalEnabled && project.hasSourcesMapped()) { + ownStrippedBytecode = input(project, OutputTypes.STRIPPED_BYTECODE_HEADERS).filter(JAVA_BYTECODE); + } + File bootstrapClasspath = config.getBootstrapClasspath(); List extraClasspath = config.getExtraClasspath(); return context -> { if (ownJavaSources.getFilesAndHashes().isEmpty()) { return;// nothing to do } + + nope: + if (false && context.lastSuccessfulOutput().isPresent()) { + if (!incrementalEnabled || !project.hasSourcesMapped()) { + break nope; + } + // maybe attempt incremental... + Path previousBuildData = context.lastSuccessfulOutput().get().resolve("build.data"); + + for (Input classpathHeader : classpathHeaders) { + if (!classpathHeader.getChanges().isEmpty()) { + break nope; + } + } + + //... do things, copying old JS that still matches current inputs, don't copy files that match deleted inputs, + // recompile files that match changed inputs or have changed dependencies + + // For all files that actually exist today, try to copy their old output to our new output path + // This will get all existing output files (note that we might still rebuild them), and skip "removed" files + // (since they aren't in the current list of getFilesAndHashes(), but "added" files will result in an error + for (CachedPath file : ownJavaSources.getFilesAndHashes()) { + for (Path existing : expandToExistingFiles(previousBuildData, file.getSourcePath())) { + // TODO make sure it exists before copying - it might be a new file and not exist + Files.copy(context.lastSuccessfulOutput().get().resolve(existing), context.outputPath().resolve(existing)); + } + } + + // TODO deal with native sources changing + + List myJavaSourcesThatHaveToBeRebuilt = new ArrayList<>(); + for (Input.ChangedCachedPath change : ownJavaSources.getChanges()) { + switch (change.changeType()) { + case ADDED: + //noop, already didn't exist + myJavaSourcesThatHaveToBeRebuilt.add(change.getSourcePath()); + break; + case REMOVED: + //noop, wasn't copied + //TODO anything that used to (transitively or without changes) depend on this must be rebuilt! + myJavaSourcesThatHaveToBeRebuilt.addAll(findFilesThatDependedOnPath(previousBuildData, change.getSourcePath())); + break; + case MODIFIED: + //delete old output for this file since we must rebuild + for (Path existing : expandToExistingFiles(previousBuildData, change.getSourcePath())) { + Files.delete(existing); + } + // TODO expand the set of files that were modified because this was modified + myJavaSourcesThatHaveToBeRebuilt.add(change.getSourcePath()); + myJavaSourcesThatHaveToBeRebuilt.addAll(findFilesThatDependedOnPath(previousBuildData, change.getSourcePath())); + break; + } + } + + // Run J2CL with + // * mySourcesThatHaveToBeRebuilt + // * any native sources that applied to those, or were changed + // * existing classpath, plus OUR OWN CURRENT SOURCES + return; + } + List classpathDirs = Stream.concat( classpathHeaders.stream().flatMap(i -> i.getParentPaths().stream().map(Path::toFile)), extraClasspath.stream() @@ -87,4 +155,12 @@ public Task resolve(Project project, Config config) { } }; } + + private Collection findFilesThatDependedOnPath(Path previousBuildData, Path sourcePath) { + return null; + } + + private List expandToExistingFiles(Path previousBuildData, Path file) { + return null; + } } From 7d56e118e33c58a749a6fb8841dee17ca7a05d3c Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 21 Dec 2022 15:12:43 -0600 Subject: [PATCH 05/12] Move to Java 11 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 298c9bd7..cab5282e 100644 --- a/pom.xml +++ b/pom.xml @@ -120,8 +120,8 @@ UTF-8 UTF-8 - 1.8 - 1.8 + 11 + 11 From e1451878c6227651c0e91bb26f4954cc7d902f56 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 21 Dec 2022 15:26:43 -0600 Subject: [PATCH 06/12] Cleanup imports --- .../com/vertispan/j2cl/build/BuildService.java | 1 - .../main/java/com/vertispan/j2cl/build/Input.java | 3 --- .../j2cl/build/provided/ClosureBundleTask.java | 14 +++++++------- .../j2cl/build/provided/StripSourcesTask.java | 1 - 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java b/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java index 868ac139..a83e559b 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java @@ -1,7 +1,6 @@ package com.vertispan.j2cl.build; import com.vertispan.j2cl.build.impl.CollectedTaskInputs; -import com.vertispan.j2cl.build.task.BuildLog; import com.vertispan.j2cl.build.task.OutputTypes; import com.vertispan.j2cl.build.task.TaskFactory; diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java index 2dc195d9..573aeaa8 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java @@ -1,8 +1,5 @@ package com.vertispan.j2cl.build; -import io.methvin.watcher.hashing.Murmur3F; - -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.PathMatcher; import java.util.*; diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java index 779d8c94..2fd67359 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java @@ -6,24 +6,25 @@ import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.javascript.jscomp.CompilationLevel; import com.google.javascript.jscomp.Compiler; import com.google.javascript.jscomp.CompilerInput; import com.google.javascript.jscomp.CompilerOptions; -import com.google.javascript.jscomp.DependencyOptions; import com.google.javascript.jscomp.SourceFile; import com.google.javascript.jscomp.deps.ClosureBundler; import com.google.javascript.jscomp.deps.DependencyInfo; import com.google.javascript.jscomp.deps.ModuleLoader; import com.google.javascript.jscomp.deps.SortedDependencies; import com.google.javascript.jscomp.parsing.parser.FeatureSet; -import com.google.javascript.jscomp.serialization.SourceFileProto; import com.google.javascript.jscomp.transpile.BaseTranspiler; import com.google.javascript.jscomp.transpile.Transpiler; -import com.google.protobuf.CodedInputStream; -import com.vertispan.j2cl.build.task.*; -import io.methvin.watcher.hashing.Murmur3F; +import com.vertispan.j2cl.build.task.CachedPath; +import com.vertispan.j2cl.build.task.Config; +import com.vertispan.j2cl.build.task.Input; +import com.vertispan.j2cl.build.task.OutputTypes; +import com.vertispan.j2cl.build.task.Project; +import com.vertispan.j2cl.build.task.TaskFactory; import com.vertispan.j2cl.tools.Closure; +import io.methvin.watcher.hashing.Murmur3F; import org.apache.commons.io.FileUtils; import java.io.BufferedInputStream; @@ -46,7 +47,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java index 9d1a15bd..452bd8ca 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java @@ -13,7 +13,6 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; @AutoService(TaskFactory.class) public class StripSourcesTask extends TaskFactory { From 30e69677441676c9b8930c6b3721a5eeb833142b Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 1 Mar 2023 16:17:29 -0600 Subject: [PATCH 07/12] Simple self-review cleanup --- .../j2cl/build/provided/ClosureBundleTask.java | 17 +++++++---------- .../j2cl/build/provided/StripSourcesTask.java | 1 - 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java index 29bd441a..4f363eda 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java @@ -44,7 +44,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -123,19 +122,18 @@ public Task resolve(Project project, Config config) { if (incrementalEnabled && context.lastSuccessfulOutput().isPresent()) { // collect any dep info from disk for existing files - Map depInfoMap = new HashMap<>(); + final Map depInfoMap; Path lastOutput = context.lastSuccessfulOutput().get(); try (InputStream inputStream = Files.newInputStream(lastOutput.resolve("depInfo.json"))) { Type listType = new TypeToken>() { }.getType(); List deps = gson.fromJson(new BufferedReader(new InputStreamReader(inputStream)), listType); - depInfoMap.putAll( - deps.stream() - .map(info -> new DependencyInfoAndSource(info, () -> { - return Files.readString(lastOutput.resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(info.getName())); - })) - .collect(Collectors.toMap(DependencyInfo::getName, Function.identity())) - ); + depInfoMap = deps.stream() + .map(info -> new DependencyInfoAndSource( + info, + () -> Files.readString(lastOutput.resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(info.getName()))) + ) + .collect(Collectors.toMap(DependencyInfo::getName, Function.identity())); } // create new dep info for any added/modified file @@ -183,7 +181,6 @@ public Task resolve(Project project, Config config) { // TODO optional/stretch-goal find first change in the list, so we can keep old prefix of bundle output - // rebundle all (optional: remaining) files using this already handled sort ClosureBundler bundler = new ClosureBundler(Transpiler.NULL, new BaseTranspiler( new BaseTranspiler.CompilerSupplier( diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java index 452bd8ca..34255dd5 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java @@ -17,7 +17,6 @@ @AutoService(TaskFactory.class) public class StripSourcesTask extends TaskFactory { public static final PathMatcher JAVA_SOURCES = withSuffix(".java"); - public static final PathMatcher NATIVE_JS_SOURCES = withSuffix(".native.js"); @Override public String getOutputType() { From b15c815f432aecd8e55f5ab0e3e2f9fbf59fdb78 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 1 Mar 2023 16:17:58 -0600 Subject: [PATCH 08/12] Revert all incremental work for the j2cl task, deferring to Dmitrii --- .../j2cl/build/provided/J2clTask.java | 79 ------------------- 1 file changed, 79 deletions(-) diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java index 627ca0ef..bd3b87ec 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/J2clTask.java @@ -6,11 +6,8 @@ import com.vertispan.j2cl.tools.J2cl; import java.io.File; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -52,80 +49,12 @@ public Task resolve(Project project, Config config) { .map(input -> input.filter(JAVA_BYTECODE)) .collect(Collectors.toUnmodifiableList()); - Input ownStrippedBytecode = null; - //TODO api question, do we not allow incremental builds of external deps? - boolean incrementalEnabled = config.isIncrementalEnabled(); - if (incrementalEnabled && project.hasSourcesMapped()) { - ownStrippedBytecode = input(project, OutputTypes.STRIPPED_BYTECODE_HEADERS).filter(JAVA_BYTECODE); - } - File bootstrapClasspath = config.getBootstrapClasspath(); List extraClasspath = config.getExtraClasspath(); return context -> { if (ownJavaSources.getFilesAndHashes().isEmpty()) { return;// nothing to do } - - nope: - if (false && context.lastSuccessfulOutput().isPresent()) { - if (!incrementalEnabled || !project.hasSourcesMapped()) { - break nope; - } - // maybe attempt incremental... - Path previousBuildData = context.lastSuccessfulOutput().get().resolve("build.data"); - - for (Input classpathHeader : classpathHeaders) { - if (!classpathHeader.getChanges().isEmpty()) { - break nope; - } - } - - //... do things, copying old JS that still matches current inputs, don't copy files that match deleted inputs, - // recompile files that match changed inputs or have changed dependencies - - // For all files that actually exist today, try to copy their old output to our new output path - // This will get all existing output files (note that we might still rebuild them), and skip "removed" files - // (since they aren't in the current list of getFilesAndHashes(), but "added" files will result in an error - for (CachedPath file : ownJavaSources.getFilesAndHashes()) { - for (Path existing : expandToExistingFiles(previousBuildData, file.getSourcePath())) { - // TODO make sure it exists before copying - it might be a new file and not exist - Files.copy(context.lastSuccessfulOutput().get().resolve(existing), context.outputPath().resolve(existing)); - } - } - - // TODO deal with native sources changing - - List myJavaSourcesThatHaveToBeRebuilt = new ArrayList<>(); - for (Input.ChangedCachedPath change : ownJavaSources.getChanges()) { - switch (change.changeType()) { - case ADDED: - //noop, already didn't exist - myJavaSourcesThatHaveToBeRebuilt.add(change.getSourcePath()); - break; - case REMOVED: - //noop, wasn't copied - //TODO anything that used to (transitively or without changes) depend on this must be rebuilt! - myJavaSourcesThatHaveToBeRebuilt.addAll(findFilesThatDependedOnPath(previousBuildData, change.getSourcePath())); - break; - case MODIFIED: - //delete old output for this file since we must rebuild - for (Path existing : expandToExistingFiles(previousBuildData, change.getSourcePath())) { - Files.delete(existing); - } - // TODO expand the set of files that were modified because this was modified - myJavaSourcesThatHaveToBeRebuilt.add(change.getSourcePath()); - myJavaSourcesThatHaveToBeRebuilt.addAll(findFilesThatDependedOnPath(previousBuildData, change.getSourcePath())); - break; - } - } - - // Run J2CL with - // * mySourcesThatHaveToBeRebuilt - // * any native sources that applied to those, or were changed - // * existing classpath, plus OUR OWN CURRENT SOURCES - return; - } - List classpathDirs = Stream.concat( classpathHeaders.stream().flatMap(i -> i.getParentPaths().stream().map(Path::toFile)), extraClasspath.stream() @@ -155,12 +84,4 @@ public Task resolve(Project project, Config config) { } }; } - - private Collection findFilesThatDependedOnPath(Path previousBuildData, Path sourcePath) { - return null; - } - - private List expandToExistingFiles(Path previousBuildData, Path file) { - return null; - } } From 11159d2a7812fc09e5a99f6dbc647431738e6fc2 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Thu, 2 Mar 2023 10:33:51 -0600 Subject: [PATCH 09/12] More self-review --- .../j2cl/build/ChangedCachedPath.java | 12 ++++--- .../com/vertispan/j2cl/build/DiskCache.java | 7 ++-- .../java/com/vertispan/j2cl/build/Input.java | 11 +++++-- .../vertispan/j2cl/build/TaskScheduler.java | 13 ++++---- .../vertispan/j2cl/build/task/CachedPath.java | 7 ---- .../j2cl/build/task/ChangedCachedPath.java | 33 +++++++++++++++++++ .../com/vertispan/j2cl/build/task/Input.java | 19 +++-------- .../j2cl/mojo/AbstractBuildMojo.java | 2 +- .../j2cl/build/provided/BundleJarTask.java | 2 +- .../build/provided/ClosureBundleTask.java | 5 +-- .../j2cl/build/provided/ClosureTask.java | 2 +- .../j2cl/build/provided/StripSourcesTask.java | 6 ++-- 12 files changed, 73 insertions(+), 46 deletions(-) create mode 100644 build-caching/src/main/java/com/vertispan/j2cl/build/task/ChangedCachedPath.java diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java b/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java index 7047b064..2d36f46b 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java @@ -1,20 +1,22 @@ package com.vertispan.j2cl.build; import com.vertispan.j2cl.build.task.CachedPath; -import com.vertispan.j2cl.build.task.Input; import java.nio.file.Path; import java.util.Optional; -public class ChangedCachedPath implements Input.ChangedCachedPath { +/** + * Implementation of the ChangedCachedPath interface. + */ +public class ChangedCachedPath implements com.vertispan.j2cl.build.task.ChangedCachedPath { private final ChangeType type; private final Path sourcePath; - private final Optional newIfAny; + private final CachedPath newIfAny; public ChangedCachedPath(ChangeType type, Path sourcePath, CachedPath newPath) { this.type = type; this.sourcePath = sourcePath; - this.newIfAny = Optional.ofNullable(newPath); + this.newIfAny = newPath; } @Override @@ -29,6 +31,6 @@ public Path getSourcePath() { @Override public Optional getNewAbsolutePath() { - return newIfAny.map(CachedPath::getAbsolutePath); + return Optional.ofNullable(newIfAny).map(CachedPath::getAbsolutePath); } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java index fd0c855f..8044ee92 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java @@ -1,6 +1,5 @@ package com.vertispan.j2cl.build; -import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.vertispan.j2cl.build.impl.CollectedTaskInputs; import com.vertispan.j2cl.build.task.CachedPath; @@ -248,7 +247,9 @@ public Path getAbsolutePath() { return absoluteParent.resolve(sourcePath); } - @Override + /** + * Internal API, as this is not at this time used by any caller. + */ public FileHash getHash() { return hash; } @@ -379,7 +380,7 @@ private String taskSummaryContents(CollectedTaskInputs inputs) { return result; }) - .collect(Collectors.toList())); + .collect(Collectors.toUnmodifiableList())); src.setConfigs(inputs.getUsedConfigs()); diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java index dfc02ba2..ad7c63fe 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java @@ -1,5 +1,7 @@ package com.vertispan.j2cl.build; +import com.vertispan.j2cl.build.task.ChangedCachedPath; + import java.nio.file.Path; import java.nio.file.PathMatcher; import java.util.*; @@ -86,7 +88,7 @@ public Collection getFilesAndHashes() { public Collection getChanges() { return wrapped.getChanges().stream() .filter(entry -> Arrays.stream(filters).anyMatch(f -> f.matches(entry.getSourcePath()))) - .collect(Collectors.toList()); + .collect(Collectors.toUnmodifiableList()); } @Override @@ -124,7 +126,7 @@ public void setCurrentContents(TaskOutput contents) { } /** - * Internal API + * Internal API. * * Once a task is finished, we can let it be used by other tasks as an input. In * order for those tasks to execute incrementally, each particular Input must @@ -136,6 +138,11 @@ public void setBuildSpecificChanges(BuildSpecificChanges buildSpecificChanges) { this.buildSpecificChanges = buildSpecificChanges; } + /** + * Internal API. + * + * Creates a simple payload that describes this input, so it can be written to disk. + */ public TaskSummaryDiskFormat.InputDiskFormat makeDiskFormat() { TaskSummaryDiskFormat.InputDiskFormat out = new TaskSummaryDiskFormat.InputDiskFormat(); diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java index 0a538901..a741b4df 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java @@ -3,7 +3,6 @@ import com.google.gson.Gson; import com.vertispan.j2cl.build.impl.CollectedTaskInputs; import com.vertispan.j2cl.build.task.BuildLog; -import com.vertispan.j2cl.build.task.CachedPath; import com.vertispan.j2cl.build.task.OutputTypes; import com.vertispan.j2cl.build.task.TaskFactory; import com.vertispan.j2cl.build.task.TaskContext; @@ -27,9 +26,9 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static com.vertispan.j2cl.build.task.Input.ChangedCachedPath.ChangeType.ADDED; -import static com.vertispan.j2cl.build.task.Input.ChangedCachedPath.ChangeType.MODIFIED; -import static com.vertispan.j2cl.build.task.Input.ChangedCachedPath.ChangeType.REMOVED; +import static com.vertispan.j2cl.build.task.ChangedCachedPath.ChangeType.ADDED; +import static com.vertispan.j2cl.build.task.ChangedCachedPath.ChangeType.MODIFIED; +import static com.vertispan.j2cl.build.task.ChangedCachedPath.ChangeType.REMOVED; /** * Decides how much work to do, and when. Naive implementation just has a threadpool and does as much work @@ -240,7 +239,7 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult } return input.getFilesAndHashes().stream() - .map(entry -> new ChangedCachedPath(ADDED, entry.getSourcePath(), entry)).collect(Collectors.toList()); + .map(entry -> new ChangedCachedPath(ADDED, entry.getSourcePath(), entry)).collect(Collectors.toUnmodifiableList()); }); } taskDetails.getTask().execute(new TaskContext(result.outputDir(), log, latestResult.map(DiskCache.CacheResult::outputDir).orElse(null))); @@ -403,7 +402,7 @@ private boolean executeFinalTask(CollectedTaskInputs taskDetails, DiskCache.Cach }); } - private List diff(Map currentFiles, Map previousFiles) { + private List diff(Map currentFiles, Map previousFiles) { List changes = new ArrayList<>(); Set added = new HashSet<>(currentFiles.keySet()); added.removeAll(previousFiles.keySet()); @@ -417,7 +416,7 @@ private List diff(Map currentFiles, Map changed = new HashMap<>(currentFiles); + Map changed = new HashMap<>(currentFiles); changed.keySet().removeAll(added); changed.forEach((possiblyModifiedPath, entry) -> { if (!entry.getHash().asString().equals(previousFiles.get(possiblyModifiedPath))) { diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/CachedPath.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/CachedPath.java index 027241de..56fdedf7 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/CachedPath.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/CachedPath.java @@ -1,7 +1,5 @@ package com.vertispan.j2cl.build.task; -import io.methvin.watcher.hashing.FileHash; - import java.nio.file.Path; /** @@ -18,9 +16,4 @@ public interface CachedPath { * The absolute path to the file on disk. */ Path getAbsolutePath(); - - /** - * The current hash of the file, can be used to diff old and new inputs to see which specific paths changed. - */ - FileHash getHash(); } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/ChangedCachedPath.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/ChangedCachedPath.java new file mode 100644 index 00000000..6311b075 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/ChangedCachedPath.java @@ -0,0 +1,33 @@ +package com.vertispan.j2cl.build.task; + +import java.nio.file.Path; +import java.util.Optional; + +/** + * Describes a file that has changed since a previously successful invocation of this task. + * Analogous to {@link CachedPath}, except there might not be an absolute path for the + * current file, if it was deleted. + */ +public interface ChangedCachedPath { + enum ChangeType { + ADDED, + REMOVED, + MODIFIED; + } + + /** + * The type of change that took place for this path + */ + ChangeType changeType(); + + /** + * The path of this file, relative to either its old or new parent. + */ + Path getSourcePath(); + + /** + * If the file was not deleted, returns the absolute path to the "new" version + * of this file. + */ + Optional getNewAbsolutePath(); +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java index 507ebc9e..b26a6d0d 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java @@ -3,7 +3,6 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.util.Collection; -import java.util.Optional; public interface Input { /** @@ -24,21 +23,13 @@ public interface Input { */ Collection getFilesAndHashes(); + /** + * Public API for tasks. + * + * Gets the changed files of this input, with a path for the old and new file, and type of change. + */ Collection getChanges(); - interface ChangedCachedPath { - enum ChangeType { - ADDED, - REMOVED, - MODIFIED; - } - ChangeType changeType(); - - Path getSourcePath(); - - Optional getNewAbsolutePath(); - } - /** * Public API for tasks. * diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java index aa53cc53..a0c5ead1 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java @@ -85,7 +85,7 @@ public abstract class AbstractBuildMojo extends AbstractCacheMojo { @Parameter(defaultValue = "AVOID_MAVEN") private AnnotationProcessorMode annotationProcessorMode; - @Parameter(defaultValue = "true") + @Parameter(defaultValue = "false", property = "j2cl.incremental") private boolean incrementalEnabled; private List defaultDependencyReplacements = Arrays.asList( diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java index ab383c36..4ae34080 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java @@ -61,7 +61,7 @@ public Task resolve(Project project, Config config) { // our own JS after any dependencies that will be included have already loaded. List sourceOrder = new ArrayList<>(); Set pendingProjectKeys = jsSources.stream().map(i -> i.getProject().getKey()).collect(Collectors.toSet()); - List remaining = jsSources.stream().sorted(Comparator.comparing(i -> i.getProject().getDependencies().size())).collect(Collectors.toList()); + List remaining = jsSources.stream().sorted(Comparator.comparing(i -> i.getProject().getDependencies().size())).collect(Collectors.toUnmodifiableList()); while (!remaining.isEmpty()) { for (Iterator iterator = remaining.iterator(); iterator.hasNext(); ) { Input input = iterator.next(); diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java index 4f363eda..5b4a3c4e 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java @@ -18,6 +18,7 @@ import com.google.javascript.jscomp.transpile.BaseTranspiler; import com.google.javascript.jscomp.transpile.Transpiler; import com.vertispan.j2cl.build.task.CachedPath; +import com.vertispan.j2cl.build.task.ChangedCachedPath; import com.vertispan.j2cl.build.task.Config; import com.vertispan.j2cl.build.task.Input; import com.vertispan.j2cl.build.task.OutputTypes; @@ -138,8 +139,8 @@ public Task resolve(Project project, Config config) { // create new dep info for any added/modified file for (Input jsInput : js) { - for (Input.ChangedCachedPath change : jsInput.getChanges()) { - if (change.changeType() == Input.ChangedCachedPath.ChangeType.REMOVED) { + for (ChangedCachedPath change : jsInput.getChanges()) { + if (change.changeType() == ChangedCachedPath.ChangeType.REMOVED) { depInfoMap.remove(change.getSourcePath().toString()); } else { // ADD or MODIFY diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java index ea1c9a96..f4069979 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java @@ -232,7 +232,7 @@ public void execute(TaskContext context) throws Exception { js = Closure.mapFromInputs(jsSources); } if (sources != null) { - for (Path path : jsSources.stream().map(Input::getParentPaths).flatMap(Collection::stream).collect(Collectors.toList())) { + for (Path path : jsSources.stream().map(Input::getParentPaths).flatMap(Collection::stream).collect(Collectors.toUnmodifiableList())) { FileUtils.copyDirectory(path.toFile(), sources); } } diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java index 34255dd5..5e4aab6d 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java @@ -48,11 +48,11 @@ public Task resolve(Project project, Config config) { Function.identity() )); //only process changed files, copy unchanged ones - for (Input.ChangedCachedPath change : inputSources.getChanges()) { + for (ChangedCachedPath change : inputSources.getChanges()) { // remove the file, since it was changed in some way unmodified.remove(change.getSourcePath()); - if (change.changeType() != Input.ChangedCachedPath.ChangeType.REMOVED) { + if (change.changeType() != ChangedCachedPath.ChangeType.REMOVED) { // track the files we actually need to process filesToProcess.add(makeFileInfo(change)); } @@ -71,7 +71,7 @@ public Task resolve(Project project, Config config) { }; } - private SourceUtils.FileInfo makeFileInfo(Input.ChangedCachedPath change) { + private SourceUtils.FileInfo makeFileInfo(ChangedCachedPath change) { assert change.getNewAbsolutePath().isPresent() : "Can't make a FileInfo if it no longer exists"; return SourceUtils.FileInfo.create(change.getNewAbsolutePath().get().toString(), change.getSourcePath().toString()); } From 4f3ae6dd02a469f46600a6c615d27d856c7cb603 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Tue, 7 Mar 2023 22:01:31 -0600 Subject: [PATCH 10/12] Allow modifications of this list --- .../java/com/vertispan/j2cl/build/provided/BundleJarTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java index 4ae34080..ab383c36 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java @@ -61,7 +61,7 @@ public Task resolve(Project project, Config config) { // our own JS after any dependencies that will be included have already loaded. List sourceOrder = new ArrayList<>(); Set pendingProjectKeys = jsSources.stream().map(i -> i.getProject().getKey()).collect(Collectors.toSet()); - List remaining = jsSources.stream().sorted(Comparator.comparing(i -> i.getProject().getDependencies().size())).collect(Collectors.toUnmodifiableList()); + List remaining = jsSources.stream().sorted(Comparator.comparing(i -> i.getProject().getDependencies().size())).collect(Collectors.toList()); while (!remaining.isEmpty()) { for (Iterator iterator = remaining.iterator(); iterator.hasNext(); ) { Input input = iterator.next(); From 3c9da784df5746ce65493f84b86a03e83cc0ebe1 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 8 Mar 2023 12:15:17 -0600 Subject: [PATCH 11/12] Correctly use latest, not oldest cache references, don't incr build if new deps --- .../java/com/vertispan/j2cl/build/Input.java | 2 +- .../j2cl/build/LocalProjectBuildCache.java | 5 ++- .../vertispan/j2cl/build/TaskScheduler.java | 40 +++++++++++++++---- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java index ad7c63fe..1712e4de 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java @@ -135,7 +135,7 @@ public void setCurrentContents(TaskOutput contents) { * executes. */ public void setBuildSpecificChanges(BuildSpecificChanges buildSpecificChanges) { - this.buildSpecificChanges = buildSpecificChanges; + this.buildSpecificChanges = BuildSpecificChanges.memoize(buildSpecificChanges); } /** diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java index ba06c899..411895c3 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java @@ -6,6 +6,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; +import java.util.Comparator; import java.util.Optional; public class LocalProjectBuildCache { @@ -25,7 +26,7 @@ public void markLocalSuccess(Project project, String task, Path taskDir) { Files.createDirectories(localTaskDir); Files.write(localTaskDir.resolve(timestamp), Collections.singleton(taskDir.toString())); - Files.list(localTaskDir).sorted().skip(5).forEach(old -> { + Files.list(localTaskDir).sorted(Comparator.naturalOrder().reversed()).skip(5).forEach(old -> { try { Files.delete(old); } catch (IOException e) { @@ -44,7 +45,7 @@ public void markLocalSuccess(Project project, String task, Path taskDir) { public Optional getLatestResult(Project project, String task) { try { Path taskDir = cacheDir.toPath().resolve(project.getKey()).resolve(task); - Optional latest = Files.list(taskDir).sorted().findFirst(); + Optional latest = Files.list(taskDir).max(Comparator.naturalOrder()); if (latest.isPresent()) { String path = Files.readAllLines(latest.get()).get(0); diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java index a741b4df..fa012b49 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java @@ -225,10 +225,25 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult Optional latestResult = buildCache.getLatestResult(taskDetails.getProject(), taskDetails.getTaskFactory().getOutputType()); final TaskSummaryDiskFormat taskSummaryDiskFormat = latestResult.map(TaskScheduler.this::getTaskSummary).orElse(null); - // deserialize its inputs list and use its outputs below - for (Input input : taskDetails.getInputs()) { - input.setBuildSpecificChanges(() -> { - if (latestResult.isPresent() && taskSummaryDiskFormat != null) { + + if (taskSummaryDiskFormat == null) { + latestResult = Optional.empty(); + } + + + // Update any existing input to reflect what has changed + if (latestResult.isPresent()) { + for (TaskSummaryDiskFormat.InputDiskFormat onDiskInput : taskSummaryDiskFormat.getInputs()) { + // if this input is not present any more, we cannot build incrementally + if (taskDetails.getInputs().stream().noneMatch(currentInput -> + currentInput.getProject().getKey().equals(onDiskInput.getProjectKey()) + && currentInput.getOutputType().equals(onDiskInput.getOutputType()) + )) { + latestResult = Optional.empty(); + } + } + for (Input input : taskDetails.getInputs()) { + input.setBuildSpecificChanges(() -> { Optional prevInput = taskSummaryDiskFormat.getInputs().stream() .filter(i -> i.getProjectKey().equals(input.getProject().getKey())) .filter(i -> i.getOutputType().equals(input.getOutputType())) @@ -236,12 +251,21 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult if (prevInput.isPresent()) { return diff(input.getFilesAndHashes().stream().collect(Collectors.toMap(e -> e.getSourcePath().toString(), Function.identity())), prevInput.get().getFileHashes()); } - } - return input.getFilesAndHashes().stream() - .map(entry -> new ChangedCachedPath(ADDED, entry.getSourcePath(), entry)).collect(Collectors.toUnmodifiableList()); - }); + return input.getFilesAndHashes().stream() + .map(entry -> new ChangedCachedPath(ADDED, entry.getSourcePath(), entry)).collect(Collectors.toUnmodifiableList()); + }); + } + } else { + for (Input input : taskDetails.getInputs()) { + input.setBuildSpecificChanges(() -> + input.getFilesAndHashes().stream() + .map(entry -> new ChangedCachedPath(ADDED, entry.getSourcePath(), entry)) + .collect(Collectors.toUnmodifiableList()) + ); + } } + taskDetails.getTask().execute(new TaskContext(result.outputDir(), log, latestResult.map(DiskCache.CacheResult::outputDir).orElse(null))); if (Thread.currentThread().isInterrupted()) { // Tried and failed to be canceled, so even though we were successful, some files might From 07fd4962a8b763bfcf1fd6d6f7d6173b3bdf19b4 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 19 Jul 2023 10:42:44 -0500 Subject: [PATCH 12/12] Clean up paths, attempt to resolve windows issue --- .../java/com/vertispan/j2cl/build/LocalProjectBuildCache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java index 411895c3..9725c9a5 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java @@ -22,7 +22,7 @@ public LocalProjectBuildCache(File cacheDir, DiskCache cache) { public void markLocalSuccess(Project project, String task, Path taskDir) { String timestamp = String.valueOf(System.currentTimeMillis()); try { - Path localTaskDir = cacheDir.toPath().resolve(project.getKey()).resolve(task); + Path localTaskDir = cacheDir.toPath().resolve(project.getKey().replaceAll("[^\\-_a-zA-Z0-9.]", "-")).resolve(task); Files.createDirectories(localTaskDir); Files.write(localTaskDir.resolve(timestamp), Collections.singleton(taskDir.toString())); @@ -44,7 +44,7 @@ public void markLocalSuccess(Project project, String task, Path taskDir) { public Optional getLatestResult(Project project, String task) { try { - Path taskDir = cacheDir.toPath().resolve(project.getKey()).resolve(task); + Path taskDir = cacheDir.toPath().resolve(project.getKey().replaceAll("[^\\-_a-zA-Z0-9.]", "-")).resolve(task); Optional latest = Files.list(taskDir).max(Comparator.naturalOrder()); if (latest.isPresent()) { String path = Files.readAllLines(latest.get()).get(0);