diff --git a/build-caching/pom.xml b/build-caching/pom.xml index a6dc4bcc..93f5324c 100644 --- a/build-caching/pom.xml +++ b/build-caching/pom.xml @@ -7,7 +7,7 @@ com.vertispan.j2cl j2cl-tools - 0.21-SNAPSHOT + 0.22-SNAPSHOT build-caching @@ -23,6 +23,13 @@ 0.15.0 + + + com.google.code.gson + gson + 2.8.6 + + junit diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java b/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java index 868ac139..a83e559b 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/BuildService.java @@ -1,7 +1,6 @@ package com.vertispan.j2cl.build; import com.vertispan.j2cl.build.impl.CollectedTaskInputs; -import com.vertispan.j2cl.build.task.BuildLog; import com.vertispan.j2cl.build.task.OutputTypes; import com.vertispan.j2cl.build.task.TaskFactory; diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java b/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java new file mode 100644 index 00000000..2d36f46b --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/ChangedCachedPath.java @@ -0,0 +1,36 @@ +package com.vertispan.j2cl.build; + +import com.vertispan.j2cl.build.task.CachedPath; + +import java.nio.file.Path; +import java.util.Optional; + +/** + * Implementation of the ChangedCachedPath interface. + */ +public class ChangedCachedPath implements com.vertispan.j2cl.build.task.ChangedCachedPath { + private final ChangeType type; + private final Path sourcePath; + private final CachedPath newIfAny; + + public ChangedCachedPath(ChangeType type, Path sourcePath, CachedPath newPath) { + this.type = type; + this.sourcePath = sourcePath; + this.newIfAny = newPath; + } + + @Override + public ChangeType changeType() { + return type; + } + + @Override + public Path getSourcePath() { + return sourcePath; + } + + @Override + public Optional getNewAbsolutePath() { + return Optional.ofNullable(newIfAny).map(CachedPath::getAbsolutePath); + } +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java index 20ffb948..8afd270b 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/DefaultDiskCache.java @@ -29,28 +29,8 @@ public DefaultDiskCache(File cacheDir, Executor executor) throws IOException { } @Override - protected Path taskDir(CollectedTaskInputs inputs) { - String projectName = inputs.getProject().getKey(); - Murmur3F hash = new Murmur3F(); - - hash.update(inputs.getTaskFactory().getClass().toString().getBytes(StandardCharsets.UTF_8)); - hash.update(inputs.getTaskFactory().getTaskName().getBytes(StandardCharsets.UTF_8)); - hash.update(inputs.getTaskFactory().getVersion().getBytes(StandardCharsets.UTF_8)); - - for (Input input : inputs.getInputs()) { - input.updateHash(hash); - } - - for (Map.Entry entry : inputs.getUsedConfigs().entrySet()) { - hash.update(entry.getKey().getBytes(StandardCharsets.UTF_8)); - if (entry.getValue() == null) { - hash.update(0); - } else { - hash.update(entry.getValue().getBytes(StandardCharsets.UTF_8)); - } - } - - return cacheDir.toPath().resolve(projectName.replaceAll("[^\\-_a-zA-Z0-9.]", "-")).resolve(hash.getValueHexString() + "-" + inputs.getTaskFactory().getOutputType()); + protected Path taskDir(String projectName, String hashString, String outputType) { + return cacheDir.toPath().resolve(projectName.replaceAll("[^\\-_a-zA-Z0-9.]", "-")).resolve(hashString + "-" + outputType); } @Override @@ -72,4 +52,9 @@ protected Path logFile(Path taskDir) { protected Path outputDir(Path taskDir) { return taskDir.resolve("results"); } + + @Override + protected Path cacheSummary(Path taskDir) { + return taskDir.resolve("cacheSummary.json"); + } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java index f60cc46b..8044ee92 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/DiskCache.java @@ -1,30 +1,32 @@ package com.vertispan.j2cl.build; +import com.google.gson.GsonBuilder; import com.vertispan.j2cl.build.impl.CollectedTaskInputs; import com.vertispan.j2cl.build.task.CachedPath; import io.methvin.watcher.PathUtils; import io.methvin.watcher.hashing.FileHash; import io.methvin.watcher.hashing.FileHasher; +import io.methvin.watcher.hashing.Murmur3F; import io.methvin.watchservice.MacOSXListeningWatchService; import io.methvin.watchservice.WatchablePath; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.nio.file.WatchService; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; import java.time.Instant; -import java.time.temporal.TemporalAmount; -import java.time.temporal.TemporalUnit; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; +import java.util.stream.Collectors; /** - * Manages the cached task inputs and outputs. + * Manages the cached task inputs and outputs, without direct knowledge of the project or task apis. */ public abstract class DiskCache { private static final boolean IS_MAC = System.getProperty("os.name").toLowerCase().contains("mac"); @@ -38,6 +40,10 @@ public CacheResult(Path taskDir) { this.taskDir = taskDir; } + public Path taskDir() { + return taskDir; + } + public Path logFile() { //TODO finish building a logger that will write to this return DiskCache.this.logFile(taskDir); @@ -54,6 +60,10 @@ public TaskOutput output() { return taskOutput; } + public Path cachedSummary() { + return DiskCache.this.cacheSummary(taskDir); + } + public void markSuccess() { markFinished(this); runningTasks.remove(taskDir); @@ -237,7 +247,9 @@ public Path getAbsolutePath() { return absoluteParent.resolve(sourcePath); } - @Override + /** + * Internal API, as this is not at this time used by any caller. + */ public FileHash getHash() { return hash; } @@ -340,16 +352,57 @@ private void deleteRecursively(Path path) throws IOException { } } - protected abstract Path taskDir(CollectedTaskInputs inputs); + private String taskSummaryContents(CollectedTaskInputs inputs) { + TaskSummaryDiskFormat src = new TaskSummaryDiskFormat(); + src.setProjectKey(inputs.getProject().getKey()); + src.setOutputType(inputs.getTaskFactory().getOutputType()); + src.setTaskImpl(inputs.getTaskFactory().getClass().getName()); + src.setTaskImplVersion(inputs.getTaskFactory().getVersion()); + + src.setInputs(inputs.getInputs().stream() + .map(Input::makeDiskFormat) + .collect(Collectors.groupingBy(i -> i.getProjectKey() + "-" + i.getOutputType())) + .values().stream() + .map(list -> { + TaskSummaryDiskFormat.InputDiskFormat result = new TaskSummaryDiskFormat.InputDiskFormat(); + result.setProjectKey(list.get(0).getProjectKey()); + result.setOutputType(list.get(0).getOutputType()); + + result.setFileHashes( + list.stream().flatMap(i -> i.getFileHashes().entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (left, right) -> { + if (left.equals(right)) { + return left; + } + throw new IllegalStateException("Two hashes for one file! " + left + " vs " + right); + })) + ); + + return result; + }) + .collect(Collectors.toUnmodifiableList())); + + src.setConfigs(inputs.getUsedConfigs()); + + return new GsonBuilder().serializeNulls().setPrettyPrinting().create().toJson(src); + } + + protected abstract Path taskDir(String projectName, String hashString, String outputType); + protected abstract Path successMarker(Path taskDir); protected abstract Path failureMarker(Path taskDir); protected abstract Path logFile(Path taskDir); protected abstract Path outputDir(Path taskDir); + protected abstract Path cacheSummary(Path taskDir); interface Listener { + /** Ready for the current listener to do the work */ void onReady(CacheResult result); + /** Someone else did it, but failed for some reason, not re-runnable */ void onFailure(CacheResult result); + /** Someone else tried to do it, but ran into an error, possibly recoverable if we try again */ void onError(Throwable throwable); + /** Someone else finished it, successfully, notify listeners */ void onSuccess(CacheResult result); } public class PendingCacheResult implements Cancelable { @@ -431,7 +484,14 @@ private synchronized void failure() { */ public void waitForTask(CollectedTaskInputs taskDetails, Listener listener) { assert taskDetails.getInputs().stream().allMatch(Input::hasContents); - final Path taskDir = taskDir(taskDetails); + + Murmur3F murmur3F = new Murmur3F(); + byte[] taskSummaryContents = taskSummaryContents(taskDetails).getBytes(StandardCharsets.UTF_8); + murmur3F.update(taskSummaryContents); + String hashString = murmur3F.getValueHexString(); + + final Path taskDir = taskDir(taskDetails.getProject().getKey(), hashString, taskDetails.getTaskFactory().getOutputType()); + PendingCacheResult cancelable = new PendingCacheResult(taskDir, listener); taskFutures.computeIfAbsent(taskDir, ignore -> Collections.newSetFromMap(new ConcurrentHashMap<>())).add(cancelable); try { @@ -449,6 +509,7 @@ public void waitForTask(CollectedTaskInputs taskDetails, Listener listener) { // caller can begin work right away Files.createDirectory(outputDir); Files.createFile(logFile(taskDir)); + Files.write(cacheSummary(taskDir), taskSummaryContents); cancelable.ready(); return; } @@ -546,4 +607,13 @@ public void markFailed(CacheResult failedResult) { } } + public Optional getCacheResult(Path taskDir) { + if (Files.exists(taskDir) || Files.exists(successMarker(taskDir))) { + CacheResult result = new CacheResult(taskDir); + knownOutputs.computeIfAbsent(taskDir, this::makeOutput); + return Optional.of(result); + } + return Optional.empty(); + } + } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java index eb51b887..1712e4de 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/Input.java @@ -1,8 +1,7 @@ package com.vertispan.j2cl.build; -import io.methvin.watcher.hashing.Murmur3F; +import com.vertispan.j2cl.build.task.ChangedCachedPath; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.PathMatcher; import java.util.*; @@ -18,10 +17,29 @@ * interested in, and take the hash of the hashes to represent */ public class Input implements com.vertispan.j2cl.build.task.Input { + public interface BuildSpecificChanges { + List compute(); + static BuildSpecificChanges memoize(BuildSpecificChanges changes) { + return new BuildSpecificChanges() { + private List results; + @Override + public List compute() { + if (results == null) { + List computed = changes.compute(); + assert computed != null; + results = new ArrayList<>(computed); + } + + return results; + } + }; + } + } private final Project project; private final String outputType; private TaskOutput contents; + private BuildSpecificChanges buildSpecificChanges; public Input(Project project, String outputType) { this.project = project; @@ -66,6 +84,13 @@ public Collection getFilesAndHashes() { .collect(Collectors.toUnmodifiableList()); } + @Override + public Collection getChanges() { + return wrapped.getChanges().stream() + .filter(entry -> Arrays.stream(filters).anyMatch(f -> f.matches(entry.getSourcePath()))) + .collect(Collectors.toUnmodifiableList()); + } + @Override public com.vertispan.j2cl.build.task.Project getProject() { return wrapped.getProject(); @@ -103,17 +128,37 @@ public void setCurrentContents(TaskOutput contents) { /** * Internal API. * - * Updates the given hash object with the filtered file inputs - their paths and their - * hashes, so that if files are moved or changed we change the hash value, but we don't - * re-hash each file every time we ask. + * Once a task is finished, we can let it be used by other tasks as an input. In + * order for those tasks to execute incrementally, each particular Input must + * only have changed relative to the last time that its owner task ran successfully. + * Instead of being set all at once, this is re-assigned before each task actually + * executes. */ - public void updateHash(Murmur3F hash) { - for (DiskCache.CacheEntry fileAndHash : getFilesAndHashes()) { - hash.update(fileAndHash.getSourcePath().toString().getBytes(StandardCharsets.UTF_8)); - hash.update(fileAndHash.getHash().asBytes()); - } + public void setBuildSpecificChanges(BuildSpecificChanges buildSpecificChanges) { + this.buildSpecificChanges = BuildSpecificChanges.memoize(buildSpecificChanges); } + /** + * Internal API. + * + * Creates a simple payload that describes this input, so it can be written to disk. + */ + public TaskSummaryDiskFormat.InputDiskFormat makeDiskFormat() { + TaskSummaryDiskFormat.InputDiskFormat out = new TaskSummaryDiskFormat.InputDiskFormat(); + + out.setProjectKey(getProject().getKey()); + out.setOutputType(getOutputType()); + + out.setFileHashes(getFilesAndHashes().stream().collect(Collectors.toMap( + e -> e.getSourcePath().toString(), + e -> e.getHash().asString() + ))); + + return out; + } + + + @Override public Project getProject() { return project; @@ -139,6 +184,14 @@ public Collection getFilesAndHashes() { return contents.filesAndHashes(); } + @Override + public Collection getChanges() { + if (buildSpecificChanges == null) { + throw new NullPointerException("Changes not yet provided " + this); + } + return buildSpecificChanges.compute(); + } + @Override public String toString() { return "Input{" + diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java b/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java new file mode 100644 index 00000000..9725c9a5 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/LocalProjectBuildCache.java @@ -0,0 +1,63 @@ +package com.vertispan.j2cl.build; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Comparator; +import java.util.Optional; + +public class LocalProjectBuildCache { + private final File cacheDir; + private final DiskCache cache; + + + public LocalProjectBuildCache(File cacheDir, DiskCache cache) { + this.cacheDir = cacheDir; + this.cache = cache; + } + + public void markLocalSuccess(Project project, String task, Path taskDir) { + String timestamp = String.valueOf(System.currentTimeMillis()); + try { + Path localTaskDir = cacheDir.toPath().resolve(project.getKey().replaceAll("[^\\-_a-zA-Z0-9.]", "-")).resolve(task); + Files.createDirectories(localTaskDir); + Files.write(localTaskDir.resolve(timestamp), Collections.singleton(taskDir.toString())); + + Files.list(localTaskDir).sorted(Comparator.naturalOrder().reversed()).skip(5).forEach(old -> { + try { + Files.delete(old); + } catch (IOException e) { + // we don't care at all about failure, as long as it sometimes works. there could be races + // to delete, or someone just cleaned, etc. + e.printStackTrace(); + } + }); + } catch (IOException e) { + // ignore, cache just won't work for now + //TODO log this, the user will be irritated + e.printStackTrace(); + } + } + + public Optional getLatestResult(Project project, String task) { + try { + Path taskDir = cacheDir.toPath().resolve(project.getKey().replaceAll("[^\\-_a-zA-Z0-9.]", "-")).resolve(task); + Optional latest = Files.list(taskDir).max(Comparator.naturalOrder()); + if (latest.isPresent()) { + String path = Files.readAllLines(latest.get()).get(0); + + return cache.getCacheResult(Paths.get(path)); + } else { + return Optional.empty(); + } + } catch (IOException e) { + // Probably file doesn't exist. This seems terrible and dirty... but on the other hand, just about any + // failure to read could mean "look just go do it from scratch, since that is always possible". + return Optional.empty(); + } + } + +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java b/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java index 4888103d..af8a3d4b 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/PropertyTrackingConfig.java @@ -77,7 +77,7 @@ private File useFileConfig(ConfigValueProvider.ConfigNode node) { } catch (IOException e) { throw new UncheckedIOException("Failed to hash file contents " + value, e); } - useKey(node.getPath(), hash); + useKey(node.getPath(), "File with hash " + hash); return value; } private void useKey(String path, String value) { @@ -227,4 +227,11 @@ public Path getWebappDirectory() { } return Paths.get(s); } + + @Override + public boolean isIncrementalEnabled() { + // TODO once we have awesome tests for this, consider skipping the cache. Could be dangerous, + // in the case of externally provided buggy tasks. + return getString("incrementalEnabled").equalsIgnoreCase("true"); + } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java index 2daeacd5..fa012b49 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskScheduler.java @@ -1,5 +1,6 @@ package com.vertispan.j2cl.build; +import com.google.gson.Gson; import com.vertispan.j2cl.build.impl.CollectedTaskInputs; import com.vertispan.j2cl.build.task.BuildLog; import com.vertispan.j2cl.build.task.OutputTypes; @@ -7,12 +8,16 @@ import com.vertispan.j2cl.build.task.TaskContext; import java.io.FileNotFoundException; -import java.nio.file.Path; +import java.io.FileReader; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; @@ -21,6 +26,10 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static com.vertispan.j2cl.build.task.ChangedCachedPath.ChangeType.ADDED; +import static com.vertispan.j2cl.build.task.ChangedCachedPath.ChangeType.MODIFIED; +import static com.vertispan.j2cl.build.task.ChangedCachedPath.ChangeType.REMOVED; + /** * Decides how much work to do, and when. Naive implementation just has a threadpool and does as much work * as possible at a time, depth first from the tree of tasks. A smarter impl could try to do work with many @@ -32,6 +41,7 @@ public class TaskScheduler { private final Executor executor; private final DiskCache diskCache; + private final LocalProjectBuildCache buildCache; private final BuildLog buildLog; // This is technically incorrect, but covers the current use cases, and should fail loudly if this assumption @@ -50,14 +60,15 @@ public class TaskScheduler { * * Caller is responsible for shutting down the executor service - canceling ongoing work * should be supported, but presently isn't. - * - * @param executor executor to submit work to, to be performed off thread + * @param executor executor to submit work to, to be performed off thread * @param diskCache cache to read results from, and save new results to + * @param buildCache * @param buildLog log to write details to about work being performed */ - public TaskScheduler(Executor executor, DiskCache diskCache, BuildLog buildLog) { + public TaskScheduler(Executor executor, DiskCache diskCache, LocalProjectBuildCache buildCache, BuildLog buildLog) { this.executor = executor; this.diskCache = diskCache; + this.buildCache = buildCache; this.buildLog = buildLog; } @@ -211,7 +222,51 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult } try { long start = System.currentTimeMillis(); - taskDetails.getTask().execute(new TaskContext(result.outputDir(), log)); + + Optional latestResult = buildCache.getLatestResult(taskDetails.getProject(), taskDetails.getTaskFactory().getOutputType()); + final TaskSummaryDiskFormat taskSummaryDiskFormat = latestResult.map(TaskScheduler.this::getTaskSummary).orElse(null); + + if (taskSummaryDiskFormat == null) { + latestResult = Optional.empty(); + } + + + // Update any existing input to reflect what has changed + if (latestResult.isPresent()) { + for (TaskSummaryDiskFormat.InputDiskFormat onDiskInput : taskSummaryDiskFormat.getInputs()) { + // if this input is not present any more, we cannot build incrementally + if (taskDetails.getInputs().stream().noneMatch(currentInput -> + currentInput.getProject().getKey().equals(onDiskInput.getProjectKey()) + && currentInput.getOutputType().equals(onDiskInput.getOutputType()) + )) { + latestResult = Optional.empty(); + } + } + for (Input input : taskDetails.getInputs()) { + input.setBuildSpecificChanges(() -> { + Optional prevInput = taskSummaryDiskFormat.getInputs().stream() + .filter(i -> i.getProjectKey().equals(input.getProject().getKey())) + .filter(i -> i.getOutputType().equals(input.getOutputType())) + .findAny(); + if (prevInput.isPresent()) { + return diff(input.getFilesAndHashes().stream().collect(Collectors.toMap(e -> e.getSourcePath().toString(), Function.identity())), prevInput.get().getFileHashes()); + } + + return input.getFilesAndHashes().stream() + .map(entry -> new ChangedCachedPath(ADDED, entry.getSourcePath(), entry)).collect(Collectors.toUnmodifiableList()); + }); + } + } else { + for (Input input : taskDetails.getInputs()) { + input.setBuildSpecificChanges(() -> + input.getFilesAndHashes().stream() + .map(entry -> new ChangedCachedPath(ADDED, entry.getSourcePath(), entry)) + .collect(Collectors.toUnmodifiableList()) + ); + } + } + + taskDetails.getTask().execute(new TaskContext(result.outputDir(), log, latestResult.map(DiskCache.CacheResult::outputDir).orElse(null))); if (Thread.currentThread().isInterrupted()) { // Tried and failed to be canceled, so even though we were successful, some files might // have been deleted. Continue deleting contents @@ -222,6 +277,7 @@ private void executeTask(CollectedTaskInputs taskDetails, DiskCache.CacheResult if (elapsedMillis > 5) { buildLog.info("Finished " + taskDetails.getDebugName() + " in " + elapsedMillis + "ms"); } + buildCache.markLocalSuccess(taskDetails.getProject(), taskDetails.getTaskFactory().getOutputType(), result.taskDir()); result.markSuccess(); } catch (Throwable exception) { @@ -324,9 +380,10 @@ private void scheduleMoreWork(DiskCache.CacheResult cacheResult) { // When something finishes, remove it from the various dependency lists and see if we can run the loop again with more work. // Presently this could be called multiple times, so we check if already removed if (tasks.complete(taskDetails)) { + TaskOutput output = cacheResult.output(); for (Input input : allInputs.computeIfAbsent(taskDetails.getAsInput(), ignore -> Collections.emptyList())) { // since we don't support running more than one thing at a time, this will not change data out from under a running task - input.setCurrentContents(cacheResult.output()); + input.setCurrentContents(output); //TODO This maybe can race with the check on line 84, and we break since the input isn't ready yet } @@ -349,7 +406,9 @@ private boolean executeFinalTask(CollectedTaskInputs taskDetails, DiskCache.Cach try { //TODO Make sure that we want to write this to _only_ the current log, and not also to any file //TODO Also be sure to write a prefix automatically - ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog)); + + // TODO also consider if lastSuccessfulPath should be null for final tasks + ((TaskFactory.FinalOutputTask) taskDetails.getTask()).finish(new TaskContext(cacheResult.outputDir(), buildLog, null)); buildLog.info("Finished final task " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms"); } catch (Throwable t) { buildLog.error("FAILED " + taskDetails.getDebugName() + " in " + (System.currentTimeMillis() - start) + "ms",t); @@ -366,4 +425,37 @@ private boolean executeFinalTask(CollectedTaskInputs taskDetails, DiskCache.Cach }); }); } + + private List diff(Map currentFiles, Map previousFiles) { + List changes = new ArrayList<>(); + Set added = new HashSet<>(currentFiles.keySet()); + added.removeAll(previousFiles.keySet()); + added.forEach(newPath -> { + changes.add(new ChangedCachedPath(ADDED, Paths.get(newPath), currentFiles.get(newPath))); + }); + + Set removed = new HashSet<>(previousFiles.keySet()); + removed.removeAll(currentFiles.keySet()); + removed.forEach(removedPath -> { + changes.add(new ChangedCachedPath(REMOVED, Paths.get(removedPath), null)); + }); + + Map changed = new HashMap<>(currentFiles); + changed.keySet().removeAll(added); + changed.forEach((possiblyModifiedPath, entry) -> { + if (!entry.getHash().asString().equals(previousFiles.get(possiblyModifiedPath))) { + changes.add(new ChangedCachedPath(MODIFIED, Paths.get(possiblyModifiedPath), entry)); + } + }); + + return changes; + } + + private TaskSummaryDiskFormat getTaskSummary(DiskCache.CacheResult latestResult) { + try { + return new Gson().fromJson(new FileReader(latestResult.cachedSummary().toFile()), TaskSummaryDiskFormat.class); + } catch (FileNotFoundException ex) { + return null; + } + } } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/TaskSummaryDiskFormat.java b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskSummaryDiskFormat.java new file mode 100644 index 00000000..fe18e924 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/TaskSummaryDiskFormat.java @@ -0,0 +1,93 @@ +package com.vertispan.j2cl.build; + +import java.util.List; +import java.util.Map; + +public class TaskSummaryDiskFormat { + private String projectKey; + private String outputType; + private String taskImpl; + private String taskImplVersion; + + private List inputs; + + private Map configs; + + public String getProjectKey() { + return projectKey; + } + + public void setProjectKey(String projectKey) { + this.projectKey = projectKey; + } + + public String getOutputType() { + return outputType; + } + + public void setOutputType(String outputType) { + this.outputType = outputType; + } + + public String getTaskImpl() { + return taskImpl; + } + + public void setTaskImpl(String taskImpl) { + this.taskImpl = taskImpl; + } + + public String getTaskImplVersion() { + return taskImplVersion; + } + + public void setTaskImplVersion(String taskImplVersion) { + this.taskImplVersion = taskImplVersion; + } + + public List getInputs() { + return inputs; + } + + public void setInputs(List inputs) { + this.inputs = inputs; + } + + public Map getConfigs() { + return configs; + } + + public void setConfigs(Map configs) { + this.configs = configs; + } + + public static class InputDiskFormat { + private String projectKey; + private String outputType; + private Map fileHashes; + + public String getProjectKey() { + return projectKey; + } + + public void setProjectKey(String projectKey) { + this.projectKey = projectKey; + } + + public String getOutputType() { + return outputType; + } + + public void setOutputType(String outputType) { + this.outputType = outputType; + } + + public Map getFileHashes() { + return fileHashes; + } + + public void setFileHashes(Map fileHashes) { + this.fileHashes = fileHashes; + } + } +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/CachedPath.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/CachedPath.java index 027241de..56fdedf7 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/CachedPath.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/CachedPath.java @@ -1,7 +1,5 @@ package com.vertispan.j2cl.build.task; -import io.methvin.watcher.hashing.FileHash; - import java.nio.file.Path; /** @@ -18,9 +16,4 @@ public interface CachedPath { * The absolute path to the file on disk. */ Path getAbsolutePath(); - - /** - * The current hash of the file, can be used to diff old and new inputs to see which specific paths changed. - */ - FileHash getHash(); } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/ChangedCachedPath.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/ChangedCachedPath.java new file mode 100644 index 00000000..6311b075 --- /dev/null +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/ChangedCachedPath.java @@ -0,0 +1,33 @@ +package com.vertispan.j2cl.build.task; + +import java.nio.file.Path; +import java.util.Optional; + +/** + * Describes a file that has changed since a previously successful invocation of this task. + * Analogous to {@link CachedPath}, except there might not be an absolute path for the + * current file, if it was deleted. + */ +public interface ChangedCachedPath { + enum ChangeType { + ADDED, + REMOVED, + MODIFIED; + } + + /** + * The type of change that took place for this path + */ + ChangeType changeType(); + + /** + * The path of this file, relative to either its old or new parent. + */ + Path getSourcePath(); + + /** + * If the file was not deleted, returns the absolute path to the "new" version + * of this file. + */ + Optional getNewAbsolutePath(); +} diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Config.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Config.java index 77c1efdb..9e40d98d 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Config.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Config.java @@ -48,4 +48,17 @@ public interface Config { */ Path getWebappDirectory(); + /** + * Allow tasks to know if they need to do the work to build incrementally. Tasks + * should decide for themselves what inputs they might need or what work they + * might do based on this, plus if sources of a given project are mapped or not. + *

+ * For example, if incremental is cheap, might as well always do it, if not, only + * do it if all markers suggest it is a good idea. However, if this flag is false, + * incremental should never be attempted (could be a bug in it, etc). + * + * @return true if incremental is enabled, false if it should be skipped + */ + boolean isIncrementalEnabled(); + } diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java index 8d8db900..b26a6d0d 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/Input.java @@ -23,6 +23,13 @@ public interface Input { */ Collection getFilesAndHashes(); + /** + * Public API for tasks. + * + * Gets the changed files of this input, with a path for the old and new file, and type of change. + */ + Collection getChanges(); + /** * Public API for tasks. * diff --git a/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java b/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java index 59575933..e9bb4fd1 100644 --- a/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java +++ b/build-caching/src/main/java/com/vertispan/j2cl/build/task/TaskContext.java @@ -1,14 +1,17 @@ package com.vertispan.j2cl.build.task; import java.nio.file.Path; +import java.util.Optional; public class TaskContext implements BuildLog { private final Path path; private final BuildLog log; + private final Path lastSuccessfulPath; - public TaskContext(Path path, BuildLog log) { + public TaskContext(Path path, BuildLog log, Path lastSuccessfulPath) { this.path = path; this.log = log; + this.lastSuccessfulPath = lastSuccessfulPath; } public Path outputPath() { @@ -19,6 +22,16 @@ public BuildLog log() { return log; } + /** + * Returns the output directory from the last time this task ran, to be used to copy other unchanged + * output files rather than regenerate them. + * + * @return empty if no previous build exists, otherwise a path to the last successful output + */ + public Optional lastSuccessfulOutput() { + return Optional.ofNullable(lastSuccessfulPath); + } + @Override public void debug(String msg) { log.debug(msg); diff --git a/j2cl-archetypes/README.md b/j2cl-archetypes/README.md index 73f77934..d1387e74 100644 --- a/j2cl-archetypes/README.md +++ b/j2cl-archetypes/README.md @@ -7,14 +7,14 @@ To create a project interactively, use: ``` mvn archetype:generate -DarchetypeGroupId=com.vertispan.j2cl.archetypes \ -DarchetypeArtifactId= \ --DarchetypeVersion=0.20 +-DarchetypeVersion=0.21 ``` To specify these four variables, add them as system properties: ``` mvn archetype:generate -DarchetypeGroupId=com.vertispan.j2cl.archetypes \ -DarchetypeArtifactId= \ --DarchetypeVersion=0.20 \ +-DarchetypeVersion=0.21 \ -DgroupId=my.project.group.id \ -DartifactId=myapp \ -Dversion=1.0-SNAPSHOT \ @@ -26,7 +26,7 @@ to replace `` with the name of the archetype: ``` mvn org.apache.maven.plugins:maven-dependency-plugin:get \ -DrepoUrl=https://repo.vertispan.com/j2cl/ \ --Dartifact=com.vertispan.j2cl.archetypes::0.20 +-Dartifact=com.vertispan.j2cl.archetypes::0.21 ``` # [`j2cl-archetype-simple`](j2cl-archetype-simple) diff --git a/j2cl-archetypes/j2cl-archetype-servlet/README.md b/j2cl-archetypes/j2cl-archetype-servlet/README.md index 4cfc4f51..3cefec6f 100644 --- a/j2cl-archetypes/j2cl-archetype-servlet/README.md +++ b/j2cl-archetypes/j2cl-archetype-servlet/README.md @@ -3,7 +3,7 @@ ``` mvn archetype:generate -DarchetypeGroupId=com.vertispan.j2cl.archetypes \ -DarchetypeArtifactId=j2cl-archetype-servlet \ --DarchetypeVersion=0.20 +-DarchetypeVersion=0.21 ``` Creates two Java modules, one for the server, and one for the client. The server module uses Jakarta diff --git a/j2cl-archetypes/j2cl-archetype-servlet/pom.xml b/j2cl-archetypes/j2cl-archetype-servlet/pom.xml index e4a16279..e406d73e 100644 --- a/j2cl-archetypes/j2cl-archetype-servlet/pom.xml +++ b/j2cl-archetypes/j2cl-archetype-servlet/pom.xml @@ -6,7 +6,7 @@ com.vertispan.j2cl.archetypes j2cl-archetypes-parent - 0.21-SNAPSHOT + 0.22-SNAPSHOT j2cl-archetype-servlet maven-archetype diff --git a/j2cl-archetypes/j2cl-archetype-simple/README.md b/j2cl-archetypes/j2cl-archetype-simple/README.md index 6c07f1c3..5ea2cc43 100644 --- a/j2cl-archetypes/j2cl-archetype-simple/README.md +++ b/j2cl-archetypes/j2cl-archetype-simple/README.md @@ -3,7 +3,7 @@ ``` mvn archetype:generate -DarchetypeGroupId=com.vertispan.j2cl.archetypes \ -DarchetypeArtifactId=j2cl-archetype-simple \ --DarchetypeVersion=0.20 +-DarchetypeVersion=0.21 ``` This project is a simple html page, with a css file, and a single Java class. It is _not_ a good diff --git a/j2cl-archetypes/j2cl-archetype-simple/pom.xml b/j2cl-archetypes/j2cl-archetype-simple/pom.xml index 1657189c..cb725386 100644 --- a/j2cl-archetypes/j2cl-archetype-simple/pom.xml +++ b/j2cl-archetypes/j2cl-archetype-simple/pom.xml @@ -6,7 +6,7 @@ com.vertispan.j2cl.archetypes j2cl-archetypes-parent - 0.21-SNAPSHOT + 0.22-SNAPSHOT j2cl-archetype-simple maven-archetype diff --git a/j2cl-archetypes/pom.xml b/j2cl-archetypes/pom.xml index f26f23a5..f4032f20 100644 --- a/j2cl-archetypes/pom.xml +++ b/j2cl-archetypes/pom.xml @@ -6,7 +6,7 @@ com.vertispan.j2cl j2cl-tools - 0.21-SNAPSHOT + 0.22-SNAPSHOT com.vertispan.j2cl.archetypes j2cl-archetypes-parent diff --git a/j2cl-maven-plugin/pom.xml b/j2cl-maven-plugin/pom.xml index 582f0250..6a46351e 100644 --- a/j2cl-maven-plugin/pom.xml +++ b/j2cl-maven-plugin/pom.xml @@ -7,7 +7,7 @@ com.vertispan.j2cl j2cl-tools - 0.21-SNAPSHOT + 0.22-SNAPSHOT j2cl-maven-plugin maven-plugin diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java index 7252b905..a0c5ead1 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractBuildMojo.java @@ -85,6 +85,9 @@ public abstract class AbstractBuildMojo extends AbstractCacheMojo { @Parameter(defaultValue = "AVOID_MAVEN") private AnnotationProcessorMode annotationProcessorMode; + @Parameter(defaultValue = "false", property = "j2cl.incremental") + private boolean incrementalEnabled; + private List defaultDependencyReplacements = Arrays.asList( new DependencyReplacement("com.google.jsinterop:base", "com.vertispan.jsinterop:base:" + Versions.VERTISPAN_JSINTEROP_BASE_VERSION), new DependencyReplacement("org.realityforge.com.google.jsinterop:base", "com.vertispan.jsinterop:base:" + Versions.VERTISPAN_JSINTEROP_BASE_VERSION), diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractCacheMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractCacheMojo.java index dc539aaf..d30cbe68 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractCacheMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/AbstractCacheMojo.java @@ -16,6 +16,9 @@ public abstract class AbstractCacheMojo extends AbstractMojo { @Parameter(defaultValue = "${project.build.directory}/gwt3BuildCache", required = true, property = "gwt3.cache.dir") private File gwt3BuildCacheDir; + @Parameter(defaultValue = "${project.build.directory}/j2cl-maven-plugin-local-cache", required = true) + protected File localBuildCache; + protected Path getCacheDir() { PluginDescriptor pluginDescriptor = (PluginDescriptor) getPluginContext().get("pluginDescriptor"); String pluginVersion = pluginDescriptor.getVersion(); diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/BuildMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/BuildMojo.java index b1eb448e..755a5a5d 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/BuildMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/BuildMojo.java @@ -4,6 +4,7 @@ import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.DefaultDiskCache; import com.vertispan.j2cl.build.DiskCache; +import com.vertispan.j2cl.build.LocalProjectBuildCache; import com.vertispan.j2cl.build.Project; import com.vertispan.j2cl.build.TaskRegistry; import com.vertispan.j2cl.build.TaskScheduler; @@ -231,7 +232,7 @@ public void execute() throws MojoExecutionException, MojoFailureException { addShutdownHook(executor, diskCache); MavenLog mavenLog = new MavenLog(getLog()); - TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, mavenLog); + TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, new LocalProjectBuildCache(localBuildCache, diskCache), mavenLog); TaskRegistry taskRegistry = createTaskRegistry(); diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/TestMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/TestMojo.java index 1f6d95ef..001d4448 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/TestMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/TestMojo.java @@ -9,6 +9,7 @@ import com.vertispan.j2cl.build.DefaultDiskCache; import com.vertispan.j2cl.build.Dependency; import com.vertispan.j2cl.build.DiskCache; +import com.vertispan.j2cl.build.LocalProjectBuildCache; import com.vertispan.j2cl.build.Project; import com.vertispan.j2cl.build.PropertyTrackingConfig; import com.vertispan.j2cl.build.TaskRegistry; @@ -335,7 +336,7 @@ public void execute() throws MojoExecutionException, MojoFailureException { addShutdownHook(executor, diskCache); MavenLog mavenLog = new MavenLog(getLog()); - TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, mavenLog); + TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, new LocalProjectBuildCache(localBuildCache, diskCache), mavenLog); TaskRegistry taskRegistry = createTaskRegistry(); // Given these, build the graph of work we need to complete to get the list of tests diff --git a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/WatchMojo.java b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/WatchMojo.java index 7e3e768b..383562c8 100644 --- a/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/WatchMojo.java +++ b/j2cl-maven-plugin/src/main/java/com/vertispan/j2cl/mojo/WatchMojo.java @@ -3,6 +3,7 @@ import com.vertispan.j2cl.build.BuildService; import com.vertispan.j2cl.build.DefaultDiskCache; import com.vertispan.j2cl.build.DiskCache; +import com.vertispan.j2cl.build.LocalProjectBuildCache; import com.vertispan.j2cl.build.Project; import com.vertispan.j2cl.build.TaskRegistry; import com.vertispan.j2cl.build.TaskScheduler; @@ -196,7 +197,7 @@ public void execute() throws MojoExecutionException, MojoFailureException { addShutdownHook(executor, diskCache); MavenLog mavenLog = new MavenLog(getLog()); - TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, mavenLog); + TaskScheduler taskScheduler = new TaskScheduler(executor, diskCache, new LocalProjectBuildCache(localBuildCache, diskCache), mavenLog); // TODO support individual task registries per execution TaskRegistry taskRegistry = createTaskRegistry(); diff --git a/j2cl-tasks/pom.xml b/j2cl-tasks/pom.xml index 6ebd4fb9..2ac00682 100644 --- a/j2cl-tasks/pom.xml +++ b/j2cl-tasks/pom.xml @@ -7,7 +7,7 @@ com.vertispan.j2cl j2cl-tools - 0.21-SNAPSHOT + 0.22-SNAPSHOT j2cl-tasks diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java index 57eec15c..5b4a3c4e 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java @@ -1,24 +1,53 @@ package com.vertispan.j2cl.build.provided; import com.google.auto.service.AutoService; -import com.google.javascript.jscomp.CompilationLevel; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.javascript.jscomp.Compiler; +import com.google.javascript.jscomp.CompilerInput; import com.google.javascript.jscomp.CompilerOptions; -import com.google.javascript.jscomp.DependencyOptions; -import com.vertispan.j2cl.build.task.*; -import io.methvin.watcher.hashing.Murmur3F; +import com.google.javascript.jscomp.SourceFile; +import com.google.javascript.jscomp.deps.ClosureBundler; +import com.google.javascript.jscomp.deps.DependencyInfo; +import com.google.javascript.jscomp.deps.ModuleLoader; +import com.google.javascript.jscomp.deps.SortedDependencies; +import com.google.javascript.jscomp.parsing.parser.FeatureSet; +import com.google.javascript.jscomp.transpile.BaseTranspiler; +import com.google.javascript.jscomp.transpile.Transpiler; +import com.vertispan.j2cl.build.task.CachedPath; +import com.vertispan.j2cl.build.task.ChangedCachedPath; +import com.vertispan.j2cl.build.task.Config; +import com.vertispan.j2cl.build.task.Input; +import com.vertispan.j2cl.build.task.OutputTypes; +import com.vertispan.j2cl.build.task.Project; +import com.vertispan.j2cl.build.task.TaskFactory; import com.vertispan.j2cl.tools.Closure; +import io.methvin.watcher.hashing.Murmur3F; import org.apache.commons.io.FileUtils; import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; +import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Optional; +import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -61,6 +90,11 @@ public Task resolve(Project project, Config config) { .collect(Collectors.toUnmodifiableList()); } + // Consider treating this always as true, since the build doesnt get more costly to be incremental + boolean incrementalEnabled = config.isIncrementalEnabled(); + + Gson gson = new GsonBuilder().setPrettyPrinting().serializeNulls().create(); + return context -> { assert Files.isDirectory(context.outputPath()); File closureOutputDir = context.outputPath().toFile(); @@ -71,50 +105,124 @@ public Task resolve(Project project, Config config) { String outputFile = closureOutputDir + "/" + fileNameKey + ".js"; Path outputFilePath = Paths.get(outputFile); - if (!js.stream().map(Input::getFilesAndHashes).flatMap(Collection::stream).findAny().isPresent()) { + if (js.stream().map(Input::getFilesAndHashes).flatMap(Collection::stream).findAny().isEmpty()) { // if there are no js sources, write an empty file and exit Files.createFile(outputFilePath); return;// nothing to do } - Closure closureCompiler = new Closure(context); - // copy the sources locally so that we can create usable sourcemaps //TODO consider a soft link File sources = new File(closureOutputDir, Closure.SOURCES_DIRECTORY_NAME); - for (Path path : js.stream().map(Input::getParentPaths).flatMap(Collection::stream).collect(Collectors.toList())) { + for (Path path : js.stream().map(Input::getParentPaths).flatMap(Collection::stream).collect(Collectors.toUnmodifiableList())) { FileUtils.copyDirectory(path.toFile(), sources); } - // create the JS bundle, only ordering these files - boolean success = closureCompiler.compile( - CompilationLevel.BUNDLE, - DependencyOptions.DependencyMode.SORT_ONLY, - CompilerOptions.LanguageMode.NO_TRANSPILE, - Collections.singletonMap( - sources.getAbsolutePath(), - js.stream() - .map(Input::getFilesAndHashes) + List dependencyInfos = new ArrayList<>(); + Compiler jsCompiler = new Compiler(System.err);//TODO before merge, write this to the log + + if (incrementalEnabled && context.lastSuccessfulOutput().isPresent()) { + // collect any dep info from disk for existing files + final Map depInfoMap; + Path lastOutput = context.lastSuccessfulOutput().get(); + try (InputStream inputStream = Files.newInputStream(lastOutput.resolve("depInfo.json"))) { + Type listType = new TypeToken>() { + }.getType(); + List deps = gson.fromJson(new BufferedReader(new InputStreamReader(inputStream)), listType); + depInfoMap = deps.stream() + .map(info -> new DependencyInfoAndSource( + info, + () -> Files.readString(lastOutput.resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(info.getName()))) + ) + .collect(Collectors.toMap(DependencyInfo::getName, Function.identity())); + } + + // create new dep info for any added/modified file + for (Input jsInput : js) { + for (ChangedCachedPath change : jsInput.getChanges()) { + if (change.changeType() == ChangedCachedPath.ChangeType.REMOVED) { + depInfoMap.remove(change.getSourcePath().toString()); + } else { + // ADD or MODIFY + CompilerInput input = new CompilerInput(SourceFile.builder() + .withPath(context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(change.getSourcePath())) + .withOriginalPath(change.getSourcePath().toString()) + .build()); + input.setCompiler(jsCompiler); + depInfoMap.put( + change.getSourcePath().toString(), + new DependencyInfoAndSource(input, input::getCode) + ); + } + } + } + + // no need to expand to include other files, since this is only computed locally + + // assign the dep info and sources we have + dependencyInfos.addAll(depInfoMap.values()); + } else { + //non-incremental, read everything + for (Input jsInput : js) { + for (CachedPath path : jsInput.getFilesAndHashes()) { + CompilerInput input = new CompilerInput(SourceFile.builder() + .withPath(context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(path.getSourcePath())) + .withOriginalPath(path.getSourcePath().toString()) + .build()); + input.setCompiler(jsCompiler); + + dependencyInfos.add(new DependencyInfoAndSource(input, input::getCode)); + } + } + } + + // re-sort that full collection + SortedDependencies sorter = new SortedDependencies<>(dependencyInfos); + + + // TODO optional/stretch-goal find first change in the list, so we can keep old prefix of bundle output + + // rebundle all (optional: remaining) files using this already handled sort + ClosureBundler bundler = new ClosureBundler(Transpiler.NULL, new BaseTranspiler( + new BaseTranspiler.CompilerSupplier( + CompilerOptions.LanguageMode.ECMASCRIPT_NEXT.toFeatureSet().without(FeatureSet.Feature.MODULES), + ModuleLoader.ResolutionMode.BROWSER, + ImmutableList.copyOf(js.stream() + .map(Input::getParentPaths) .flatMap(Collection::stream) - .map(CachedPath::getSourcePath) .map(Path::toString) - .collect(Collectors.toUnmodifiableList()) + .collect(Collectors.toUnmodifiableList())), + ImmutableMap.of() ), - sources, - Collections.emptyList(), - Collections.emptyMap(), - Collections.emptyList(),//TODO actually pass these in when we can restrict and cache them sanely - Optional.empty(), - true,//TODO have this be passed in, - true,//default to true, will have no effect anyway - false, - false, - "CUSTOM", // doesn't matter, bundle won't check this - outputFile - ); - - if (!success) { - throw new IllegalStateException("Closure Compiler failed, check log for details"); + "" + )).useEval(true); + + try (OutputStream outputStream = Files.newOutputStream(Paths.get(outputFile)); + BufferedWriter bundleOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) { + for (DependencyInfoAndSource info : sorter.getSortedList()) { + String code = info.getSource(); + String name = info.getName(); + + //TODO do we actually need this? + if (Compiler.isFillFileName(name) && code.isEmpty()) { + continue; + } + + // append this file and a comment where it came from + bundleOut.append("//").append(name).append("\n"); + bundler.withPath(name).withSourceUrl(Closure.SOURCES_DIRECTORY_NAME + "/" + name).appendTo(bundleOut, info, code); + bundleOut.append("\n"); + + } + + } + // append dependency info to deserialize on some incremental rebuild + try (OutputStream outputStream = Files.newOutputStream(context.outputPath().resolve("depInfo.json")); + BufferedWriter jsonOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) { + List jsonList = sorter.getSortedList().stream() + .map(DependencyInfoFormat::new) + .collect(Collectors.toUnmodifiableList()); + gson.toJson(jsonList, jsonOut); } // hash the file itself, rename to include that hash @@ -129,4 +237,174 @@ public Task resolve(Project project, Config config) { //TODO when back to keyboard rename sourcemap? is that a thing we need to do? }; } + + public interface SourceSupplier { + String get() throws IOException; + } + public static class DependencyInfoAndSource extends DependencyInfo.Base { + private final DependencyInfo delegate; + private final SourceSupplier sourceSupplier; + + public DependencyInfoAndSource(DependencyInfo delegate, SourceSupplier sourceSupplier) { + this.delegate = delegate; + this.sourceSupplier = sourceSupplier; + } + + public String getSource() throws IOException { + return sourceSupplier.get(); + } + + @Override + public String getName() { + return delegate.getName(); + } + + @Override + public String getPathRelativeToClosureBase() { + return delegate.getPathRelativeToClosureBase(); + } + + @Override + public ImmutableList getProvides() { + return delegate.getProvides(); + } + + @Override + public ImmutableList getRequires() { + return delegate.getRequires(); + } + + @Override + public ImmutableList getRequiredSymbols() { + //deliberately overriding the base impl + return delegate.getRequiredSymbols(); + } + + @Override + public ImmutableList getTypeRequires() { + return delegate.getTypeRequires(); + } + + @Override + public ImmutableMap getLoadFlags() { + return delegate.getLoadFlags(); + } + + @Override + public boolean isModule() { + return delegate.isModule(); + } + + @Override + public boolean isEs6Module() { + return delegate.isEs6Module(); + } + + @Override + public boolean isGoogModule() { + return delegate.isGoogModule(); + } + + @Override + public boolean getHasExternsAnnotation() { + return delegate.getHasExternsAnnotation(); + } + + @Override + public boolean getHasNoCompileAnnotation() { + return delegate.getHasNoCompileAnnotation(); + } + } + + public static class DependencyInfoFormat extends DependencyInfo.Base { + private String name; +// private String pathRelativeToClosureBase = name; + private List provides; +// private List requires; //skipping requires as it isnt used by the dep sorter + private List requiredSymbols; + private List typeRequires; + private Map loadFlags; + private boolean hasExternsAnnotation; + private boolean hasNoCompileAnnotation; + + public DependencyInfoFormat() { + + } + + public DependencyInfoFormat(DependencyInfo info) { + setName(info.getName()); + setHasExternsAnnotation(info.getHasExternsAnnotation()); + setHasNoCompileAnnotation(info.getHasExternsAnnotation()); + setProvides(info.getProvides()); + setLoadFlags(info.getLoadFlags()); + setTypeRequires(info.getTypeRequires()); + setRequiredSymbols(info.getRequiredSymbols()); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public String getPathRelativeToClosureBase() { + return getName(); + } + + public ImmutableList getProvides() { + return ImmutableList.copyOf(provides); + } + + public void setProvides(List provides) { + this.provides = provides; + } + + public ImmutableList getRequires() { + return ImmutableList.of(); + } + + @Override + public ImmutableList getRequiredSymbols() { + return ImmutableList.copyOf(requiredSymbols); + } + + public void setRequiredSymbols(List requiredSymbols) { + this.requiredSymbols = requiredSymbols; + } + + public ImmutableList getTypeRequires() { + return ImmutableList.copyOf(typeRequires); + } + + public void setTypeRequires(List typeRequires) { + this.typeRequires = typeRequires; + } + + public ImmutableMap getLoadFlags() { + return ImmutableMap.copyOf(loadFlags); + } + + public void setLoadFlags(Map loadFlags) { + this.loadFlags = loadFlags; + } + + public boolean getHasExternsAnnotation() { + return hasExternsAnnotation; + } + + public void setHasExternsAnnotation(boolean hasExternsAnnotation) { + this.hasExternsAnnotation = hasExternsAnnotation; + } + + public boolean getHasNoCompileAnnotation() { + return hasNoCompileAnnotation; + } + + public void setHasNoCompileAnnotation(boolean hasNoCompileAnnotation) { + this.hasNoCompileAnnotation = hasNoCompileAnnotation; + } + } } diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java index ea1c9a96..93f6e957 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureTask.java @@ -170,7 +170,11 @@ public Task resolve(Project project, Config config) { .collect(Collectors.toUnmodifiableList()); // grab configs we plan to use - String compilationLevelConfig = config.getCompilationLevel(); + CompilationLevel compilationLevel = CompilationLevel.fromString(config.getCompilationLevel()); + if (compilationLevel == null) { + throw new IllegalArgumentException("Unrecognized compilationLevel: " + config.getCompilationLevel()); + } + String initialScriptFilename = config.getInitialScriptFilename(); Map configDefines = config.getDefines(); DependencyOptions.DependencyMode dependencyMode = DependencyOptions.DependencyMode.valueOf(config.getDependencyMode()); @@ -201,8 +205,6 @@ public void execute(TaskContext context) throws Exception { File closureOutputDir = context.outputPath().toFile(); - CompilationLevel compilationLevel = CompilationLevel.fromString(compilationLevelConfig); - // set up a source directory to build from, and to make sourcemaps work // TODO move logic to the "post" phase to decide whether or not to copy the sourcemap dir String jsOutputDir = new File(closureOutputDir + "/" + initialScriptFilename).getParent(); @@ -232,7 +234,7 @@ public void execute(TaskContext context) throws Exception { js = Closure.mapFromInputs(jsSources); } if (sources != null) { - for (Path path : jsSources.stream().map(Input::getParentPaths).flatMap(Collection::stream).collect(Collectors.toList())) { + for (Path path : jsSources.stream().map(Input::getParentPaths).flatMap(Collection::stream).collect(Collectors.toUnmodifiableList())) { FileUtils.copyDirectory(path.toFile(), sources); } } diff --git a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java index ea71d910..5e4aab6d 100644 --- a/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java +++ b/j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/StripSourcesTask.java @@ -5,14 +5,18 @@ import com.vertispan.j2cl.build.task.*; import com.vertispan.j2cl.tools.GwtIncompatiblePreprocessor; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.PathMatcher; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; @AutoService(TaskFactory.class) public class StripSourcesTask extends TaskFactory { public static final PathMatcher JAVA_SOURCES = withSuffix(".java"); - public static final PathMatcher NATIVE_JS_SOURCES = withSuffix(".native.js"); @Override public String getOutputType() { @@ -37,12 +41,42 @@ public Task resolve(Project project, Config config) { if (inputSources.getFilesAndHashes().isEmpty()) { return;// nothing to do } + List filesToProcess = new ArrayList<>(); + if (context.lastSuccessfulOutput().isPresent()) { + Map unmodified = inputSources.getFilesAndHashes().stream().collect(Collectors.toMap( + CachedPath::getSourcePath, + Function.identity() + )); + //only process changed files, copy unchanged ones + for (ChangedCachedPath change : inputSources.getChanges()) { + // remove the file, since it was changed in some way + unmodified.remove(change.getSourcePath()); + + if (change.changeType() != ChangedCachedPath.ChangeType.REMOVED) { + // track the files we actually need to process + filesToProcess.add(makeFileInfo(change)); + } + } + for (CachedPath path : unmodified.values()) { + Files.createDirectories(context.outputPath().resolve(path.getSourcePath()).getParent()); + Files.copy(context.lastSuccessfulOutput().get().resolve(path.getSourcePath()), context.outputPath().resolve(path.getSourcePath())); + } + } else { + for (CachedPath path : inputSources.getFilesAndHashes()) { + filesToProcess.add(makeFileInfo(path)); + } + } GwtIncompatiblePreprocessor preprocessor = new GwtIncompatiblePreprocessor(context.outputPath().toFile(), context); - preprocessor.preprocess( - inputSources.getFilesAndHashes().stream() - .map(p -> SourceUtils.FileInfo.create(p.getAbsolutePath().toString(), p.getSourcePath().toString())) - .collect(Collectors.toUnmodifiableList()) - ); + preprocessor.preprocess(filesToProcess); }; } + + private SourceUtils.FileInfo makeFileInfo(ChangedCachedPath change) { + assert change.getNewAbsolutePath().isPresent() : "Can't make a FileInfo if it no longer exists"; + return SourceUtils.FileInfo.create(change.getNewAbsolutePath().get().toString(), change.getSourcePath().toString()); + } + + private SourceUtils.FileInfo makeFileInfo(CachedPath path) { + return SourceUtils.FileInfo.create(path.getAbsolutePath().toString(), path.getSourcePath().toString()); + } } diff --git a/pom.xml b/pom.xml index cab5282e..af37ffbd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.vertispan.j2cl j2cl-tools - 0.21-SNAPSHOT + 0.22-SNAPSHOT pom J2CL Build Tools