From bf80bdbfc5570224516c2a299686e5fe9d5cdc71 Mon Sep 17 00:00:00 2001 From: Knut Wannheden Date: Fri, 22 Dec 2023 15:02:50 +0100 Subject: [PATCH] Add `ExecutionContext#getWorkingDirectory()` (#3815) * Add `ExecutionContext#getWorkingDirectory()` Any recipe can always call `ExecutionContext#getWorkingDirectory()` to get a working directory private to that recipe instance, where the recipe can store files. These files will be managed by that recipe for the entire recipe run, at the end of which the directory is automatically deleted again. * Make sure working directory is deleted Also add some unit tests * Add some more validations * Delete working directory at end of recipe run only * RecipeRunCycle now public, consolidate and limit access in WorkingDirectoryExecutionContext * Add missing license header * Fix some test expectations * Also initialize recipe position in generate phase * Extracted new `RecipeStack` class * Remove `mapForRecipeRecursively()` * Polish * Added protected `applyToSourceSet()` * Replace `applyToSourceSet()` with `BiFunction` The idea is to not require subclassing for that use case. * Added generic type params to `RecipeRunCycle` * Fix checking of max cycles --------- Co-authored-by: Jonathan Schneider --- .../org/openrewrite/ExecutionContext.java | 5 + .../src/main/java/org/openrewrite/Recipe.java | 1 + .../java/org/openrewrite/RecipeScheduler.java | 316 +++--------------- .../scheduling/RecipeRunCycle.java | 270 +++++++++++++++ .../openrewrite/scheduling/RecipeStack.java | 85 +++++ .../scheduling/WatchableExecutionContext.java | 5 +- .../WorkingDirectoryExecutionContextView.java | 80 +++++ .../org/openrewrite/RecipeSchedulerTest.java | 164 ++++++++- 8 files changed, 648 insertions(+), 278 deletions(-) create mode 100644 rewrite-core/src/main/java/org/openrewrite/scheduling/RecipeRunCycle.java create mode 100644 rewrite-core/src/main/java/org/openrewrite/scheduling/RecipeStack.java create mode 100644 rewrite-core/src/main/java/org/openrewrite/scheduling/WorkingDirectoryExecutionContextView.java diff --git a/rewrite-core/src/main/java/org/openrewrite/ExecutionContext.java b/rewrite-core/src/main/java/org/openrewrite/ExecutionContext.java index d7cb109d9b9..14e7c7c3d17 100644 --- a/rewrite-core/src/main/java/org/openrewrite/ExecutionContext.java +++ b/rewrite-core/src/main/java/org/openrewrite/ExecutionContext.java @@ -16,6 +16,7 @@ package org.openrewrite; import org.openrewrite.internal.lang.Nullable; +import org.openrewrite.scheduling.RecipeRunCycle; import java.util.*; import java.util.function.BiConsumer; @@ -97,6 +98,10 @@ default void putCurrentRecipe(Recipe recipe) { BiConsumer getOnTimeout(); default int getCycle() { + return getCycleDetails().getCycle(); + } + + default RecipeRunCycle getCycleDetails() { return requireNonNull(getMessage(CURRENT_CYCLE)); } } diff --git a/rewrite-core/src/main/java/org/openrewrite/Recipe.java b/rewrite-core/src/main/java/org/openrewrite/Recipe.java index f7109ecc087..6393fef4440 100644 --- a/rewrite-core/src/main/java/org/openrewrite/Recipe.java +++ b/rewrite-core/src/main/java/org/openrewrite/Recipe.java @@ -281,6 +281,7 @@ public final RecipeRun run(LargeSourceSet before, ExecutionContext ctx, int maxC return new RecipeScheduler().scheduleRun(this, before, ctx, maxCycles, minCycles); } + @SuppressWarnings("unused") public Validated validate(ExecutionContext ctx) { Validated validated = validate(); diff --git a/rewrite-core/src/main/java/org/openrewrite/RecipeScheduler.java b/rewrite-core/src/main/java/org/openrewrite/RecipeScheduler.java index 0f8a5362a38..eea036c77a1 100644 --- a/rewrite-core/src/main/java/org/openrewrite/RecipeScheduler.java +++ b/rewrite-core/src/main/java/org/openrewrite/RecipeScheduler.java @@ -15,28 +15,20 @@ */ package org.openrewrite; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; -import org.openrewrite.internal.ExceptionUtils; -import org.openrewrite.internal.FindRecipeRunException; -import org.openrewrite.internal.RecipeRunException; -import org.openrewrite.internal.lang.Nullable; -import org.openrewrite.marker.Generated; -import org.openrewrite.marker.RecipesThatMadeChanges; +import org.openrewrite.scheduling.RecipeRunCycle; import org.openrewrite.scheduling.WatchableExecutionContext; import org.openrewrite.table.RecipeRunStats; import org.openrewrite.table.SourcesFileErrors; import org.openrewrite.table.SourcesFileResults; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiFunction; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; import static java.util.Collections.emptyMap; -import static java.util.Objects.requireNonNull; import static org.openrewrite.Recipe.PANIC; +import static org.openrewrite.scheduling.WorkingDirectoryExecutionContextView.WORKING_DIRECTORY_ROOT; public class RecipeScheduler { @@ -45,6 +37,21 @@ public RecipeRun scheduleRun(Recipe recipe, ExecutionContext ctx, int maxCycles, int minCycles) { + try { + LargeSourceSet after = runRecipeCycles(recipe, sourceSet, ctx, maxCycles, minCycles); + return new RecipeRun( + after.getChangeset(), + ctx.getMessage(ExecutionContext.DATA_TABLES, emptyMap()) + ); + } finally { + Path workingDirectoryRoot = ctx.getMessage(WORKING_DIRECTORY_ROOT); + if (workingDirectoryRoot != null) { + deleteWorkingDirectory(workingDirectoryRoot); + } + } + } + + private LargeSourceSet runRecipeCycles(Recipe recipe, LargeSourceSet sourceSet, ExecutionContext ctx, int maxCycles, int minCycles) { WatchableExecutionContext ctxWithWatch = new WatchableExecutionContext(ctx); RecipeRunStats recipeRunStats = new RecipeRunStats(Recipe.noop()); @@ -54,36 +61,39 @@ public RecipeRun scheduleRun(Recipe recipe, LargeSourceSet after = sourceSet; for (int i = 1; i <= maxCycles; i++) { - ctxWithWatch.putCycle(i); - after.beforeCycle(); + if (ctx.getMessage(PANIC) != null) { + break; + } // this root cursor is shared by all `TreeVisitor` instances used created from `getVisitor` and // single source applicable tests so that data can be shared at the root (especially for caching // use cases like sharing a `JavaTypeCache` between `JavaTemplate` parsers). Cursor rootCursor = new Cursor(null, Cursor.ROOT_VALUE); try { - RecipeRunCycle cycle = new RecipeRunCycle(recipe, i, rootCursor, ctxWithWatch, - recipeRunStats, sourceFileResults, errorsTable); + RecipeRunCycle cycle = new RecipeRunCycle<>(recipe, i, rootCursor, ctxWithWatch, + recipeRunStats, sourceFileResults, errorsTable, LargeSourceSet::edit); + ctxWithWatch.putCycle(cycle); + after.beforeCycle(); // pre-transformation scanning phase where there can only be modifications to capture exceptions // occurring during the scanning phase if (hasScanningRecipe(recipe)) { - after = cycle.scanSources(after, i); + after = cycle.scanSources(after); } // transformation phases - after = cycle.generateSources(after, i); - after = cycle.editSources(after, i); + after = cycle.generateSources(after); + after = cycle.editSources(after); boolean anyRecipeCausingAnotherCycle = false; - for (Recipe madeChanges : cycle.madeChangesInThisCycle) { + for (Recipe madeChanges : cycle.getMadeChangesInThisCycle()) { if (madeChanges.causesAnotherCycle()) { anyRecipeCausingAnotherCycle = true; } } if (i >= minCycles && - (cycle.madeChangesInThisCycle.isEmpty() || !anyRecipeCausingAnotherCycle)) { + (cycle.getMadeChangesInThisCycle().isEmpty() || !anyRecipeCausingAnotherCycle)) { after.afterCycle(true); break; } @@ -102,11 +112,7 @@ public RecipeRun scheduleRun(Recipe recipe, } recipeRunStats.flush(ctx); - - return new RecipeRun( - after.getChangeset(), - ctx.getMessage(ExecutionContext.DATA_TABLES, emptyMap()) - ); + return after; } private boolean hasScanningRecipe(Recipe recipe) { @@ -121,254 +127,16 @@ private boolean hasScanningRecipe(Recipe recipe) { return false; } - @RequiredArgsConstructor - @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) - static class RecipeRunCycle { - Recipe recipe; - int cycle; - Cursor rootCursor; - WatchableExecutionContext ctx; - RecipeRunStats recipeRunStats; - SourcesFileResults sourcesFileResults; - SourcesFileErrors errorsTable; - Map> recipeLists = new IdentityHashMap<>(); - - long cycleStartTime = System.nanoTime(); - AtomicBoolean thrownErrorOnTimeout = new AtomicBoolean(); - Set madeChangesInThisCycle = Collections.newSetFromMap(new IdentityHashMap<>()); - - public LargeSourceSet scanSources(LargeSourceSet sourceSet, int cycle) { - return mapForRecipeRecursively(sourceSet, (recipeStack, sourceFile) -> { - Recipe recipe = recipeStack.peek(); - if (sourceFile == null || recipe.maxCycles() < cycle) { - return sourceFile; - } - - SourceFile after = sourceFile; - - if (recipe instanceof ScanningRecipe) { - try { - //noinspection unchecked - ScanningRecipe scanningRecipe = (ScanningRecipe) recipe; - Object acc = scanningRecipe.getAccumulator(rootCursor, ctx); - recipeRunStats.recordScan(recipe, () -> { - TreeVisitor scanner = scanningRecipe.getScanner(acc); - if (scanner.isAcceptable(sourceFile, ctx)) { - scanner.visit(sourceFile, ctx, rootCursor); - } - return sourceFile; - }); - } catch (Throwable t) { - after = handleError(recipe, sourceFile, after, t); - } - } - return after; - }); - } - - public LargeSourceSet generateSources(LargeSourceSet sourceSet, int cycle) { - List generatedInThisCycle = new ArrayList<>(); - - Stack> allRecipesStack = initRecipeStack(); - LargeSourceSet acc = sourceSet; - while (!allRecipesStack.isEmpty()) { - Stack recipeStack = allRecipesStack.pop(); - Recipe recipe = recipeStack.peek(); - if (recipe.maxCycles() < cycle) { - continue; - } - - if (recipe instanceof ScanningRecipe) { - //noinspection unchecked - ScanningRecipe scanningRecipe = (ScanningRecipe) recipe; - sourceSet.setRecipe(recipeStack); - List generated = new ArrayList<>(scanningRecipe.generate(scanningRecipe.getAccumulator(rootCursor, ctx), generatedInThisCycle, ctx)); - generated.replaceAll(source -> addRecipesThatMadeChanges(recipeStack, source)); - generatedInThisCycle.addAll(generated); - if (!generated.isEmpty()) { - madeChangesInThisCycle.add(recipe); - } - } - recurseRecipeList(allRecipesStack, recipeStack); - } - - acc = acc.generate(generatedInThisCycle); - return acc; - } - - public LargeSourceSet editSources(LargeSourceSet sourceSet, int cycle) { - return mapForRecipeRecursively(sourceSet, (recipeStack, sourceFile) -> { - Recipe recipe = recipeStack.peek(); - if (sourceFile == null || recipe.maxCycles() < cycle) { - return sourceFile; - } - - SourceFile after = sourceFile; - - try { - Duration duration = Duration.ofNanos(System.nanoTime() - cycleStartTime); - if (duration.compareTo(ctx.getMessage(ExecutionContext.RUN_TIMEOUT, Duration.ofMinutes(4))) > 0) { - if (thrownErrorOnTimeout.compareAndSet(false, true)) { - RecipeTimeoutException t = new RecipeTimeoutException(recipe); - ctx.getOnError().accept(t); - ctx.getOnTimeout().accept(t, ctx); - } - return sourceFile; - } - - if (ctx.getMessage(PANIC) != null) { - return sourceFile; - } - - TreeVisitor visitor = recipe.getVisitor(); - // set root cursor as it is required by the `ScanningRecipe#isAcceptable()` - visitor.setCursor(rootCursor); - - after = recipeRunStats.recordEdit(recipe, () -> { - if (visitor.isAcceptable(sourceFile, ctx)) { - // propagate shared root cursor - return (SourceFile) visitor.visit(sourceFile, ctx, rootCursor); - } - return sourceFile; - }); - - if (after != sourceFile) { - madeChangesInThisCycle.add(recipeStack.peek()); - recordSourceFileResult(sourceFile, after, recipeStack, ctx); - if (sourceFile.getMarkers().findFirst(Generated.class).isPresent()) { - // skip edits made to generated source files so that they don't show up in a diff - // that later fails to apply on a freshly cloned repository - return sourceFile; - } - recipeRunStats.recordSourceFileChanged(sourceFile, after); - } else if (ctx.hasNewMessages()) { - // consider any recipes adding new messages as a changing recipe (which can request another cycle) - madeChangesInThisCycle.add(recipeStack.peek()); - ctx.resetHasNewMessages(); - } - } catch (Throwable t) { - after = handleError(recipe, sourceFile, after, t); - } - if (after != null && after != sourceFile) { - after = addRecipesThatMadeChanges(recipeStack, after); + // Delete any files created in the working directory + private static void deleteWorkingDirectory(Path path) { + try { + if (Files.isDirectory(path)) { + try (Stream files = Files.list(path)) { + files.forEach(RecipeScheduler::deleteWorkingDirectory); } - return after; - }); - } - - private void recordSourceFileResult(@Nullable SourceFile before, @Nullable SourceFile after, Stack recipeStack, ExecutionContext ctx) { - String beforePath = (before == null) ? "" : before.getSourcePath().toString(); - String afterPath = (after == null) ? "" : after.getSourcePath().toString(); - Recipe recipe = recipeStack.peek(); - Long effortSeconds = (recipe.getEstimatedEffortPerOccurrence() == null) ? 0L : recipe.getEstimatedEffortPerOccurrence().getSeconds(); - String parentName = ""; - boolean hierarchical = recipeStack.size() > 1; - if (hierarchical) { - parentName = recipeStack.get(recipeStack.size() - 2).getName(); - } - String recipeName = recipe.getName(); - sourcesFileResults.insertRow(ctx, new SourcesFileResults.Row( - beforePath, - afterPath, - parentName, - recipeName, - effortSeconds, - cycle)); - if (hierarchical) { - recordSourceFileResult(beforePath, afterPath, recipeStack.subList(0, recipeStack.size() - 1), effortSeconds, ctx); - } - } - - private void recordSourceFileResult(@Nullable String beforePath, @Nullable String afterPath, List recipeStack, Long effortSeconds, ExecutionContext ctx) { - if (recipeStack.size() <= 1) { - // No reason to record the synthetic root recipe which contains the recipe run - return; } - String parentName; - if (recipeStack.size() == 2) { - // Record the parent name as blank rather than CompositeRecipe when the parent is the synthetic root recipe - parentName = ""; - } else { - parentName = recipeStack.get(recipeStack.size() - 2).getName(); - } - Recipe recipe = recipeStack.get(recipeStack.size() - 1); - sourcesFileResults.insertRow(ctx, new SourcesFileResults.Row( - beforePath, - afterPath, - parentName, - recipe.getName(), - effortSeconds, - cycle)); - recordSourceFileResult(beforePath, afterPath, recipeStack.subList(0, recipeStack.size() - 1), effortSeconds, ctx); + Files.delete(path); + } catch (IOException ignore) { } - - @Nullable - private SourceFile handleError(Recipe recipe, SourceFile sourceFile, @Nullable SourceFile after, - Throwable t) { - ctx.getOnError().accept(t); - - if (t instanceof RecipeRunException) { - RecipeRunException vt = (RecipeRunException) t; - after = (SourceFile) new FindRecipeRunException(vt).visitNonNull(requireNonNull(after, "after is null"), 0); - } - - // Use the original source file to record the error, not the one that may have been modified by the visitor. - // This is so the error is associated with the original source file, and its original source path. - errorsTable.insertRow(ctx, new SourcesFileErrors.Row( - sourceFile.getSourcePath().toString(), - recipe.getName(), - ExceptionUtils.sanitizeStackTrace(t, RecipeScheduler.class) - )); - - return after; - } - - private LargeSourceSet mapForRecipeRecursively(LargeSourceSet sourceSet, BiFunction, @Nullable SourceFile, @Nullable SourceFile> mapFn) { - return sourceSet.edit(sourceFile -> { - Stack> allRecipesStack = initRecipeStack(); - - SourceFile acc = sourceFile; - while (!allRecipesStack.isEmpty()) { - Stack recipeStack = allRecipesStack.pop(); - sourceSet.setRecipe(recipeStack); - acc = mapFn.apply(recipeStack, acc); - recurseRecipeList(allRecipesStack, recipeStack); - } - - return acc; - }); - } - - private Stack> initRecipeStack() { - Stack> allRecipesStack = new Stack<>(); - Stack rootRecipeStack = new Stack<>(); - rootRecipeStack.push(recipe); - allRecipesStack.push(rootRecipeStack); - return allRecipesStack; - } - - private void recurseRecipeList(Stack> allRecipesStack, Stack recipeStack) { - List recipeList = recipeLists.computeIfAbsent(recipeStack.peek(), Recipe::getRecipeList); - for (int i = recipeList.size() - 1; i >= 0; i--) { - Recipe r = recipeList.get(i); - if (ctx.getMessage(PANIC) != null) { - break; - } - Stack nextStack = new Stack<>(); - nextStack.addAll(recipeStack); - nextStack.push(r); - allRecipesStack.push(nextStack); - } - } - } - - private static S addRecipesThatMadeChanges(List recipeStack, S afterFile) { - return afterFile.withMarkers(afterFile.getMarkers().computeByType( - RecipesThatMadeChanges.create(recipeStack), - (r1, r2) -> { - r1.getRecipes().addAll(r2.getRecipes()); - return r1; - }) - ); } } diff --git a/rewrite-core/src/main/java/org/openrewrite/scheduling/RecipeRunCycle.java b/rewrite-core/src/main/java/org/openrewrite/scheduling/RecipeRunCycle.java new file mode 100644 index 00000000000..840164fe861 --- /dev/null +++ b/rewrite-core/src/main/java/org/openrewrite/scheduling/RecipeRunCycle.java @@ -0,0 +1,270 @@ +/* + * Copyright 2023 the original author or authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * https://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openrewrite.scheduling; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import org.openrewrite.*; +import org.openrewrite.internal.ExceptionUtils; +import org.openrewrite.internal.FindRecipeRunException; +import org.openrewrite.internal.RecipeRunException; +import org.openrewrite.internal.lang.Nullable; +import org.openrewrite.marker.Generated; +import org.openrewrite.marker.RecipesThatMadeChanges; +import org.openrewrite.table.RecipeRunStats; +import org.openrewrite.table.SourcesFileErrors; +import org.openrewrite.table.SourcesFileResults; + +import java.time.Duration; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiFunction; +import java.util.function.UnaryOperator; + +import static java.util.Collections.unmodifiableList; +import static java.util.Objects.requireNonNull; +import static org.openrewrite.Recipe.PANIC; + +@RequiredArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class RecipeRunCycle { + /** + * The root recipe that is running, which may contain a recipe list which will + * also be iterated as part of this cycle. + */ + Recipe recipe; + + /** + * The current cycle in the range [1, maxCycles]. + */ + @Getter + int cycle; + + Cursor rootCursor; + WatchableExecutionContext ctx; + RecipeRunStats recipeRunStats; + SourcesFileResults sourcesFileResults; + SourcesFileErrors errorsTable; + BiFunction, LSS> sourceSetEditor; + + RecipeStack allRecipeStack = new RecipeStack(); + long cycleStartTime = System.nanoTime(); + AtomicBoolean thrownErrorOnTimeout = new AtomicBoolean(); + + @Getter + Set madeChangesInThisCycle = Collections.newSetFromMap(new IdentityHashMap<>()); + + public int getRecipePosition() { + return allRecipeStack.getRecipePosition(); + } + + public LSS scanSources(LSS sourceSet) { + return sourceSetEditor.apply(sourceSet, sourceFile -> + allRecipeStack.reduce(sourceSet, recipe, ctx, (source, recipeStack) -> { + Recipe recipe = recipeStack.peek(); + if (source == null) { + return null; + } + + SourceFile after = source; + + if (recipe instanceof ScanningRecipe) { + try { + //noinspection unchecked + ScanningRecipe scanningRecipe = (ScanningRecipe) recipe; + Object acc = scanningRecipe.getAccumulator(rootCursor, ctx); + recipeRunStats.recordScan(recipe, () -> { + TreeVisitor scanner = scanningRecipe.getScanner(acc); + if (scanner.isAcceptable(source, ctx)) { + scanner.visit(source, ctx, rootCursor); + } + return source; + }); + } catch (Throwable t) { + after = handleError(recipe, source, after, t); + } + } + return after; + }, sourceFile) + ); + } + + public LSS generateSources(LSS sourceSet) { + List generatedInThisCycle = allRecipeStack.reduce(sourceSet, recipe, ctx, (acc, recipeStack) -> { + Recipe recipe = recipeStack.peek(); + if (recipe instanceof ScanningRecipe) { + //noinspection unchecked + ScanningRecipe scanningRecipe = (ScanningRecipe) recipe; + List generated = new ArrayList<>(scanningRecipe.generate(scanningRecipe.getAccumulator(rootCursor, ctx), unmodifiableList(acc), ctx)); + generated.replaceAll(source -> addRecipesThatMadeChanges(recipeStack, source)); + acc.addAll(generated); + if (!generated.isEmpty()) { + madeChangesInThisCycle.add(recipe); + } + } + return acc; + }, new ArrayList<>()); + + // noinspection unchecked + return (LSS) sourceSet.generate(generatedInThisCycle); + } + + public LSS editSources(LSS sourceSet) { + // set root cursor as it is required by the `ScanningRecipe#isAcceptable()` + // propagate shared root cursor + // skip edits made to generated source files so that they don't show up in a diff + // that later fails to apply on a freshly cloned repository + // consider any recipes adding new messages as a changing recipe (which can request another cycle) + return sourceSetEditor.apply(sourceSet, sourceFile -> + allRecipeStack.reduce(sourceSet, recipe, ctx, (source, recipeStack) -> { + Recipe recipe = recipeStack.peek(); + if (source == null) { + return null; + } + + SourceFile after = source; + + try { + Duration duration = Duration.ofNanos(System.nanoTime() - cycleStartTime); + if (duration.compareTo(ctx.getMessage(ExecutionContext.RUN_TIMEOUT, Duration.ofMinutes(4))) > 0) { + if (thrownErrorOnTimeout.compareAndSet(false, true)) { + RecipeTimeoutException t = new RecipeTimeoutException(recipe); + ctx.getOnError().accept(t); + ctx.getOnTimeout().accept(t, ctx); + } + return source; + } + + if (ctx.getMessage(PANIC) != null) { + return source; + } + + TreeVisitor visitor = recipe.getVisitor(); + // set root cursor as it is required by the `ScanningRecipe#isAcceptable()` + visitor.setCursor(rootCursor); + + after = recipeRunStats.recordEdit(recipe, () -> { + if (visitor.isAcceptable(source, ctx)) { + // propagate shared root cursor + return (SourceFile) visitor.visit(source, ctx, rootCursor); + } + return source; + }); + + if (after != source) { + madeChangesInThisCycle.add(recipe); + recordSourceFileResult(source, after, recipeStack, ctx); + if (source.getMarkers().findFirst(Generated.class).isPresent()) { + // skip edits made to generated source files so that they don't show up in a diff + // that later fails to apply on a freshly cloned repository + return source; + } + recipeRunStats.recordSourceFileChanged(source, after); + } else if (ctx.hasNewMessages()) { + // consider any recipes adding new messages as a changing recipe (which can request another cycle) + madeChangesInThisCycle.add(recipe); + ctx.resetHasNewMessages(); + } + } catch (Throwable t) { + after = handleError(recipe, source, after, t); + } + if (after != null && after != source) { + after = addRecipesThatMadeChanges(recipeStack, after); + } + return after; + }, sourceFile) + ); + } + + private void recordSourceFileResult(@Nullable SourceFile before, @Nullable SourceFile after, Stack recipeStack, ExecutionContext ctx) { + String beforePath = (before == null) ? "" : before.getSourcePath().toString(); + String afterPath = (after == null) ? "" : after.getSourcePath().toString(); + Recipe recipe = recipeStack.peek(); + Long effortSeconds = (recipe.getEstimatedEffortPerOccurrence() == null) ? 0L : recipe.getEstimatedEffortPerOccurrence().getSeconds(); + String parentName = ""; + boolean hierarchical = recipeStack.size() > 1; + if (hierarchical) { + parentName = recipeStack.get(recipeStack.size() - 2).getName(); + } + String recipeName = recipe.getName(); + sourcesFileResults.insertRow(ctx, new SourcesFileResults.Row( + beforePath, + afterPath, + parentName, + recipeName, + effortSeconds, + cycle)); + if (hierarchical) { + recordSourceFileResult(beforePath, afterPath, recipeStack.subList(0, recipeStack.size() - 1), effortSeconds, ctx); + } + } + + private void recordSourceFileResult(@Nullable String beforePath, @Nullable String afterPath, List recipeStack, Long effortSeconds, ExecutionContext ctx) { + if (recipeStack.size() <= 1) { + // No reason to record the synthetic root recipe which contains the recipe run + return; + } + String parentName; + if (recipeStack.size() == 2) { + // Record the parent name as blank rather than CompositeRecipe when the parent is the synthetic root recipe + parentName = ""; + } else { + parentName = recipeStack.get(recipeStack.size() - 2).getName(); + } + Recipe recipe = recipeStack.get(recipeStack.size() - 1); + sourcesFileResults.insertRow(ctx, new SourcesFileResults.Row( + beforePath, + afterPath, + parentName, + recipe.getName(), + effortSeconds, + cycle)); + recordSourceFileResult(beforePath, afterPath, recipeStack.subList(0, recipeStack.size() - 1), effortSeconds, ctx); + } + + @Nullable + private SourceFile handleError(Recipe recipe, SourceFile sourceFile, @Nullable SourceFile after, + Throwable t) { + ctx.getOnError().accept(t); + + if (t instanceof RecipeRunException) { + RecipeRunException vt = (RecipeRunException) t; + after = (SourceFile) new FindRecipeRunException(vt).visitNonNull(requireNonNull(after, "after is null"), 0); + } + + // Use the original source file to record the error, not the one that may have been modified by the visitor. + // This is so the error is associated with the original source file, and its original source path. + errorsTable.insertRow(ctx, new SourcesFileErrors.Row( + sourceFile.getSourcePath().toString(), + recipe.getName(), + ExceptionUtils.sanitizeStackTrace(t, RecipeScheduler.class) + )); + + return after; + } + + private static S addRecipesThatMadeChanges(List recipeStack, S afterFile) { + return afterFile.withMarkers(afterFile.getMarkers().computeByType( + RecipesThatMadeChanges.create(recipeStack), + (r1, r2) -> { + r1.getRecipes().addAll(r2.getRecipes()); + return r1; + }) + ); + } +} diff --git a/rewrite-core/src/main/java/org/openrewrite/scheduling/RecipeStack.java b/rewrite-core/src/main/java/org/openrewrite/scheduling/RecipeStack.java new file mode 100644 index 00000000000..9268bf1b4c3 --- /dev/null +++ b/rewrite-core/src/main/java/org/openrewrite/scheduling/RecipeStack.java @@ -0,0 +1,85 @@ +/* + * Copyright 2023 the original author or authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * https://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openrewrite.scheduling; + +import lombok.Getter; +import lombok.experimental.NonFinal; +import org.openrewrite.ExecutionContext; +import org.openrewrite.LargeSourceSet; +import org.openrewrite.Recipe; + +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; + +import static org.openrewrite.Recipe.PANIC; + +class RecipeStack { + private final Map> recipeLists = new IdentityHashMap<>(); + private Stack> allRecipesStack; + + /** + * The zero-based position of the recipe that is currently doing a scan/generate/edit. + */ + @NonFinal + @Getter + int recipePosition; + + public T reduce(LargeSourceSet sourceSet, Recipe recipe, ExecutionContext ctx, + BiFunction, T> consumer, T acc) { + init(recipe); + AtomicInteger recipePosition = new AtomicInteger(0); + while (!allRecipesStack.isEmpty()) { + if (ctx.getMessage(PANIC) != null) { + break; + } + + this.recipePosition = recipePosition.getAndIncrement(); + Stack recipeStack = allRecipesStack.pop(); + if (recipeStack.peek().maxCycles() >= ctx.getCycle()) { + sourceSet.setRecipe(recipeStack); + acc = consumer.apply(acc, recipeStack); + } + recurseRecipeList(recipeStack); + } + return acc; + } + + private void init(Recipe recipe) { + allRecipesStack = new Stack<>(); + Stack rootRecipeStack = new Stack<>(); + rootRecipeStack.push(recipe); + allRecipesStack.push(rootRecipeStack); + } + + private void recurseRecipeList(Stack recipeStack) { + List recipeList = getRecipeList(recipeStack); + for (int i = recipeList.size() - 1; i >= 0; i--) { + Recipe r = recipeList.get(i); + Stack nextStack = new Stack<>(); + nextStack.addAll(recipeStack); + nextStack.push(r); + allRecipesStack.push(nextStack); + } + } + + private List getRecipeList(Stack recipeStack) { + return recipeLists.computeIfAbsent(recipeStack.peek(), Recipe::getRecipeList); + } +} diff --git a/rewrite-core/src/main/java/org/openrewrite/scheduling/WatchableExecutionContext.java b/rewrite-core/src/main/java/org/openrewrite/scheduling/WatchableExecutionContext.java index 156b0b0dfd3..aba03bed03c 100644 --- a/rewrite-core/src/main/java/org/openrewrite/scheduling/WatchableExecutionContext.java +++ b/rewrite-core/src/main/java/org/openrewrite/scheduling/WatchableExecutionContext.java @@ -25,8 +25,7 @@ @RequiredArgsConstructor public class WatchableExecutionContext implements ExecutionContext { private final ExecutionContext delegate; - - private boolean hasNewMessages = false; + private boolean hasNewMessages; public boolean hasNewMessages() { return hasNewMessages; @@ -42,7 +41,7 @@ public void putMessage(String key, @Nullable Object value) { delegate.putMessage(key, value); } - public void putCycle(int cycle) { + public void putCycle(RecipeRunCycle cycle) { delegate.putMessage(CURRENT_CYCLE, cycle); } diff --git a/rewrite-core/src/main/java/org/openrewrite/scheduling/WorkingDirectoryExecutionContextView.java b/rewrite-core/src/main/java/org/openrewrite/scheduling/WorkingDirectoryExecutionContextView.java new file mode 100644 index 00000000000..9cb2514b2a1 --- /dev/null +++ b/rewrite-core/src/main/java/org/openrewrite/scheduling/WorkingDirectoryExecutionContextView.java @@ -0,0 +1,80 @@ +/* + * Copyright 2023 the original author or authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * https://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openrewrite.scheduling; + +import org.openrewrite.DelegatingExecutionContext; +import org.openrewrite.ExecutionContext; +import org.openrewrite.Incubating; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Recipes that need to write (or download somehow and read) resources from disk as part of their + * operation can use this view to receive a safe directory to use. This directory will not be created + * unless requested by a recipe, and will be deleted by {@link org.openrewrite.RecipeScheduler} at the + * end of each cycle. Each recipe in a recipe list will get its own directory to use so there is no + * cross-contamination of the directory between recipes. + */ +@Incubating(since = "8.12.0") +public class WorkingDirectoryExecutionContextView extends DelegatingExecutionContext { + public static final String WORKING_DIRECTORY_ROOT = "org.openrewrite.scheduling.workingDirectory"; + + private WorkingDirectoryExecutionContextView(ExecutionContext delegate) { + super(delegate); + } + + public static WorkingDirectoryExecutionContextView view(ExecutionContext ctx) { + return new WorkingDirectoryExecutionContextView(ctx); + } + + /** + * This should not be called from recipes, but only from tools that are creating instantiating + * recipe runs directly. These tools should select a directory that are appropriate to their + * context and ensure they are cleaned up. + * + * @param path The root directory from which individual recipe(+cycle) working directories will + * be created. + */ + public void setRoot(Path path) { + if (getMessage(CURRENT_CYCLE) != null) { + throw new IllegalStateException("The root working directory cannot be set once " + + "recipe execution has begun."); + } + putMessage(WORKING_DIRECTORY_ROOT, path); + } + + /** + * @return A working directory that a recipe may write to. Created only when a recipe + * requests it by calling this method, and deleted at the end of the recipe cycle. + */ + public Path getWorkingDirectory() { + try { + Path root = getMessage(WORKING_DIRECTORY_ROOT); + if (root == null) { + root = Files.createTempDirectory("rewrite-work"); + putMessage(WORKING_DIRECTORY_ROOT, root); + } + RecipeRunCycle cycle = getCycleDetails(); + return Files.createDirectories(root.resolve("cycle" + cycle.getCycle() + "_" + + "recipe" + cycle.getRecipePosition())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/rewrite-core/src/test/java/org/openrewrite/RecipeSchedulerTest.java b/rewrite-core/src/test/java/org/openrewrite/RecipeSchedulerTest.java index 8caaae7acf7..e56200423b6 100644 --- a/rewrite-core/src/test/java/org/openrewrite/RecipeSchedulerTest.java +++ b/rewrite-core/src/test/java/org/openrewrite/RecipeSchedulerTest.java @@ -16,16 +16,32 @@ package org.openrewrite; import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.openrewrite.config.DeclarativeRecipe; +import org.openrewrite.internal.lang.Nullable; import org.openrewrite.marker.Markup; +import org.openrewrite.scheduling.WorkingDirectoryExecutionContextView; import org.openrewrite.test.RewriteTest; import org.openrewrite.text.PlainText; import org.openrewrite.text.PlainTextVisitor; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.openrewrite.scheduling.WorkingDirectoryExecutionContextView.WORKING_DIRECTORY_ROOT; +import static org.openrewrite.test.RewriteTest.toRecipe; import static org.openrewrite.test.SourceSpecs.text; class RecipeSchedulerTest implements RewriteTest { @@ -56,6 +72,69 @@ void exceptionsCauseResult() { ) ); } + + @Test + void suppliedWorkingDirectoryRoot(@TempDir Path path) { + InMemoryExecutionContext ctx = new InMemoryExecutionContext(); + WorkingDirectoryExecutionContextView.view(ctx).setRoot(path); + AtomicInteger cycle = new AtomicInteger(0); + rewriteRun( + spec -> spec.executionContext(ctx).recipe(toRecipe(() -> new TreeVisitor<>() { + @Override + public Tree visit(@Nullable Tree tree, ExecutionContext ctx) { + assert tree != null; + PlainText plainText = (PlainText) tree; + Path workingDirectory = WorkingDirectoryExecutionContextView.view(ctx) + .getWorkingDirectory(); + assertThat(workingDirectory).hasParent(path); + if (cycle.incrementAndGet() == 2) { + assertThat(workingDirectory.resolve("foo.txt")).hasContent("foo"); + } + assertDoesNotThrow(() -> { + Files.writeString(workingDirectory.resolve("foo.txt"), plainText.getText()); + }); + return plainText.withText("bar"); + } + })), + text("foo", "bar") + ); + assertThat(path).doesNotExist(); + } + + @Test + void managedWorkingDirectoryWithRecipe(@TempDir Path path) { + InMemoryExecutionContext ctx = new InMemoryExecutionContext(); + WorkingDirectoryExecutionContextView.view(ctx).setRoot(path); + rewriteRun( + spec -> spec.executionContext(ctx).recipe(new RecipeWritingToFile(0)), + text("foo", "bar") + ); + assertThat(path).doesNotExist(); + } + + @Test + void managedWorkingDirectoryWithMultipleRecipes(@TempDir Path path) { + InMemoryExecutionContext ctx = new InMemoryExecutionContext(); + WorkingDirectoryExecutionContextView.view(ctx).setRoot(path); + DeclarativeRecipe recipe = new DeclarativeRecipe( + "root", + "Root recipe", + "Root recipe.", + emptySet(), + null, + URI.create("dummy:recipe.yml"), + false, + emptyList() + ); + recipe.addUninitialized(new RecipeWritingToFile(1)); + recipe.addUninitialized(new RecipeWritingToFile(2)); + recipe.initialize(List.of(), Map.of()); + rewriteRun( + spec -> spec.executionContext(ctx).recipe(recipe), + text("foo", "bar") + ); + assertThat(path).doesNotExist(); + } } @AllArgsConstructor @@ -96,3 +175,86 @@ public StackTraceElement[] getStackTrace() { .toArray(StackTraceElement[]::new); } } + +@AllArgsConstructor +class RecipeWritingToFile extends ScanningRecipe { + + final int position; + + @Override + public String getDisplayName() { + return "Write text to a file"; + } + + @Override + public String getDescription() { + return "Writes text to a file."; + } + + @Override + public Accumulator getInitialValue(ExecutionContext ctx) { + Path workingDirectory = validateExecutionContext(ctx); + return new Accumulator(workingDirectory); + } + + private Path validateExecutionContext(ExecutionContext ctx) { + Path workingDirectory = WorkingDirectoryExecutionContextView.view(ctx) + .getWorkingDirectory(); + assertThat(workingDirectory).isDirectory(); + assertThat(workingDirectory).hasParent(ctx.getMessage(WORKING_DIRECTORY_ROOT)); + assertThat(ctx.getCycleDetails().getRecipePosition()).isEqualTo(position); + assertThat(workingDirectory.getFileName().toString()) + .isEqualTo("cycle" + ctx.getCycle() + "_recipe" + position); + return workingDirectory; + } + + @Override + public TreeVisitor getScanner(Accumulator acc) { + return new TreeVisitor<>() { + @Override + public Tree visit(@Nullable Tree tree, ExecutionContext ctx) { + assert tree != null; + Path workingDirectory = validateExecutionContext(ctx); + assertThat(acc.workingDirectory()).isEqualTo(workingDirectory); + assertThat(workingDirectory).isEmptyDirectory(); + assertDoesNotThrow(() -> { + Files.writeString(workingDirectory.resolve("manifest.txt"), ((SourceFile) tree).getSourcePath().toString(), StandardOpenOption.APPEND, StandardOpenOption.CREATE); + }); + return tree; + } + }; + } + + @Override + public Collection generate(Accumulator acc, ExecutionContext ctx) { + Path workingDirectory = validateExecutionContext(ctx); + assertThat(acc.workingDirectory()).isEqualTo(workingDirectory); + assertThat(workingDirectory).isDirectoryContaining(path -> path.getFileName().toString().equals("manifest.txt")); + assertDoesNotThrow(() -> { + assertThat(workingDirectory.resolve("manifest.txt")).hasContent("file.txt"); + }); + return List.of(); + } + + @Override + public TreeVisitor getVisitor(Accumulator acc) { + return new TreeVisitor<>() { + @Override + public Tree visit(@Nullable Tree tree, ExecutionContext ctx) { + Path workingDirectory = WorkingDirectoryExecutionContextView + .view(ctx).getWorkingDirectory(); + assertThat(workingDirectory).isDirectory(); + assertThat(acc.workingDirectory()).isEqualTo(workingDirectory); + assertThat(workingDirectory).isDirectoryContaining(path -> path.getFileName().toString().equals("manifest.txt")); + assertDoesNotThrow(() -> { + assertThat(workingDirectory.resolve("manifest.txt")).hasContent("file.txt"); + }); + assert tree instanceof PlainText; + return ((PlainText) tree).withText("bar"); + } + }; + } + + public record Accumulator(Path workingDirectory) { + } +}