From c1b2376043dfe8b0900a6c113521e971a4ac6cec Mon Sep 17 00:00:00 2001 From: Alex Beggs Date: Mon, 8 Feb 2021 14:48:38 -0500 Subject: [PATCH] Add execute-command-before BuildMutator - BuildMutator allows commands to be executed prior to the SCENARIO or BUILD phase. Enabled for all types of builds, Gradle, Bazel, Buck, and Maven. Signed-off-by: Alex Beggs --- README.md | 41 ++++++ .../org/gradle/profiler/ScenarioLoader.java | 17 ++- .../org/gradle/profiler/ScenarioUtil.java | 44 ++++++ .../profiler/mutations/CommandInvoker.java | 16 ++ .../mutations/ExecuteCommandBuildMutator.java | 121 +++++++++++++++ .../ProcessBuilderCommandInvoker.java | 24 +++ .../ExecuteCommandBuildMutatorTest.groovy | 139 ++++++++++++++++++ 7 files changed, 396 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/gradle/profiler/ScenarioUtil.java create mode 100644 src/main/java/org/gradle/profiler/mutations/CommandInvoker.java create mode 100644 src/main/java/org/gradle/profiler/mutations/ExecuteCommandBuildMutator.java create mode 100644 src/main/java/org/gradle/profiler/mutations/ProcessBuilderCommandInvoker.java create mode 100644 src/test/groovy/org/gradle/profiler/mutations/ExecuteCommandBuildMutatorTest.groovy diff --git a/README.md b/README.md index 9e0303ea..33f58600 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,7 @@ A scenario can define changes that should be applied to the source before each b - `clear-project-cache-before`: Deletes the contents of the `.gradle` and `buildSrc/.gradle` project cache directories before the scenario is executed (`SCENARIO`), before cleanup (`CLEANUP`) or before the build is executed (`BUILD`). - `clear-transform-cache-before`: Deletes the contents of the transform cache before the scenario is executed (`SCENARIO`), before cleanup (`CLEANUP`) or before the build is executed (`BUILD`). - `clear-jars-cache-before`: Deletes the contents of the instrumented jars cache before the scenario is executed (`SCENARIO`), before cleanup (`CLEANUP`) or before the build is executed (`BUILD`). +- `execute-command-before`: Execute a command before the scenario is executed (SCENARIO) or before the build is executed (BUILD). This can be applied to Gradle, Bazel, Buck, or Maven builds. - `git-checkout`: Checks out a specific commit for the build step, and a different one for the cleanup step. - `git-revert`: Reverts a given set of commits before the build and resets it afterward. - `iterations`: Number of builds to actually measure @@ -325,11 +326,43 @@ You can compare Gradle against Bazel, Buck, and Maven by specifying their equiva build_some_target { tasks = ["assemble"] + execute-command-before = [ + # Gradle specific executions + # Execute a command in the bash shell + { + schedule = BUILD + commands = ["/bin/bash","-c","echo helloworld"] + }, + } bazel { # If empty, it will be infered from BAZEL_HOME environment variable home = "/path/to/bazel/home" targets = ["build" "//some/target"] + execute-command-before = [ + # Bazel specific executions + # execute a command prior to the scenario running. + { + schedule = SCENARIO + # Note: Ensure that this is calling the same Bazel that is used in the benchmarks + commands = ["bazel","clean","--expunge"] + }, + # Execute a command in the bash shell + { + schedule = BUILD + commands = ["/bin/bash","-c","echo helloworld"] + }, + # Remove the contents of the remote cache Bazel bucket + { + schedule = BUILD + commands = ["gsutil","-o","Credentials:gs_service_key_file=bazel-benchmark-bucket.json","-m","rm","gs://bazel-benchmark-bucket/**"] + }, + # Display the total size of the bucket + { + schedule = BUILD + commands = ["gsutil","-o","Credentials:gs_service_key_file=bazel-benchmark-bucket.json","du","-sh","gs://bazel-benchmark-bucket"] + }, + ] } } @@ -344,6 +377,14 @@ You can compare Gradle against Bazel, Buck, and Maven by specifying their equiva # If empty, it will be infered from BUCK_HOME environment variable home = "/path/to/buck/home" type = "android_binary" // can be a Buck build rule type or "all" + execute-command-before = [ + # Buck specific executions + # Execute a command in the bash shell + { + schedule = BUILD + commands = ["/bin/bash","-c","echo helloworld"] + }, + } } } build_resources { diff --git a/src/main/java/org/gradle/profiler/ScenarioLoader.java b/src/main/java/org/gradle/profiler/ScenarioLoader.java index 6d6d3fa6..05ccc67e 100644 --- a/src/main/java/org/gradle/profiler/ScenarioLoader.java +++ b/src/main/java/org/gradle/profiler/ScenarioLoader.java @@ -15,6 +15,7 @@ import org.gradle.profiler.mutations.ApplyNonAbiChangeToSourceFileMutator; import org.gradle.profiler.mutations.ApplyValueChangeToAndroidResourceFileMutator; import org.gradle.profiler.mutations.BuildMutatorConfigurator; +import org.gradle.profiler.mutations.ExecuteCommandBuildMutator; import org.gradle.profiler.mutations.ClearArtifactTransformCacheMutator; import org.gradle.profiler.mutations.ClearBuildCacheMutator; import org.gradle.profiler.mutations.ClearConfigurationCacheStateMutator; @@ -37,6 +38,10 @@ import java.util.stream.Collectors; class ScenarioLoader { + + static final String BAZEL = "bazel"; + static final String BUCK = "buck"; + static final String MAVEN = "maven"; private static final String TITLE = "title"; private static final String VERSIONS = "versions"; private static final String TASKS = "tasks"; @@ -45,9 +50,6 @@ class ScenarioLoader { private static final String RUN_USING = "run-using"; private static final String DAEMON = "daemon"; private static final String SYSTEM_PROPERTIES = "system-properties"; - private static final String BAZEL = "bazel"; - private static final String BUCK = "buck"; - private static final String MAVEN = "maven"; private static final String TOOL_HOME = "home"; private static final String WARM_UP_COUNT = "warm-ups"; private static final String ITERATIONS = "iterations"; @@ -61,6 +63,7 @@ class ScenarioLoader { private static final String APPLY_ANDROID_MANIFEST_CHANGE_TO = "apply-android-manifest-change-to"; private static final String APPLY_CPP_SOURCE_CHANGE_TO = "apply-cpp-change-to"; private static final String APPLY_H_SOURCE_CHANGE_TO = "apply-h-change-to"; + private static final String EXECUTE_COMMAND_BEFORE = "execute-command-before"; private static final String CLEAR_BUILD_CACHE_BEFORE = "clear-build-cache-before"; private static final String CLEAR_GRADLE_USER_HOME_BEFORE = "clear-gradle-user-home-before"; private static final String CLEAR_INSTANT_EXECUTION_STATE_BEFORE = "clear-instant-execution-state-before"; @@ -97,6 +100,7 @@ class ScenarioLoader { .put(SHOW_BUILD_CACHE_SIZE, new ShowBuildCacheSizeMutator.Configurator()) .put(GIT_CHECKOUT, new GitCheckoutMutator.Configurator()) .put(GIT_REVERT, new GitRevertMutator.Configurator()) + .put(EXECUTE_COMMAND_BEFORE, new ExecuteCommandBuildMutator.Configurator()) .build(); private static final List ALL_SCENARIO_KEYS = ImmutableList.builder() @@ -105,6 +109,7 @@ class ScenarioLoader { TITLE, VERSIONS, TASKS, + EXECUTE_COMMAND_BEFORE, CLEANUP_TASKS, GRADLE_ARGS, RUN_USING, @@ -123,9 +128,9 @@ class ScenarioLoader { )) .build(); - private static final List BAZEL_KEYS = Arrays.asList(TARGETS, TOOL_HOME); - private static final List BUCK_KEYS = Arrays.asList(TARGETS, TYPE, TOOL_HOME); - private static final List MAVEN_KEYS = Arrays.asList(TARGETS, TOOL_HOME); + private static final List BAZEL_KEYS = Arrays.asList(TARGETS, TOOL_HOME, EXECUTE_COMMAND_BEFORE); + private static final List BUCK_KEYS = Arrays.asList(TARGETS, TYPE, TOOL_HOME, EXECUTE_COMMAND_BEFORE); + private static final List MAVEN_KEYS = Arrays.asList(TARGETS, TOOL_HOME, EXECUTE_COMMAND_BEFORE); private final GradleBuildConfigurationReader gradleBuildConfigurationReader; diff --git a/src/main/java/org/gradle/profiler/ScenarioUtil.java b/src/main/java/org/gradle/profiler/ScenarioUtil.java new file mode 100644 index 00000000..f4f9cf2d --- /dev/null +++ b/src/main/java/org/gradle/profiler/ScenarioUtil.java @@ -0,0 +1,44 @@ +package org.gradle.profiler; + +import static org.gradle.profiler.ScenarioLoader.BAZEL; +import static org.gradle.profiler.ScenarioLoader.BUCK; +import static org.gradle.profiler.ScenarioLoader.MAVEN; + +import com.typesafe.config.Config; +import javax.annotation.Nullable; + +public class ScenarioUtil { + + /** + * Returns the specific config for type of build that is running, + * if the scenario doesn't define the build, then this default value is returned + * + * @param rootScenario the root scenario config + * @param settings the invocation settings that indicates the type of build selected + * @param defaultScenario if the scenario doesn't define the build, then this default value is returned, this can be null. + * @return if the build config exists or if the scenario doesn't define the build, then this default value is returned + */ + public static Config getBuildConfig(Config rootScenario, InvocationSettings settings, @Nullable + Config defaultScenario) { + Config scenario; + if (settings.isBazel()) { + scenario = getConfigOrDefault(rootScenario, BAZEL, defaultScenario); + } else if (settings.isBuck()) { + scenario = getConfigOrDefault(rootScenario, BUCK, defaultScenario); + } else if (settings.isMaven()) { + scenario = getConfigOrDefault(rootScenario, MAVEN, defaultScenario); + } else { + scenario = rootScenario; + } + return scenario; + } + + private static Config getConfigOrDefault(Config rootScenario, String key, @Nullable Config defaultScenario) { + Config scenario; + if (rootScenario.hasPath(key)) { + scenario = rootScenario.getConfig(key); + } else { + scenario = defaultScenario; + } return scenario; + } +} diff --git a/src/main/java/org/gradle/profiler/mutations/CommandInvoker.java b/src/main/java/org/gradle/profiler/mutations/CommandInvoker.java new file mode 100644 index 00000000..a4d75c79 --- /dev/null +++ b/src/main/java/org/gradle/profiler/mutations/CommandInvoker.java @@ -0,0 +1,16 @@ +package org.gradle.profiler.mutations; + +import java.util.List; + +/** + * Interface which provides a means to execute a command + */ +public interface CommandInvoker { + + /** + * @param command the command to execute + * @return the exit code of the result of executing the command + */ + int execute(List command); + +} diff --git a/src/main/java/org/gradle/profiler/mutations/ExecuteCommandBuildMutator.java b/src/main/java/org/gradle/profiler/mutations/ExecuteCommandBuildMutator.java new file mode 100644 index 00000000..35133796 --- /dev/null +++ b/src/main/java/org/gradle/profiler/mutations/ExecuteCommandBuildMutator.java @@ -0,0 +1,121 @@ +package org.gradle.profiler.mutations; + +import static org.gradle.profiler.ScenarioUtil.getBuildConfig; + +import com.google.common.annotations.VisibleForTesting; +import com.typesafe.config.Config; +import java.util.ArrayList; +import java.util.List; +import org.gradle.profiler.BuildContext; +import org.gradle.profiler.BuildMutator; +import org.gradle.profiler.CompositeBuildMutator; +import org.gradle.profiler.ConfigUtil; +import org.gradle.profiler.InvocationSettings; +import org.gradle.profiler.ScenarioContext; + +public class ExecuteCommandBuildMutator implements BuildMutator { + + private ExecuteCommandSchedule schedule; + private List commands; + private CommandInvoker commandInvoker; + + public ExecuteCommandBuildMutator(ExecuteCommandSchedule schedule, + List commands, CommandInvoker commandInvoker) { + this.schedule = schedule; + this.commands = commands; + this.commandInvoker = commandInvoker; + } + + @Override + public void beforeBuild(BuildContext context) { + if (schedule == ExecuteCommandSchedule.BUILD) { + execute(); + } + } + + @Override + public void beforeScenario(ScenarioContext context) { + if (schedule == ExecuteCommandSchedule.SCENARIO) { + execute(); + } + } + + protected void execute() { + String commandStr = String.join(" ", commands); + System.out.println(String.format("> Executing command `%s`", commandStr)); + int result = commandInvoker.execute(commands); + if (result != 0) { + System.err.println( + String.format("Unexpected exit code %s for command `%s`", result, commandStr) + ); + } + } + + public static class Configurator implements BuildMutatorConfigurator { + + private CommandInvoker commandInvoker; + + public Configurator() { + this(new ProcessBuilderCommandInvoker()); + } + + @VisibleForTesting + Configurator(CommandInvoker commandInvoker) { + this.commandInvoker = commandInvoker; + } + + private BuildMutator newInstance(Config scenario, String scenarioName, + InvocationSettings settings, String key, + CommandInvoker commandInvoker, ExecuteCommandSchedule schedule, List commands) { + return new ExecuteCommandBuildMutator(schedule, commands, commandInvoker); + } + + @Override + public BuildMutator configure(Config rootScenario, String scenarioName, + InvocationSettings settings, String key) { + if (enabled(rootScenario, scenarioName, settings, key)) { + Config scenario = getBuildConfig(rootScenario, settings, null); + final List mutators = new ArrayList<>(); + final List list = scenario.getConfigList(key); + for (Config config : list) { + final ExecuteCommandSchedule schedule = ConfigUtil + .enumValue(config, "schedule", ExecuteCommandSchedule.class, null); + if (schedule == null) { + throw new IllegalArgumentException( + "Schedule for executing commands is not specified"); + } + List commands = ConfigUtil.strings(config, "commands"); + if (commands.isEmpty()) { + throw new IllegalArgumentException( + String.format( + "No commands specified for 'execute-command-before' in scenario %s", + scenarioName) + ); + } + mutators.add( + newInstance(scenario, scenarioName, settings, key, commandInvoker, schedule, + commands)); + } + return new CompositeBuildMutator(mutators); + } else { + return BuildMutator.NOOP; + } + } + + private boolean enabled(Config rootScenario, String scenarioName, InvocationSettings settings, String key) { + Config scenario = getBuildConfig(rootScenario, settings, null); + return scenario != null && scenario.hasPath(key) && !scenario.getConfigList(key) + .isEmpty(); + } + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(" + schedule + ")"; + } + + public enum ExecuteCommandSchedule { + SCENARIO, BUILD + } + +} diff --git a/src/main/java/org/gradle/profiler/mutations/ProcessBuilderCommandInvoker.java b/src/main/java/org/gradle/profiler/mutations/ProcessBuilderCommandInvoker.java new file mode 100644 index 00000000..bac9b93d --- /dev/null +++ b/src/main/java/org/gradle/profiler/mutations/ProcessBuilderCommandInvoker.java @@ -0,0 +1,24 @@ +package org.gradle.profiler.mutations; + +import java.io.IOException; +import java.util.List; + +public class ProcessBuilderCommandInvoker implements CommandInvoker { + + @Override + public int execute(List command) { + try { + if (command == null || command.isEmpty()) { + throw new IllegalArgumentException( + String.format("command cannot be null or empty, was %s", command)); + } + ProcessBuilder processBuilder = new ProcessBuilder(command) + .redirectOutput(ProcessBuilder.Redirect.INHERIT) + .redirectError(ProcessBuilder.Redirect.INHERIT); + Process process = processBuilder.start(); + return process.waitFor(); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/test/groovy/org/gradle/profiler/mutations/ExecuteCommandBuildMutatorTest.groovy b/src/test/groovy/org/gradle/profiler/mutations/ExecuteCommandBuildMutatorTest.groovy new file mode 100644 index 00000000..dfeb62f3 --- /dev/null +++ b/src/test/groovy/org/gradle/profiler/mutations/ExecuteCommandBuildMutatorTest.groovy @@ -0,0 +1,139 @@ +package org.gradle.profiler.mutations + + +import com.typesafe.config.ConfigFactory +import org.gradle.profiler.BuildInvoker +import org.gradle.profiler.InvocationSettings +import org.gradle.profiler.report.CsvGenerator + +class ExecuteCommandBuildMutatorTest extends AbstractMutatorTest { + + + def static createInvocationSettings(BuildInvoker buildInvoker, String target) { + return new InvocationSettings(new File("."), + null, + true, + new File("outputdir"), + buildInvoker, + false, + new File("scenariofile"), + Collections.emptyList(), + Arrays.asList(target), + Collections.emptyMap(), + new File("gradlehome"), + new File("studioinstall"), + 5, + 10, + false, + false, + Collections.emptyList(), + CsvGenerator.Format.LONG, + "benchmarkTitle", + new File("buildLog") + ) + } + + def invoker = new CommandInvoker() { + private List> commands + + @Override + int execute(final List commands) throws InterruptedException, IOException { + this.commands.add(commands) + return 0 + } + + void reset() { + commands = new ArrayList<>() + } + } + + def setup() { + invoker.reset() + } + + def "executes the commands before scenario starts when Schedule is SCENARIO"() { + def mutator = new ExecuteCommandBuildMutator(ExecuteCommandBuildMutator.ExecuteCommandSchedule.SCENARIO, + Arrays.asList("echo", "Hello World"), + invoker) + + when: + mutator.beforeScenario(scenarioContext) + then: + invoker.commands.size() == 1 + invoker.commands.get(0) == Arrays.asList("echo", "Hello World") + } + + def "doesn't execute the commands before scenario starts when Schedule is BUILD"() { + def mutator = new ExecuteCommandBuildMutator(ExecuteCommandBuildMutator.ExecuteCommandSchedule.BUILD, + Arrays.asList("echo", "Hello World"), + invoker) + + when: + mutator.beforeScenario(scenarioContext) + then: + invoker.commands.isEmpty() + } + + def "Configure ExecuteCommandBuildMutator from scenario"() { + def mutator = new ExecuteCommandBuildMutator(ExecuteCommandBuildMutator.ExecuteCommandSchedule.BUILD, + Arrays.asList("echo", "Hello World"), + invoker) + + when: + mutator.beforeScenario(scenarioContext) + then: + invoker.commands.isEmpty() + } + + def "execute the commands before scenario starts when Schedule is BUILD"() { + def mutator = new ExecuteCommandBuildMutator(ExecuteCommandBuildMutator.ExecuteCommandSchedule.BUILD, + Arrays.asList("echo", "Hello World"), + invoker) + + when: + mutator.beforeBuild(buildContext) + then: + invoker.commands.size() == 1 + invoker.commands.get(0) == Arrays.asList("echo", "Hello World") + } + + def "Given Gradle scenario verify that the configuration creates a mutator that executes"() { + given: + def config = ConfigFactory.parseString( + """ + testScenario { + tasks = ["tasks"] + clear-build-cache-before = SCENARIO + gradle-args = [ + "-Pbuild.cache.local.enabled=true" + ] + execute-command-before = [ + { + schedule = SCENARIO + commands = ["gsutil", "version"] + }, + { + schedule = BUILD + commands = ["/bin/bash", "-c", "echo helloworld"] + } + ] + } + """ + ) + + def mutator = new ExecuteCommandBuildMutator.Configurator(invoker).configure( + config.getConfig("testScenario"), + "testScenario", + createInvocationSettings(null, "testScenario"), + "execute-command-before" + ) + when: + mutator.beforeScenario(scenarioContext) + mutator.beforeBuild(buildContext) + then: + invoker.commands.size() == 2 + invoker.commands.get(0) == Arrays.asList("gsutil", "version") + invoker.commands.get(1) == Arrays.asList("/bin/bash", "-c", "echo helloworld") + } + +}