From 49d1354c0bac63a9acdc923a178e621376dff623 Mon Sep 17 00:00:00 2001 From: Abduqodiri Qurbonzoda Date: Tue, 10 Sep 2019 16:47:18 +0300 Subject: [PATCH] Implement level configuration at which fixture method should run --- .../kotlin/FixtureLevelBenchmark.kt | 36 ++++++++ .../benchmark/gradle/SuiteSourceGenerator.kt | 91 ++++++++++++++----- .../benchmark/CommonBenchmarkAnnotations.kt | 7 +- .../src/kotlinx/benchmark/SuiteDescriptor.kt | 12 ++- .../benchmark/JsBenchmarkAnnotations.kt | 8 +- .../src/kotlinx/benchmark/js/JsExecutor.kt | 13 ++- .../benchmark/JvmBenchmarkAnnotations.kt | 4 + .../benchmark/NativeBenchmarkAnnotations.kt | 8 +- .../benchmark/native/NativeExecutor.kt | 80 +++++++++++++--- 9 files changed, 213 insertions(+), 46 deletions(-) create mode 100644 examples/kotlin-multiplatform/src/commonMain/kotlin/FixtureLevelBenchmark.kt diff --git a/examples/kotlin-multiplatform/src/commonMain/kotlin/FixtureLevelBenchmark.kt b/examples/kotlin-multiplatform/src/commonMain/kotlin/FixtureLevelBenchmark.kt new file mode 100644 index 00000000..705cfc2f --- /dev/null +++ b/examples/kotlin-multiplatform/src/commonMain/kotlin/FixtureLevelBenchmark.kt @@ -0,0 +1,36 @@ +package test + +import kotlinx.benchmark.* +import kotlin.math.* + +@State(Scope.Benchmark) +@Measurement(iterations = 3, time = 1, timeUnit = BenchmarkTimeUnit.SECONDS) +@OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) +@BenchmarkMode(Mode.Throughput) +class FixtureLevelBenchmark { + private var data: Double = 0.0 + + private var iterations: Int = 0 + + @Setup(Level.Trial) + fun trialSetup() { + data = 3.0 + iterations = 0 + } + + + @Setup(Level.Iteration) + fun iterationSetup() { + iterations++ + } + + @TearDown(Level.Trial) + fun trialTearDown() { +// println("Total iterations = $iterations") + } + + @Benchmark + fun mathBenchmark(): Double { + return log(sqrt(data) * cos(data), 2.0) + } +} \ No newline at end of file diff --git a/plugin/main/src/kotlinx/benchmark/gradle/SuiteSourceGenerator.kt b/plugin/main/src/kotlinx/benchmark/gradle/SuiteSourceGenerator.kt index 4c71275b..f1321665 100644 --- a/plugin/main/src/kotlinx/benchmark/gradle/SuiteSourceGenerator.kt +++ b/plugin/main/src/kotlinx/benchmark/gradle/SuiteSourceGenerator.kt @@ -18,8 +18,11 @@ enum class Platform { class SuiteSourceGenerator(val title: String, val module: ModuleDescriptor, val output: File, val platform: Platform) { companion object { - val setupFunctionName = "setUp" - val teardownFunctionName = "tearDown" + val fixtureFunctionLevels = listOf("Invocation", "Iteration", "Trial") + val defaultFixtureFunctionLevel = "Trial" + val setupFunctionSuffix = "Setup" + val teardownFunctionSuffix = "TearDown" + val parametersFunctionName = "parametrize" val externalConfigurationFQN = "kotlinx.benchmark.ExternalConfiguration" @@ -136,31 +139,41 @@ class SuiteSourceGenerator(val title: String, val module: ModuleDescriptor, val val benchmarkFunctions = functions.filter { it.annotations.any { it.fqName.toString() == benchmarkAnnotationFQN } } - val setupFunctions = functions - .filter { it.annotations.any { it.fqName.toString() == setupAnnotationFQN } } + val levelToSetupFunctions = fixtureFunctionLevels.associateWith { level -> + functions.filterFixtureFunctions(setupAnnotationFQN, level, level == defaultFixtureFunctionLevel, false) + } - val teardownFunctions = functions - .filter { it.annotations.any { it.fqName.toString() == teardownAnnotationFQN } }.reversed() + val levelToTeardownFunctions = fixtureFunctionLevels.associateWith { level -> + functions.filterFixtureFunctions(teardownAnnotationFQN, level, level == defaultFixtureFunctionLevel, true) + } val file = FileSpec.builder(mainBenchmarkPackage, benchmarkName).apply { declareObject(benchmarkClass) { addAnnotation(suppressUnusedParameter) - function(setupFunctionName) { - addModifiers(KModifier.PRIVATE) - addParameter("instance", originalClass) - for (fn in setupFunctions) { - val functionName = fn.name.toString() - addStatement("instance.%N()", functionName) + for ((level, setupFunctions) in levelToSetupFunctions) { + val setupFunctionName = setupFunctionName(level) + + function(setupFunctionName) { + addModifiers(KModifier.PRIVATE) + addParameter("instance", originalClass) + for (fn in setupFunctions) { + val functionName = fn.name.toString() + addStatement("instance.%N()", functionName) + } } } - function(teardownFunctionName) { - addModifiers(KModifier.PRIVATE) - addParameter("instance", originalClass) - for (fn in teardownFunctions) { - val functionName = fn.name.toString() - addStatement("instance.%N()", functionName) + for ((level, teardownFunctions) in levelToTeardownFunctions) { + val teardownFunctionName = teardownFunctionName(level) + + function(teardownFunctionName) { + addModifiers(KModifier.PRIVATE) + addParameter("instance", originalClass) + for (fn in teardownFunctions) { + val functionName = fn.name.toString() + addStatement("instance.%N()", functionName) + } } } @@ -201,15 +214,27 @@ class SuiteSourceGenerator(val title: String, val module: ModuleDescriptor, val function("describe") { returns(suiteDescriptorType.parameterizedBy(originalClass)) addCode( - "«val descriptor = %T(name = %S, factory = ::%T, setup = ::%N, teardown = ::%N, parametrize = ::%N", + "«val descriptor = %T(name = %S, factory = ::%T", suiteDescriptorType, originalName, - originalClass, - setupFunctionName, - teardownFunctionName, - parametersFunctionName + originalClass ) + val hasInvocationFixture = listOf(levelToSetupFunctions, levelToTeardownFunctions).any { + it.getValue("Invocation").isNotEmpty() + } + val setupFunctions = fixtureFunctionLevels.joinToString(separator = ", ") { level -> + val functionName = setupFunctionName(level) + "$functionName = ::$functionName" + } + val teardownFunctions = fixtureFunctionLevels.joinToString(separator = ", ") { level -> + val functionName = teardownFunctionName(level) + "$functionName = ::$functionName" + } + addCode(", hasInvocationFixture = $hasInvocationFixture, $setupFunctions, $teardownFunctions") + + addCode(", parametrize = ::%N", parametersFunctionName) + val params = parameterProperties.joinToString(prefix = "listOf(", postfix = ")") { "\"${it.name}\"" } addCode(", parameters = $params") @@ -258,6 +283,26 @@ class SuiteSourceGenerator(val title: String, val module: ModuleDescriptor, val file.writeTo(output) } + + private fun setupFunctionName(level: String) = fixtureFunctionName(level, setupFunctionSuffix) + + private fun teardownFunctionName(level: String) = fixtureFunctionName(level, teardownFunctionSuffix) + + private fun fixtureFunctionName(level: String, suffix: String) = level.decapitalize() + suffix +} + +private fun List.filterFixtureFunctions( + annotationFQN: String, + level: String, + isDefaultLevel: Boolean, + reversed: Boolean +): List { + return let { if (reversed) it.reversed() else it } + .filter { + val annotation = it.annotations.firstOrNull { it.fqName.toString() == annotationFQN } ?: return@filter false + val levelValue = annotation.argumentValue("value") as? EnumValue + return@filter levelValue?.enumEntryName?.toString()?.let { it == level } ?: isDefaultLevel + } } inline fun codeBlock(builderAction: CodeBlock.Builder.() -> Unit): CodeBlock { diff --git a/runtime/commonMain/src/kotlinx/benchmark/CommonBenchmarkAnnotations.kt b/runtime/commonMain/src/kotlinx/benchmark/CommonBenchmarkAnnotations.kt index e5e8d99c..7383304f 100644 --- a/runtime/commonMain/src/kotlinx/benchmark/CommonBenchmarkAnnotations.kt +++ b/runtime/commonMain/src/kotlinx/benchmark/CommonBenchmarkAnnotations.kt @@ -1,11 +1,14 @@ package kotlinx.benchmark @Target(AnnotationTarget.FUNCTION) +expect annotation class Setup(val value: Level = Level.Trial) -expect annotation class Setup() +expect enum class Level { + Trial, Iteration, Invocation +} @Target(AnnotationTarget.FUNCTION) -expect annotation class TearDown() +expect annotation class TearDown(val value: Level = Level.Trial) @Target(AnnotationTarget.FUNCTION) expect annotation class Benchmark() diff --git a/runtime/commonMain/src/kotlinx/benchmark/SuiteDescriptor.kt b/runtime/commonMain/src/kotlinx/benchmark/SuiteDescriptor.kt index 2ceccbae..39aef5d7 100644 --- a/runtime/commonMain/src/kotlinx/benchmark/SuiteDescriptor.kt +++ b/runtime/commonMain/src/kotlinx/benchmark/SuiteDescriptor.kt @@ -4,8 +4,16 @@ open class SuiteDescriptor( val name: String, val factory: () -> T, val parametrize: (T, Map) -> Unit, - val setup: (T) -> Unit, - val teardown: (T) -> Unit, + + val hasInvocationFixture: Boolean, + + val invocationSetup: (T) -> Unit, + val iterationSetup: (T) -> Unit, + val trialSetup: (T) -> Unit, + + val invocationTearDown: (T) -> Unit, + val iterationTearDown: (T) -> Unit, + val trialTearDown: (T) -> Unit, val parameters: List, val defaultParameters: Map>, diff --git a/runtime/jsMain/src/kotlinx/benchmark/JsBenchmarkAnnotations.kt b/runtime/jsMain/src/kotlinx/benchmark/JsBenchmarkAnnotations.kt index 6c7d8c4c..2e1c0724 100644 --- a/runtime/jsMain/src/kotlinx/benchmark/JsBenchmarkAnnotations.kt +++ b/runtime/jsMain/src/kotlinx/benchmark/JsBenchmarkAnnotations.kt @@ -4,9 +4,13 @@ actual enum class Scope { Benchmark } +actual enum class Level { + Trial, Iteration, Invocation +} + actual annotation class State(actual val value: Scope) -actual annotation class Setup -actual annotation class TearDown +actual annotation class Setup(actual val value: Level) +actual annotation class TearDown(actual val value: Level) actual annotation class Benchmark diff --git a/runtime/jsMain/src/kotlinx/benchmark/js/JsExecutor.kt b/runtime/jsMain/src/kotlinx/benchmark/js/JsExecutor.kt index 420f9c41..8d47645f 100644 --- a/runtime/jsMain/src/kotlinx/benchmark/js/JsExecutor.kt +++ b/runtime/jsMain/src/kotlinx/benchmark/js/JsExecutor.kt @@ -21,6 +21,11 @@ class JsExecutor(name: String, @Suppress("UNUSED_PARAMETER") dummy_args: Array val suite = benchmark.suite + + if (suite.hasInvocationFixture) { + throw UnsupportedOperationException("Fixture methods with `Invocation` level are not supported") + } + val config = BenchmarkConfiguration(runnerConfiguration, suite) val jsDescriptor = benchmark as JsBenchmarkDescriptor @@ -63,7 +68,8 @@ class JsExecutor(name: String, @Suppress("UNUSED_PARAMETER") dummy_args: Array reporter.startBenchmark(executionName, id) - suite.setup(instance) + suite.trialSetup(instance) + suite.iterationSetup(instance) } var iteration = 0 jsBenchmark.on("cycle") { event -> @@ -76,9 +82,12 @@ class JsExecutor(name: String, @Suppress("UNUSED_PARAMETER") dummy_args: Array - suite.teardown(instance) + suite.iterationTearDown(instance) + suite.trialTearDown(instance) val stats = event.target.stats val samples = stats.sample .unsafeCast() diff --git a/runtime/jvmMain/src/kotlinx/benchmark/JvmBenchmarkAnnotations.kt b/runtime/jvmMain/src/kotlinx/benchmark/JvmBenchmarkAnnotations.kt index 2f11fe9f..4417657e 100644 --- a/runtime/jvmMain/src/kotlinx/benchmark/JvmBenchmarkAnnotations.kt +++ b/runtime/jvmMain/src/kotlinx/benchmark/JvmBenchmarkAnnotations.kt @@ -4,8 +4,12 @@ actual typealias Scope = org.openjdk.jmh.annotations.Scope actual typealias State = org.openjdk.jmh.annotations.State +@Suppress("ACTUAL_ANNOTATION_CONFLICTING_DEFAULT_ARGUMENT_VALUE") actual typealias Setup = org.openjdk.jmh.annotations.Setup +actual typealias Level = org.openjdk.jmh.annotations.Level + +@Suppress("ACTUAL_ANNOTATION_CONFLICTING_DEFAULT_ARGUMENT_VALUE") actual typealias TearDown = org.openjdk.jmh.annotations.TearDown actual typealias Benchmark = org.openjdk.jmh.annotations.Benchmark diff --git a/runtime/nativeMain/src/kotlinx/benchmark/NativeBenchmarkAnnotations.kt b/runtime/nativeMain/src/kotlinx/benchmark/NativeBenchmarkAnnotations.kt index 6c7d8c4c..2e1c0724 100644 --- a/runtime/nativeMain/src/kotlinx/benchmark/NativeBenchmarkAnnotations.kt +++ b/runtime/nativeMain/src/kotlinx/benchmark/NativeBenchmarkAnnotations.kt @@ -4,9 +4,13 @@ actual enum class Scope { Benchmark } +actual enum class Level { + Trial, Iteration, Invocation +} + actual annotation class State(actual val value: Scope) -actual annotation class Setup -actual annotation class TearDown +actual annotation class Setup(actual val value: Level) +actual annotation class TearDown(actual val value: Level) actual annotation class Benchmark diff --git a/runtime/nativeMain/src/kotlinx/benchmark/native/NativeExecutor.kt b/runtime/nativeMain/src/kotlinx/benchmark/native/NativeExecutor.kt index ef67c022..4b82671f 100644 --- a/runtime/nativeMain/src/kotlinx/benchmark/native/NativeExecutor.kt +++ b/runtime/nativeMain/src/kotlinx/benchmark/native/NativeExecutor.kt @@ -19,28 +19,29 @@ class NativeExecutor(name: String, args: Array) : SuiteExecutor(name val instance = suite.factory() // TODO: should we create instance per bench or per suite? suite.parametrize(instance, params) - benchmark.suite.setup(instance) + benchmark.suite.trialSetup(instance) reporter.startBenchmark(executionName, id) var exception: Throwable? = null val samples = try { - // Execute warmup - val cycles = warmup(suite.name, config, instance, benchmark) - DoubleArray(config.iterations) { iteration -> - val nanosecondsPerOperation = measure(instance, benchmark, cycles) - val text = nanosecondsPerOperation.nanosToText(config.mode, config.outputTimeUnit) - reporter.output( - executionName, - id, - "Iteration #$iteration: $text" - ) - nanosecondsPerOperation.nanosToSample(config.mode, config.outputTimeUnit) + val nanosecondsPerOperation = if (suite.hasInvocationFixture) { + // Skip warmup + DoubleArray(config.iterations) { iteration -> + measureWithInvocationFixtures(config, instance, benchmark).also { report(it, iteration, id, config) } + } + } else { + // Execute warmup + val cycles = warmup(suite.name, config, instance, benchmark) + DoubleArray(config.iterations) { iteration -> + measure(instance, benchmark, cycles).also { report(it, iteration, id, config) } + } } + nanosecondsPerOperation.map { it.nanosToSample(config.mode, config.outputTimeUnit) }.toDoubleArray() } catch (e: Throwable) { exception = e doubleArrayOf() } finally { - benchmark.suite.teardown(instance) + benchmark.suite.trialTearDown(instance) } if (exception == null) { @@ -69,6 +70,20 @@ class NativeExecutor(name: String, args: Array) : SuiteExecutor(name } complete() } + + private fun report( + nanosecondsPerOperation: Double, + iteration: Int, + id: String, + config: BenchmarkConfiguration + ) { + val text = nanosecondsPerOperation.nanosToText(config.mode, config.outputTimeUnit) + reporter.output( + executionName, + id, + "Iteration #$iteration: $text" + ) + } private fun Throwable.stacktrace(): String { val nested = cause ?: return getStackTrace().joinToString("\n") @@ -80,6 +95,8 @@ class NativeExecutor(name: String, args: Array) : SuiteExecutor(name benchmark: BenchmarkDescriptor, cycles: Int ): Double { + benchmark.suite.iterationSetup(instance) + val executeFunction = benchmark.function var counter = cycles val startTime = getTimeNanos() @@ -89,9 +106,42 @@ class NativeExecutor(name: String, args: Array) : SuiteExecutor(name } val endTime = getTimeNanos() val time = endTime - startTime + + benchmark.suite.iterationTearDown(instance) + return time.toDouble() / cycles } + private fun measureWithInvocationFixtures( + config: BenchmarkConfiguration, + instance: T, + benchmark: BenchmarkDescriptor + ): Double { + benchmark.suite.iterationSetup(instance) + + val benchmarkNanos = config.iterationTime * config.iterationTimeUnit.toMultiplier() + val startTime = getTimeNanos() + var endTime = startTime + var executionTime = 0L + val executeFunction = benchmark.function + var invocations = 0 + while (endTime - startTime < benchmarkNanos) { + benchmark.suite.invocationSetup(instance) + val timeBeforeExecution = getTimeNanos() + @Suppress("UNUSED_VARIABLE") + val result = instance.executeFunction() // ignore result for now, but might need to consume it somehow + executionTime += getTimeNanos() - timeBeforeExecution + benchmark.suite.invocationTearDown(instance) + + invocations++ + endTime = getTimeNanos() + } + + benchmark.suite.iterationTearDown(instance) + + return executionTime.toDouble() / invocations + } + private fun warmup( name: String, config: BenchmarkConfiguration, @@ -100,6 +150,8 @@ class NativeExecutor(name: String, args: Array) : SuiteExecutor(name ): Int { var iterations = 0 repeat(config.warmups) { iteration -> + benchmark.suite.iterationSetup(instance) + val benchmarkNanos = config.iterationTime * config.iterationTimeUnit.toMultiplier() val startTime = getTimeNanos() var endTime = startTime @@ -114,6 +166,8 @@ class NativeExecutor(name: String, args: Array) : SuiteExecutor(name val metric = time.toDouble() / iterations // TODO: metric val sample = metric.nanosToText(config.mode, config.outputTimeUnit) reporter.output(name, benchmark.name, "Warm-up #$iteration: $sample") + + benchmark.suite.iterationTearDown(instance) } return iterations }