Provide ability to exclude queries from the measurement and execute a…

…ll queries together #8
IncQueryLabs · Dec 4, 2019 · cf866af · cf866af
1 parent ebbcc4d
commit cf866af
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 9 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -13,10 +13,16 @@ pipeline {
 
 		string( 
 			defaultValue: 'transitiveSubstatesWithCheck3',
-			description: 'Comma separated list of queries to test (with a single space after each comma).',
+			description: 'Comma separated list of queries to test (with a single space after each comma). If empty, all queries are taken into account. Use the value "all" if you want to apply all patterns in one run.',
 			name: 'BENCHMARK_QUERIES' 
 		)
 
+		string( 
+			defaultValue: '',
+			description: 'Comma separated list of queries to exclude from the benchmark (with a single space after each comma)',
+			name: 'BENCHMARK_QUERIES_EXCLUDE' 
+		)
+
 		string( 
 			defaultValue: '300000',
 			description: 'Comma separated list of model sizes to test (with a single space after each comma).',

diff --git a/com.incquerylabs.magicdraw.benchmark/run.sh b/com.incquerylabs.magicdraw.benchmark/run.sh
@@ -48,6 +48,11 @@ BENCHMARK_QUERIES="transitiveSubstatesWithCheck3"
 fi
 echo "Selected queries: ${BENCHMARK_QUERIES}"
 
+if [ -z "$BENCHMARK_QUERIES_EXCLUDE" ]; then
+BENCHMARK_QUERIES_EXCLUDE=""
+fi
+echo "Queries to be excluded: ${BENCHMARK_QUERIES_EXCLUDE}"
+
 if [ -z "$BENCHMARK_SIZES" ]; then
 #BENCHMARK_SIZES="300000, 540000, 780000, 1040000, 1200000"
 BENCHMARK_SIZES="5000"
@@ -63,6 +68,7 @@ OUTPUT_DIR=${WORKSPACE_BENCHMARK}/com.incquerylabs.magicdraw.benchmark/results
 
 IFS=', ' read -r -a engines <<< "$BENCHMARK_ENGINES"
 IFS=', ' read -r -a queries <<< "$BENCHMARK_QUERIES"
+IFS=', ' read -r -a excluded <<< "$BENCHMARK_QUERIES_EXCLUDE"
 IFS=', ' read -r -a modelsizes <<< "$BENCHMARK_SIZES"
 
 # Run benchmark
@@ -81,13 +87,16 @@ do
 
 			for query in "${queries[@]}";
 			do
-				echo "Query: $query"
-				echo "Running measurement on $query with $engine (model size: $size ; runIndex: $runIndex )"
-
-				# Call MD
-				./gradlew -Pquery="$query" -Pmodel="${MODEL_LOCATION}/TMT$size.mdzip" -Pwarmup="${MODEL_LOCATION}/Warmup.mdzip" -Pindex="$runIndex" -Psize="$size" \
-				-Poutput="${OUTPUT_DIR}" -Pengine="$engine" runBenchmark
-
+			if [[ " ${excluded[@]} " =~ " ${query} " ]]; then
+					echo "No benchmark is needed: $query is excluded - $BENCHMARK_QUERIES_EXCLUDE"
+				else
+					echo "Query: $query"
+					echo "Running measurement on $query with $engine (model size: $size ; runIndex: $runIndex )"
+
+					# Call MD
+					./gradlew -Pquery="$query" -Pmodel="${MODEL_LOCATION}/TMT$size.mdzip" -Pwarmup="${MODEL_LOCATION}/Warmup.mdzip" -Pindex="$runIndex" -Psize="$size" \
+					-Poutput="${OUTPUT_DIR}" -Pengine="$engine" -Pexclude="$BENCHMARK_QUERIES_EXCLUDE" runBenchmark
+				fi
 			done
 		done
 	done

diff --git a/...icdraw.benchmark/src/main/java/com/incquerylabs/magicdraw/benchmark/BackendSelection.java b/...icdraw.benchmark/src/main/java/com/incquerylabs/magicdraw/benchmark/BackendSelection.java
@@ -1,6 +1,8 @@
 package com.incquerylabs.magicdraw.benchmark;
 
 import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.eclipse.collections.impl.factory.Lists;
 import org.eclipse.viatra.query.runtime.api.IQueryGroup;
@@ -27,6 +29,8 @@ public enum BackendSelection {
 	LOCAL_SEARCH_HINTS_TC_FIRST,
 	HYBRID;
 
+	private static final String ALL = "all";
+
 	public Iterable<IQuerySpecification<?>> findQueries(BenchmarkParameters parameters) throws ViatraQueryException {
 		switch(this) {
 		case RETE:
@@ -112,9 +116,16 @@ public boolean canHandleParameters(BenchmarkParameters parameters) throws Viatra
 	}
 
 	private Iterable<IQuerySpecification<?>> findQueries(IQueryGroup querySpecifications, String queryName, BenchmarkParameters parameters) throws ViatraQueryException {
+		if(queryName.equals(ALL)) {
+			return findAllQueries(querySpecifications).stream().filter(x -> !parameters.isExcluded(getName(x))).collect(Collectors.toSet());
+		}
 		return Lists.immutable.of(findQueryBySimpleName(querySpecifications, queryName));
 	}
 
+	private Set<IQuerySpecification<?>> findAllQueries(IQueryGroup querySpecifications) throws ViatraQueryException {
+		return querySpecifications.getSpecifications();
+	}
+
 	private IQuerySpecification<?> findQueryBySimpleName(IQueryGroup querySpecifications, String queryName) throws ViatraQueryException {
 		return querySpecifications.getSpecifications().stream().filter(spec -> Objects.deepEquals(getName(spec), queryName)).findAny()
 				.orElseThrow(() -> new InvalidBenchmarkParameterizationException("Query " + queryName + " not found "));

diff --git a/...raw.benchmark/src/main/java/com/incquerylabs/magicdraw/benchmark/BenchmarkParameters.java b/...raw.benchmark/src/main/java/com/incquerylabs/magicdraw/benchmark/BenchmarkParameters.java
@@ -130,6 +130,9 @@ public void setPassword(String password) {
 		}
 
 		public void setExcludedQueries(String excludedQueries) {
+			if(excludedQueries.isEmpty()) 
+				return;
+
 			String[] array = excludedQueries.split(", ");
 			for (int i = 0; i < array.length; i++) {
 				this.excludedQueries.add(array[i]);

diff --git a/...chmark/src/main/java/com/incquerylabs/magicdraw/benchmark/PerformanceBenchmarkRunner.java b/...chmark/src/main/java/com/incquerylabs/magicdraw/benchmark/PerformanceBenchmarkRunner.java
@@ -101,7 +101,10 @@ private BenchmarkParameters parseParameters(String[] args) {
 			} else if (Objects.equals("-password", arguments[argIndex])) {
 				parameters.setPassword(arguments[argIndex + 1]);
 				argIndex+=2;
-			} else {
+			} else if (Objects.equals("-exclude", arguments[argIndex])) {
+				parameters.setExcludedQueries(arguments[argIndex + 1]);
+				argIndex+=2;
+			}else {
 				System.err.println("Unexpected parameter " + arguments[argIndex]);
 				argIndex++; // Skip unknown parameter
 			}