Merge pull request #63 from OP-TED/add-schematron-benchmark

Add a benchmark for schematron rules

README.md

@@ -3,6 +3,8 @@
 The eForms SDK Analyzer is a command-line application for the static analysis of the content of the eForms SDK.
 It loads the files from the eForms SDK, and applies various checks and verifications, to try to ensure that their content is correct and consistent.
 
+It can also run a benchmark of the Schematron rules in an eForms SDK, to help detect potential performance issues in those validation rules.
+
 ## Building
 
 Requirements:
@@ -18,7 +20,9 @@ mvn clean package
 
 ## Usage
 
-To run the application, execute the runnable JAR built as described above, indicating the path to the folder containing the eForms-SDK:
+### SDK Analysis
+
+To analyze the content of an eForms SDK, execute the runnable JAR built as described above, indicating the path to the folder containing the eForms SDK:
 
 ```shell
 java -jar target/eforms-sdk-analyzer-1.8.0-SNAPSHOT.jar path/to/eforms-sdk
@@ -28,6 +32,18 @@ This will return the exit code 0 if no errors are found, and 1 otherwise.
 
 Any error or warning found during the analysis will be logged at the corresponding level. By default, logs go to the standard output.
 
+### Schematron rules benchmark
+
+To run the benchmark of the Schematron rules in an eForms SDK, execute the runnable JAR built as described above, indicating the path to the folder containing the eForms SDK and the command `benchmark`:
+
+```shell
+java -jar target/eforms-sdk-analyzer-1.8.0-SNAPSHOT.jar path/to/eforms-sdk benchmark
+```
+
+This will execute the Schematron rules on a few large XML notices included as resources in this project, with an appropriate warm-up and number of iterations. The results are written to a JSON file.
+
+You can compare those results with the ones from another SDK version, to detect whether the rules have become slower to execute.
+
 ## Contributing
 
 ### Adding a check

pom.xml

@@ -49,11 +49,12 @@
     <version.jackson>2.14.1</version.jackson>
     <version.jaxb-bind-api>4.0.0</version.jaxb-bind-api>
     <version.jaxb-impl>4.0.4</version.jaxb-impl>
+    <jmh.version>1.37</jmh.version>
     <version.junit>5.9.2</version.junit>
     <version.logback>1.4.14</version.logback>
     <version.maven-model>3.9.1</version.maven-model>
     <version.picocli>4.6.3</version.picocli>
-    <version.ph-schematron>8.0.0</version.ph-schematron>
+    <version.ph-schematron>8.0.3</version.ph-schematron>
     <version.semver4j>3.1.0</version.semver4j>
     <version.slf4j>2.0.3</version.slf4j>
     <version.xmlschema>2.3.0</version.xmlschema>
@@ -164,6 +165,17 @@
       </dependency>
 
       <!-- Other -->
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-core</artifactId>
+        <version>${jmh.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-generator-annprocess</artifactId>
+        <version>${jmh.version}</version>
+        <scope>provided</scope>
+      </dependency>
       <dependency>
         <groupId>ch.qos.logback</groupId>
         <artifactId>logback-classic</artifactId>
@@ -322,6 +334,14 @@
     </dependency>
 
     <!-- Other -->
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-generator-annprocess</artifactId>
+    </dependency>
     <dependency>
       <groupId>ch.qos.logback</groupId>
       <artifactId>logback-classic</artifactId>
@@ -411,11 +431,16 @@
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
           <annotationProcessorPaths>
-            <annotationProcessorPath>
+            <path>
               <groupId>info.picocli</groupId>
               <artifactId>picocli-codegen</artifactId>
               <version>${version.picocli}</version>
-            </annotationProcessorPath>
+            </path>
+            <path>
+              <groupId>org.openjdk.jmh</groupId>
+              <artifactId>jmh-generator-annprocess</artifactId>
+              <version>${version.jmh}</version>
+            </path>
           </annotationProcessorPaths>
         </configuration>
       </plugin>

src/main/java/eu/europa/ted/eforms/sdk/analysis/CliCommand.java

@@ -30,9 +30,15 @@ public Integer call() throws Exception {
     return SdkAnalyzer.analyze(sdkRoot);
   }
 
+  @Command(name = "benchmark", mixinStandardHelpOptions = true, 
+      description = "Run benchmark of Schematron rules")
+  public int runBenchmark() throws Exception {
+    return SchematronBenchmark.run(sdkRoot);
+  }
+
   /**
    * {@link IVersionProvider} implementation that returns version information from the
-   * picocli-x.x.jar file's {@code /META-INF/MANIFEST.MF} file.
+   * built JAR file's {@code /META-INF/MANIFEST.MF} file.
    */
   static class ManifestVersionProvider implements IVersionProvider {
     public String[] getVersion() throws Exception {

src/main/java/eu/europa/ted/eforms/sdk/analysis/SchematronBenchmark.java

@@ -0,0 +1,95 @@
+package eu.europa.ted.eforms.sdk.analysis;
+
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.text.MessageFormat;
+
+import javax.xml.transform.Source;
+import javax.xml.transform.stream.StreamSource;
+
+import org.apache.commons.lang3.Validate;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.results.format.ResultFormatType;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.helger.commons.io.resource.FileSystemResource;
+import com.helger.commons.io.resource.IReadableResource;
+import com.helger.schematron.ISchematronResource;
+import com.helger.schematron.pure.SchematronResourcePure;
+
+@State(Scope.Benchmark)
+public class SchematronBenchmark {
+  private static final Logger logger = LoggerFactory.getLogger(SchematronBenchmark.class);
+
+  // Path to the root of the eForms SDK, overriden with the path given on the command line
+  @Param("../eforms-sdk")
+  public String sdkRootPath;
+
+  // Filenames of XML notices included as resources in this project
+  @Param({"16.xml", "29.xml"})
+  public String noticeFilename;
+
+  private ISchematronResource phSchematron;
+
+  public static int run(final Path sdkRoot) throws Exception {
+    logger.warn("Benchmarking Schematron rules from SDK under folder [{}]", sdkRoot);
+
+    Validate.notNull(sdkRoot, "Undefined SDK root path");
+    if (!Files.isDirectory(sdkRoot)) {
+      throw new FileNotFoundException(sdkRoot.toString());
+    }
+
+    Options opt = new OptionsBuilder()
+                .include(SchematronBenchmark.class.getSimpleName())
+                .param("sdkRootPath", sdkRoot.toString())
+                .resultFormat(ResultFormatType.JSON)
+                .build();
+
+    new Runner(opt).run();
+
+    return 0;
+  }
+
+  @Setup
+  public void setup() {
+    SdkLoader sdkLoader = new SdkLoader(Path.of(sdkRootPath));
+
+    // Use the first set of Schematron rules (dynamic/static)
+    // They are mostly identical, so it does not make a difference for the benchmark
+    Path file = sdkLoader.getSchematronFilesPaths().get(0);
+    IReadableResource schematron = new FileSystemResource(file);
+    phSchematron = new SchematronResourcePure(schematron);
+  }
+
+  @Benchmark
+  @Fork(1)
+  @Warmup(iterations = 1)
+  @Measurement(iterations = 3)
+  @BenchmarkMode(Mode.AverageTime)
+  public int executeValidation() throws Exception {
+
+    String resourceName = MessageFormat.format("benchmark/notices/{0}", noticeFilename);
+    InputStream stream = this.getClass().getClassLoader().getResourceAsStream(resourceName);
+    Validate.notNull(stream, "Notice file not found in resources: " + resourceName);
+
+    Source source = new StreamSource(stream);
+    phSchematron.applySchematronValidation(source);
+
+    return 0;
+  }
+}