Simple LHA benchmark workflow

.github/workflows/lha_bot.yml

@@ -0,0 +1,44 @@
+# A single CI script with github workflow.
+name: LHA Benchmarks
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types:
+      - closed
+      - ready_for_review
+      - review_requested
+  pull_request_review:
+    types:
+      - submitted
+
+jobs:
+  lhabench:
+    name: LHA paper Benchmarks
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/nnpdf/bench-evol:v2
+      credentials:
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # tags needed for dynamic versioning
+          fetch-depth: 0
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+          installer-parallel: true
+      - name: Install project
+        run: |
+          poetry install --no-interaction --with test -E mark -E box
+      - name: Install task runner
+        run: pip install poethepoet
+      - name: Run benchmark
+        run: |
+          NUMBA_DISABLE_JIT=0 poe lha_bot

benchmarks/lha_paper_bench.py

@@ -1,9 +1,11 @@
 """
 Benchmark to :cite:`Giele:2002hx` (LO + NLO) and :cite:`Dittmar:2005ed` (NNLO).
 """
+import argparse
 from math import nan
 
 import numpy as np
+import pytest
 from banana import register
 from banana.data import cartesian_product
 
@@ -38,7 +40,7 @@
 # ffns_skip_pdfs.extend([-5, 5, "T24"])
 
 
-class LHABenchmark(Runner):
+class LHA(Runner):
     """Globally set the external program to LHA."""
 
     def __init__(self):
@@ -125,16 +127,43 @@ def run_lha(self, theory_updates):
             ["ToyLH"],
         )
 
-    def benchmark_plain(self, pto):
+    def run_plain(self, pto):
         """Run plain configuration."""
         self.run_lha(self.plain_theory(pto))
 
-    def benchmark_sv(self, pto):
+    def run_sv(self, pto):
         """Run scale variations."""
         self.run_lha(self.sv_theories(pto))
 
 
-class BenchmarkVFNS(LHABenchmark):
+class BaseBenchmark:
+    def runner(self):
+        raise NotImplementedError("runner method has to be overwritten!")
+
+    @pytest.mark.lo
+    def benchmark_plain_lo(self):
+        self.runner().run_plain(0)
+
+    @pytest.mark.nlo
+    def benchmark_plain_nlo(self):
+        self.runner().run_plain(1)
+
+    @pytest.mark.nnlo
+    def benchmark_plain_nnlo(self):
+        self.runner().run_plain(2)
+
+    @pytest.mark.nlo
+    @pytest.mark.sv
+    def benchmark_sv_nlo(self):
+        self.runner().run_sv(1)
+
+    @pytest.mark.nnlo
+    @pytest.mark.sv
+    def benchmark_sv_nnlo(self):
+        self.runner().run_sv(2)
+
+
+class VFNS(LHA):
     """Provide |VFNS| settings."""
 
     def __init__(self):
@@ -152,7 +181,13 @@ def __init__(self):
         )
 
 
-class BenchmarkFFNS(LHABenchmark):
+@pytest.mark.vfns
+class BenchmarkVFNS(BaseBenchmark):
+    def runner(self):
+        return VFNS()
+
+
+class FFNS(LHA):
     """Provide |FFNS| settings."""
 
     def __init__(self):
@@ -192,7 +227,42 @@ def skip_pdfs(theory):
         return ffns_skip_pdfs
 
 
-class BenchmarkRunner(BenchmarkVFNS):
+@pytest.mark.ffns
+class BenchmarkFFNS(BaseBenchmark):
+    def runner(self):
+        return FFNS()
+
+
+class FFNS_polarized(FFNS):
+    def run_lha(self, theory_updates):
+        """Enforce operator grid and PDF.
+
+        Parameters
+        ----------
+        theory_updates : list(dict)
+            theory updates
+        """
+        self.run(
+            theory_updates,
+            [
+                {
+                    "Q2grid": [1e4],
+                    "ev_op_iterations": 10,
+                    "interpolation_xgrid": lambertgrid(60).tolist(),
+                    "polarized": True,
+                }
+            ],
+            ["ToyLH_polarized"],
+        )
+
+
+@pytest.mark.ffns_pol
+class BenchmarkFFNS_polarized(BaseBenchmark):
+    def runner(self):
+        return FFNS_polarized()
+
+
+class CommonRunner(VFNS):
     """Generic benchmark runner using the LHA |VFNS| settings."""
 
     def __init__(self, external):
@@ -224,39 +294,41 @@ def benchmark_sv(self, pto):
         self.run_lha([low, high])
 
 
-class BenchmarkFFNS_polarized(BenchmarkFFNS):
-    def run_lha(self, theory_updates):
-        """Enforce operator grid and PDF.
-
-        Parameters
-        ----------
-        theory_updates : list(dict)
-            theory updates
-        """
-        self.run(
-            theory_updates,
-            [
-                {
-                    "Q2grid": [1e4],
-                    "ev_op_iterations": 10,
-                    "interpolation_xgrid": lambertgrid(60).tolist(),
-                    "polarized": True,
-                }
-            ],
-            ["ToyLH_polarized"],
-        )
-
-
 if __name__ == "__main__":
-    # Benchmark to LHA
-    obj = BenchmarkFFNS_polarized()
-    # obj = BenchmarkFFNS()
-    # obj.benchmark_plain(1)
-    obj.benchmark_sv(1)
-
-    # # VFNS benchmarks with LHA settings
-    # programs = ["LHA", "pegasus", "apfel"]
-    # for p in programs:
-    #     obj = BenchmarkRunner(p)
-    #     # obj.benchmark_plain(2)
-    #     obj.benchmark_sv(2)
+    parser = argparse.ArgumentParser(
+        prog="LHA_benchmark",
+        description="EKO benchmark with LHA settings",
+    )
+    parser.add_argument(
+        "-o", "--pto", help="QCD perturbative order 0=LO, ... ", type=int, default=2
+    )
+    parser.add_argument(
+        "-e",
+        "--external",
+        help="External program to benchmark against (only for VFNS)",
+        default="LHA",
+    )
+    parser.add_argument(
+        "-s",
+        "--use_sv",
+        help="Run Scale Variations (only for VFNS)",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-f", "--use_ffns", help="Run FFNS evolution", action="store_true"
+    )
+    parser.add_argument(
+        "-p", "--use_pol", help="Run Polarized evolution", action="store_true"
+    )
+    args = parser.parse_args()
+
+    if args.use_ffns:
+        obj = FFNS()
+    elif args.use_pol:
+        obj = FFNS_polarized()
+    else:
+        obj = CommonRunner(args.external)
+    if args.use_sv:
+        obj.benchmark_sv(args.pto)
+    else:
+        obj.benchmark_plain(args.pto)

pyproject.toml

@@ -116,6 +116,7 @@ lint = "pylint src/**/*.py -E"
 lint-warnings = "pylint src/**/*.py --exit-zero"
 sandbox = "python benchmarks/sandbox.py"
 lha = "python benchmarks/lha_paper_bench.py"
+lha_bot = "pytest benchmarks/lha_paper_bench.py -s"
 nav = "ekonav --config benchmarks/banana.yaml"
 navigator = "ekonav --config benchmarks/banana.yaml"
 docs = { "shell" = "cd doc; make html" }
@@ -145,7 +146,16 @@ addopts = [
   '--strict-markers',
 ]
 env = ["D:NUMBA_DISABLE_JIT=1"]
-markers = ["isolated: marks benchmarks as isolated"]
+markers = [
+  "isolated: marks benchmarks as isolated",
+  "ffns: Fixed flavor configuration",
+  "ffns_pol: Polarized fixed flavor configuration",
+  "vfns: Variable flavor configuration",
+  "lo: Leading order",
+  "nlo: Next-to-leading order",
+  "nnlo: Next-to-next-to-leading order",
+  "sv: Scale variations",
+]
 
 [tool.pylint.master]
 # extensions not to check