From 94e2d3bd5ac0ca9a3ef7993b3129c5f19f3aae3a Mon Sep 17 00:00:00 2001
From: giacomomagni <giac.magni@gmail.com>
Date: Wed, 22 Mar 2023 09:59:57 +0100
Subject: [PATCH 01/15] init lha simple workflow

---
 .github/workflows/lha_bot.yml | 44 ++++++++++++++++++++++++++++++
 benchmarks/lha_paper_bench.py | 51 ++++++++++++++++++++++++++---------
 pyproject.toml                |  1 +
 3 files changed, 83 insertions(+), 13 deletions(-)
 create mode 100644 .github/workflows/lha_bot.yml

diff --git a/.github/workflows/lha_bot.yml b/.github/workflows/lha_bot.yml
new file mode 100644
index 000000000..e075ec8fe
--- /dev/null
+++ b/.github/workflows/lha_bot.yml
@@ -0,0 +1,44 @@
+# A single CI script with github workflow.
+name: LHA Benchmarks
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types:
+      - closed
+      - ready_for_review
+      - review_requested
+  pull_request_review:
+    types:
+      - submitted
+
+jobs:
+  lhabench:
+    name: LHA paper Benchmarks
+    runs-on: ubuntu-latest
+    container:
+      image: docker://ghcr.io/nnpdf/bench-evol:latest
+      credentials:
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # tags needed for dynamic versioning
+          fetch-depth: 0
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+          installer-parallel: true
+      - name: Install project
+        run: |
+          poetry install --no-interaction --with test -E mark -E box
+      - name: Install task runner
+        run: pip install poethepoet
+      - name: Run benchmark
+        run: |
+          poe lha_sv
diff --git a/benchmarks/lha_paper_bench.py b/benchmarks/lha_paper_bench.py
index 316acffdd..07a62281c 100644
--- a/benchmarks/lha_paper_bench.py
+++ b/benchmarks/lha_paper_bench.py
@@ -1,11 +1,11 @@
 """
 Benchmark to :cite:`Giele:2002hx` (LO + NLO) and :cite:`Dittmar:2005ed` (NNLO).
 """
+import argparse
 from math import nan
 
 import numpy as np
 from banana import register
-from banana.data import cartesian_product
 
 from eko.interpolation import lambertgrid
 from ekomark.benchmark.runner import Runner
@@ -248,15 +248,40 @@ def run_lha(self, theory_updates):
 
 
 if __name__ == "__main__":
-    # Benchmark to LHA
-    obj = BenchmarkFFNS_polarized()
-    # obj = BenchmarkFFNS()
-    # obj.benchmark_plain(1)
-    obj.benchmark_sv(1)
-
-    # # VFNS benchmarks with LHA settings
-    # programs = ["LHA", "pegasus", "apfel"]
-    # for p in programs:
-    #     obj = BenchmarkRunner(p)
-    #     # obj.benchmark_plain(2)
-    #     obj.benchmark_sv(2)
+    parser = argparse.ArgumentParser(
+        prog="LHA_benchmark",
+        description="EKO benchmark with LHA settings",
+    )
+    parser.add_argument(
+        "-o", "--pto", help="QCD perturbative order 0=LO, ... ", type=int, default=2
+    )
+    parser.add_argument(
+        "-e",
+        "--external",
+        help="External program to benchmark against (only for VFNS)",
+        default="LHA",
+    )
+    parser.add_argument(
+        "-s",
+        "--use_sv",
+        help="Run Scale Variations (only for VFNS)",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-f", "--use_ffns", help="Run FFNS evolution", action="store_true"
+    )
+    parser.add_argument(
+        "-p", "--use_pol", help="Run Polarized evolution", action="store_true"
+    )
+    args = parser.parse_args()
+
+    if args.use_ffns:
+        obj = BenchmarkFFNS()
+    elif args.use_pol:
+        obj = BenchmarkFFNS_polarized()
+    else:
+        obj = BenchmarkRunner(args.external)
+    if args.use_sv:
+        obj.benchmark_sv(args.pto)
+    else:
+        obj.benchmark_plain(args.pto)
diff --git a/pyproject.toml b/pyproject.toml
index a385d9f79..d70e39ce5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -116,6 +116,7 @@ lint = "pylint src/**/*.py -E"
 lint-warnings = "pylint src/**/*.py --exit-zero"
 sandbox = "python benchmarks/sandbox.py"
 lha = "python benchmarks/lha_paper_bench.py"
+lha_bot = "python benchmarks/lha_paper_bench.py -s -o 2"
 nav = "ekonav --config benchmarks/banana.yaml"
 navigator = "ekonav --config benchmarks/banana.yaml"
 docs = { "shell" = "cd doc; make html" }

From dc3dd82b8167e2cbfd88e423998b751898da2681 Mon Sep 17 00:00:00 2001
From: giacomomagni <giac.magni@gmail.com>
Date: Wed, 22 Mar 2023 10:04:08 +0100
Subject: [PATCH 02/15] attempt fix container

---
 .github/workflows/lha_bot.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/lha_bot.yml b/.github/workflows/lha_bot.yml
index e075ec8fe..1dbd87c97 100644
--- a/.github/workflows/lha_bot.yml
+++ b/.github/workflows/lha_bot.yml
@@ -19,7 +19,7 @@ jobs:
     name: LHA paper Benchmarks
     runs-on: ubuntu-latest
     container:
-      image: docker://ghcr.io/nnpdf/bench-evol:latest
+      image: ghcr.io/n3pdf/bench-evol:v2
       credentials:
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}

From c1e28b42150475dd09b32949fad5bb1a321d7c79 Mon Sep 17 00:00:00 2001
From: Alessandro Candido <candido.ale@gmail.com>
Date: Wed, 22 Mar 2023 10:27:13 +0100
Subject: [PATCH 03/15] Fix a couple of names

---
 .github/workflows/lha_bot.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/lha_bot.yml b/.github/workflows/lha_bot.yml
index 1dbd87c97..5235c613d 100644
--- a/.github/workflows/lha_bot.yml
+++ b/.github/workflows/lha_bot.yml
@@ -19,7 +19,7 @@ jobs:
     name: LHA paper Benchmarks
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/n3pdf/bench-evol:v2
+      image: ghcr.io/nnpdf/bench-evol:v2
       credentials:
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
@@ -41,4 +41,4 @@ jobs:
         run: pip install poethepoet
       - name: Run benchmark
         run: |
-          poe lha_sv
+          poe lha_bot

From 1dc2fdeabda9c102805aeac79db47b8ed0323912 Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Wed, 22 Mar 2023 11:12:49 +0100
Subject: [PATCH 04/15] Run below pytest

---
 benchmarks/lha_paper_bench.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/benchmarks/lha_paper_bench.py b/benchmarks/lha_paper_bench.py
index 07a62281c..81ae8e24c 100644
--- a/benchmarks/lha_paper_bench.py
+++ b/benchmarks/lha_paper_bench.py
@@ -5,7 +5,9 @@
 from math import nan
 
 import numpy as np
+import pytest
 from banana import register
+from banana.data import cartesian_product
 
 from eko.interpolation import lambertgrid
 from ekomark.benchmark.runner import Runner
@@ -38,7 +40,7 @@
 # ffns_skip_pdfs.extend([-5, 5, "T24"])
 
 
-class LHABenchmark(Runner):
+class LHA(Runner):
     """Globally set the external program to LHA."""
 
     def __init__(self):
@@ -125,16 +127,16 @@ def run_lha(self, theory_updates):
             ["ToyLH"],
         )
 
-    def benchmark_plain(self, pto):
+    def run_plain(self, pto):
         """Run plain configuration."""
         self.run_lha(self.plain_theory(pto))
 
-    def benchmark_sv(self, pto):
+    def run_sv(self, pto):
         """Run scale variations."""
         self.run_lha(self.sv_theories(pto))
 
 
-class BenchmarkVFNS(LHABenchmark):
+class VFNS(LHA):
     """Provide |VFNS| settings."""
 
     def __init__(self):
@@ -152,7 +154,7 @@ def __init__(self):
         )
 
 
-class BenchmarkFFNS(LHABenchmark):
+class FFNS(LHA):
     """Provide |FFNS| settings."""
 
     def __init__(self):
@@ -192,7 +194,14 @@ def skip_pdfs(theory):
         return ffns_skip_pdfs
 
 
-class BenchmarkRunner(BenchmarkVFNS):
+@pytest.mark.ffns
+class BenchmarkFFNS:
+    @pytest.mark.lo
+    def benchmark_plain_lo(self):
+        FFNS().run_plain(0)
+
+
+class CommonRunner(VFNS):
     """Generic benchmark runner using the LHA |VFNS| settings."""
 
     def __init__(self, external):
@@ -224,7 +233,7 @@ def benchmark_sv(self, pto):
         self.run_lha([low, high])
 
 
-class BenchmarkFFNS_polarized(BenchmarkFFNS):
+class FFNS_polarized(FFNS):
     def run_lha(self, theory_updates):
         """Enforce operator grid and PDF.
 

From 544c7a8a72ef31bcec9956e6d0ae9f21c4179c28 Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Wed, 22 Mar 2023 11:18:54 +0100
Subject: [PATCH 05/15] Fix hickups

---
 .github/workflows/lha_bot.yml | 2 +-
 benchmarks/lha_paper_bench.py | 6 +++---
 pyproject.toml                | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/lha_bot.yml b/.github/workflows/lha_bot.yml
index 5235c613d..34c0824d8 100644
--- a/.github/workflows/lha_bot.yml
+++ b/.github/workflows/lha_bot.yml
@@ -41,4 +41,4 @@ jobs:
         run: pip install poethepoet
       - name: Run benchmark
         run: |
-          poe lha_bot
+          NUMBA_DISABLE_JIT=0 poe lha_bot
diff --git a/benchmarks/lha_paper_bench.py b/benchmarks/lha_paper_bench.py
index 81ae8e24c..da2f7201b 100644
--- a/benchmarks/lha_paper_bench.py
+++ b/benchmarks/lha_paper_bench.py
@@ -285,11 +285,11 @@ def run_lha(self, theory_updates):
     args = parser.parse_args()
 
     if args.use_ffns:
-        obj = BenchmarkFFNS()
+        obj = FFNS()
     elif args.use_pol:
-        obj = BenchmarkFFNS_polarized()
+        obj = FFNS_polarized()
     else:
-        obj = BenchmarkRunner(args.external)
+        obj = CommonRunner(args.external)
     if args.use_sv:
         obj.benchmark_sv(args.pto)
     else:
diff --git a/pyproject.toml b/pyproject.toml
index d70e39ce5..02a40c0d0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -116,7 +116,7 @@ lint = "pylint src/**/*.py -E"
 lint-warnings = "pylint src/**/*.py --exit-zero"
 sandbox = "python benchmarks/sandbox.py"
 lha = "python benchmarks/lha_paper_bench.py"
-lha_bot = "python benchmarks/lha_paper_bench.py -s -o 2"
+lha_bot = "pytest benchmarks/lha_paper_bench.py -s"
 nav = "ekonav --config benchmarks/banana.yaml"
 navigator = "ekonav --config benchmarks/banana.yaml"
 docs = { "shell" = "cd doc; make html" }
@@ -146,7 +146,7 @@ addopts = [
   '--strict-markers',
 ]
 env = ["D:NUMBA_DISABLE_JIT=1"]
-markers = ["isolated: marks benchmarks as isolated"]
+markers = ["isolated: marks benchmarks as isolated", "ffns: Fixed flavor configuration", "lo: leading order"]
 
 [tool.pylint.master]
 # extensions not to check

From d456cf6a03b4a11c805ae2c70ebf6d45c420d4cc Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Wed, 22 Mar 2023 11:36:46 +0100
Subject: [PATCH 06/15] Restore OO pattern

---
 benchmarks/lha_paper_bench.py | 30 ++++++++++++++++++++++++++----
 pyproject.toml                |  9 ++++++++-
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/benchmarks/lha_paper_bench.py b/benchmarks/lha_paper_bench.py
index da2f7201b..0d380e6d7 100644
--- a/benchmarks/lha_paper_bench.py
+++ b/benchmarks/lha_paper_bench.py
@@ -136,6 +136,23 @@ def run_sv(self, pto):
         self.run_lha(self.sv_theories(pto))
 
 
+class BaseBenchmark:
+    def runner(self):
+        raise NotImplementedError("runner method has to be overwritten!")
+
+    @pytest.mark.lo
+    def benchmark_plain_lo(self):
+        self.runner().run_plain(0)
+
+    @pytest.mark.nlo
+    def benchmark_plain_nlo(self):
+        self.runner().run_plain(1)
+
+    @pytest.mark.nnlo
+    def benchmark_plain_nnlo(self):
+        self.runner().run_plain(2)
+
+
 class VFNS(LHA):
     """Provide |VFNS| settings."""
 
@@ -154,6 +171,12 @@ def __init__(self):
         )
 
 
+@pytest.mark.vfns
+class BenchmarkVFNS(BaseBenchmark):
+    def runner(self):
+        return VFNS()
+
+
 class FFNS(LHA):
     """Provide |FFNS| settings."""
 
@@ -195,10 +218,9 @@ def skip_pdfs(theory):
 
 
 @pytest.mark.ffns
-class BenchmarkFFNS:
-    @pytest.mark.lo
-    def benchmark_plain_lo(self):
-        FFNS().run_plain(0)
+class BenchmarkFFNS(BaseBenchmark):
+    def runner(self):
+        return FFNS()
 
 
 class CommonRunner(VFNS):
diff --git a/pyproject.toml b/pyproject.toml
index 02a40c0d0..54daf0330 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -146,7 +146,14 @@ addopts = [
   '--strict-markers',
 ]
 env = ["D:NUMBA_DISABLE_JIT=1"]
-markers = ["isolated: marks benchmarks as isolated", "ffns: Fixed flavor configuration", "lo: leading order"]
+markers = [
+  "isolated: marks benchmarks as isolated",
+  "ffns: Fixed flavor configuration",
+  "vfns: Variable flavor configuration",
+  "lo: Leading order",
+  "nlo: Next-to-leading order",
+  "nnlo: Next-to-next-to-leading order",
+]
 
 [tool.pylint.master]
 # extensions not to check

From 7825487da3fdf51ab76f86eff599ca7e31849083 Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Wed, 22 Mar 2023 11:48:35 +0100
Subject: [PATCH 07/15] Complete benchmarks

---
 benchmarks/lha_paper_bench.py | 65 ++++++++++++++++++++++-------------
 pyproject.toml                |  3 ++
 2 files changed, 45 insertions(+), 23 deletions(-)

diff --git a/benchmarks/lha_paper_bench.py b/benchmarks/lha_paper_bench.py
index 0d380e6d7..1afd05dfc 100644
--- a/benchmarks/lha_paper_bench.py
+++ b/benchmarks/lha_paper_bench.py
@@ -141,17 +141,30 @@ def runner(self):
         raise NotImplementedError("runner method has to be overwritten!")
 
     @pytest.mark.lo
+    @pytest.mark.nosv
     def benchmark_plain_lo(self):
         self.runner().run_plain(0)
 
     @pytest.mark.nlo
+    @pytest.mark.nosv
     def benchmark_plain_nlo(self):
         self.runner().run_plain(1)
 
     @pytest.mark.nnlo
+    @pytest.mark.nosv
     def benchmark_plain_nnlo(self):
         self.runner().run_plain(2)
 
+    @pytest.mark.nlo
+    @pytest.mark.sv
+    def benchmark_sv_nlo(self):
+        self.runner().run_sv(1)
+
+    @pytest.mark.nnlo
+    @pytest.mark.sv
+    def benchmark_sv_nnlo(self):
+        self.runner().run_sv(2)
+
 
 class VFNS(LHA):
     """Provide |VFNS| settings."""
@@ -223,6 +236,35 @@ def runner(self):
         return FFNS()
 
 
+class FFNS_polarized(FFNS):
+    def run_lha(self, theory_updates):
+        """Enforce operator grid and PDF.
+
+        Parameters
+        ----------
+        theory_updates : list(dict)
+            theory updates
+        """
+        self.run(
+            theory_updates,
+            [
+                {
+                    "Q2grid": [1e4],
+                    "ev_op_iterations": 10,
+                    "interpolation_xgrid": lambertgrid(60).tolist(),
+                    "polarized": True,
+                }
+            ],
+            ["ToyLH_polarized"],
+        )
+
+
+@pytest.mark.ffns_pol
+class BenchmarkFFNS_polarized(BaseBenchmark):
+    def runner(self):
+        return FFNS_polarized()
+
+
 class CommonRunner(VFNS):
     """Generic benchmark runner using the LHA |VFNS| settings."""
 
@@ -255,29 +297,6 @@ def benchmark_sv(self, pto):
         self.run_lha([low, high])
 
 
-class FFNS_polarized(FFNS):
-    def run_lha(self, theory_updates):
-        """Enforce operator grid and PDF.
-
-        Parameters
-        ----------
-        theory_updates : list(dict)
-            theory updates
-        """
-        self.run(
-            theory_updates,
-            [
-                {
-                    "Q2grid": [1e4],
-                    "ev_op_iterations": 10,
-                    "interpolation_xgrid": lambertgrid(60).tolist(),
-                    "polarized": True,
-                }
-            ],
-            ["ToyLH_polarized"],
-        )
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         prog="LHA_benchmark",
diff --git a/pyproject.toml b/pyproject.toml
index 54daf0330..83a5c4049 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -149,10 +149,13 @@ env = ["D:NUMBA_DISABLE_JIT=1"]
 markers = [
   "isolated: marks benchmarks as isolated",
   "ffns: Fixed flavor configuration",
+  "ffns_pol: Polarized fixed flavor configuration",
   "vfns: Variable flavor configuration",
   "lo: Leading order",
   "nlo: Next-to-leading order",
   "nnlo: Next-to-next-to-leading order",
+  "nosv: Without scale variations",
+  "sv: Scale variations",
 ]
 
 [tool.pylint.master]

From 190d0d2428a3cce63e56c732123317a25698d6fa Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Wed, 22 Mar 2023 11:52:24 +0100
Subject: [PATCH 08/15] Drop nosv

---
 benchmarks/lha_paper_bench.py | 3 ---
 pyproject.toml                | 1 -
 2 files changed, 4 deletions(-)

diff --git a/benchmarks/lha_paper_bench.py b/benchmarks/lha_paper_bench.py
index 1afd05dfc..e6c9ef1e5 100644
--- a/benchmarks/lha_paper_bench.py
+++ b/benchmarks/lha_paper_bench.py
@@ -141,17 +141,14 @@ def runner(self):
         raise NotImplementedError("runner method has to be overwritten!")
 
     @pytest.mark.lo
-    @pytest.mark.nosv
     def benchmark_plain_lo(self):
         self.runner().run_plain(0)
 
     @pytest.mark.nlo
-    @pytest.mark.nosv
     def benchmark_plain_nlo(self):
         self.runner().run_plain(1)
 
     @pytest.mark.nnlo
-    @pytest.mark.nosv
     def benchmark_plain_nnlo(self):
         self.runner().run_plain(2)
 
diff --git a/pyproject.toml b/pyproject.toml
index 83a5c4049..b91280f95 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -154,7 +154,6 @@ markers = [
   "lo: Leading order",
   "nlo: Next-to-leading order",
   "nnlo: Next-to-next-to-leading order",
-  "nosv: Without scale variations",
   "sv: Scale variations",
 ]
 

From fec9e408ac47af8ad4de3623879484566a56d002 Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Wed, 22 Mar 2023 12:10:57 +0100
Subject: [PATCH 09/15] Rename back to poe lha, drop main

---
 .github/workflows/lha_bot.yml |  2 +-
 benchmarks/lha_paper_bench.py | 40 -----------------------------------
 pyproject.toml                |  4 ++--
 3 files changed, 3 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/lha_bot.yml b/.github/workflows/lha_bot.yml
index 34c0824d8..df273a578 100644
--- a/.github/workflows/lha_bot.yml
+++ b/.github/workflows/lha_bot.yml
@@ -41,4 +41,4 @@ jobs:
         run: pip install poethepoet
       - name: Run benchmark
         run: |
-          NUMBA_DISABLE_JIT=0 poe lha_bot
+          poe lha
diff --git a/benchmarks/lha_paper_bench.py b/benchmarks/lha_paper_bench.py
index e6c9ef1e5..056692b75 100644
--- a/benchmarks/lha_paper_bench.py
+++ b/benchmarks/lha_paper_bench.py
@@ -292,43 +292,3 @@ def benchmark_sv(self, pto):
         high["XIR"] = np.sqrt(0.5)
 
         self.run_lha([low, high])
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        prog="LHA_benchmark",
-        description="EKO benchmark with LHA settings",
-    )
-    parser.add_argument(
-        "-o", "--pto", help="QCD perturbative order 0=LO, ... ", type=int, default=2
-    )
-    parser.add_argument(
-        "-e",
-        "--external",
-        help="External program to benchmark against (only for VFNS)",
-        default="LHA",
-    )
-    parser.add_argument(
-        "-s",
-        "--use_sv",
-        help="Run Scale Variations (only for VFNS)",
-        action="store_true",
-    )
-    parser.add_argument(
-        "-f", "--use_ffns", help="Run FFNS evolution", action="store_true"
-    )
-    parser.add_argument(
-        "-p", "--use_pol", help="Run Polarized evolution", action="store_true"
-    )
-    args = parser.parse_args()
-
-    if args.use_ffns:
-        obj = FFNS()
-    elif args.use_pol:
-        obj = FFNS_polarized()
-    else:
-        obj = CommonRunner(args.external)
-    if args.use_sv:
-        obj.benchmark_sv(args.pto)
-    else:
-        obj.benchmark_plain(args.pto)
diff --git a/pyproject.toml b/pyproject.toml
index b91280f95..043e32d4b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -115,8 +115,8 @@ bench-run.env.NUMBA_DISABLE_JIT.default = "0"
 lint = "pylint src/**/*.py -E"
 lint-warnings = "pylint src/**/*.py --exit-zero"
 sandbox = "python benchmarks/sandbox.py"
-lha = "python benchmarks/lha_paper_bench.py"
-lha_bot = "pytest benchmarks/lha_paper_bench.py -s"
+lha.cmd = "pytest benchmarks/lha_paper_bench.py -s"
+lha.env.NUMBA_DISABLE_JIT.default = "0"
 nav = "ekonav --config benchmarks/banana.yaml"
 navigator = "ekonav --config benchmarks/banana.yaml"
 docs = { "shell" = "cd doc; make html" }

From 5b4a546f804a9ea863eae25217c1056ee85005c0 Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Wed, 22 Mar 2023 13:44:24 +0100
Subject: [PATCH 10/15] Skip FFNS pol NNLO

---
 benchmarks/lha_paper_bench.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/benchmarks/lha_paper_bench.py b/benchmarks/lha_paper_bench.py
index 056692b75..18b352951 100644
--- a/benchmarks/lha_paper_bench.py
+++ b/benchmarks/lha_paper_bench.py
@@ -261,6 +261,15 @@ class BenchmarkFFNS_polarized(BaseBenchmark):
     def runner(self):
         return FFNS_polarized()
 
+    @pytest.mark.nnlo
+    def benchmark_plain_nnlo(self):
+        pass
+
+    @pytest.mark.nnlo
+    @pytest.mark.sv
+    def benchmark_sv_nnlo(self):
+        pass
+
 
 class CommonRunner(VFNS):
     """Generic benchmark runner using the LHA |VFNS| settings."""

From bcca8040a12b35f8e688dc785e86cbea32de4f07 Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Wed, 22 Mar 2023 14:08:43 +0100
Subject: [PATCH 11/15] Allow to silence logs in benchmark

---
 benchmarks/lha_paper_bench.py   | 22 ++++++++++++++++------
 src/ekomark/benchmark/runner.py | 14 ++++++++------
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/benchmarks/lha_paper_bench.py b/benchmarks/lha_paper_bench.py
index 18b352951..4009f8fc3 100644
--- a/benchmarks/lha_paper_bench.py
+++ b/benchmarks/lha_paper_bench.py
@@ -2,6 +2,7 @@
 Benchmark to :cite:`Giele:2002hx` (LO + NLO) and :cite:`Dittmar:2005ed` (NNLO).
 """
 import argparse
+import os
 from math import nan
 
 import numpy as np
@@ -137,30 +138,39 @@ def run_sv(self, pto):
 
 
 class BaseBenchmark:
-    def runner(self):
+    """Abstract common benchmark tasks."""
+
+    def runner(self) -> LHA:
+        """Runner to run."""
         raise NotImplementedError("runner method has to be overwritten!")
 
+    def transformed_runner(self):
+        """Prepare runner for benchmark setup"""
+        r = self.runner()
+        r.log_to_stdout = os.environ.get("EKO_LOG_STDOUT", False)
+        return r
+
     @pytest.mark.lo
     def benchmark_plain_lo(self):
-        self.runner().run_plain(0)
+        self.transformed_runner().run_plain(0)
 
     @pytest.mark.nlo
     def benchmark_plain_nlo(self):
-        self.runner().run_plain(1)
+        self.transformed_runner().run_plain(1)
 
     @pytest.mark.nnlo
     def benchmark_plain_nnlo(self):
-        self.runner().run_plain(2)
+        self.transformed_runner().run_plain(2)
 
     @pytest.mark.nlo
     @pytest.mark.sv
     def benchmark_sv_nlo(self):
-        self.runner().run_sv(1)
+        self.transformed_runner().run_sv(1)
 
     @pytest.mark.nnlo
     @pytest.mark.sv
     def benchmark_sv_nnlo(self):
-        self.runner().run_sv(2)
+        self.transformed_runner().run_sv(2)
 
 
 class VFNS(LHA):
diff --git a/src/ekomark/benchmark/runner.py b/src/ekomark/benchmark/runner.py
index 8292f8460..5a59f80cf 100644
--- a/src/ekomark/benchmark/runner.py
+++ b/src/ekomark/benchmark/runner.py
@@ -27,6 +27,7 @@ class Runner(BenchmarkRunner):
     rotate_to_evolution_basis = False
     sandbox = False
     plot_operator = False
+    log_to_stdout = True
 
     def __init__(self):
         self.banana_cfg = banana_cfg.cfg
@@ -74,12 +75,13 @@ def run_me(self, theory, ocard, _pdf):
                 DGLAP result
         """
         # activate logging
-        logStdout = logging.StreamHandler(sys.stdout)
-        logStdout.setLevel(logging.INFO)
-        logStdout.setFormatter(logging.Formatter("%(message)s"))
-        logging.getLogger("eko").handlers = []
-        logging.getLogger("eko").addHandler(logStdout)
-        logging.getLogger("eko").setLevel(logging.INFO)
+        if self.log_to_stdout:
+            logStdout = logging.StreamHandler(sys.stdout)
+            logStdout.setLevel(logging.INFO)
+            logStdout.setFormatter(logging.Formatter("%(message)s"))
+            logging.getLogger("eko").handlers = []
+            logging.getLogger("eko").addHandler(logStdout)
+            logging.getLogger("eko").setLevel(logging.INFO)
 
         ops_id = f"o{ocard['hash'][:6]}_t{theory['hash'][:6]}"
         root = banana_cfg.cfg["paths"]["database"].parents[0]

From 2c041d2a65947138294e9ab34fe1ecfde674836c Mon Sep 17 00:00:00 2001
From: Alessandro Candido <candido.ale@gmail.com>
Date: Fri, 31 Mar 2023 18:26:57 +0200
Subject: [PATCH 12/15] Change LHA workflow triggering events: swap submitted
 review with workflow dispatch

---
 .github/workflows/lha_bot.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/lha_bot.yml b/.github/workflows/lha_bot.yml
index df273a578..cd0422f57 100644
--- a/.github/workflows/lha_bot.yml
+++ b/.github/workflows/lha_bot.yml
@@ -10,9 +10,7 @@ on:
       - closed
       - ready_for_review
       - review_requested
-  pull_request_review:
-    types:
-      - submitted
+  workflow_dispatch:
 
 jobs:
   lhabench:

From 405be2561d952f166d2d5db59e3795ee73ad9a25 Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Fri, 31 Mar 2023 18:30:16 +0200
Subject: [PATCH 13/15] Run lha_bot on tags instead of commit on master

---
 .github/workflows/lha_bot.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/lha_bot.yml b/.github/workflows/lha_bot.yml
index cd0422f57..78e1d5b5b 100644
--- a/.github/workflows/lha_bot.yml
+++ b/.github/workflows/lha_bot.yml
@@ -3,8 +3,7 @@ name: LHA Benchmarks
 
 on:
   push:
-    branches:
-      - master
+    tags:
   pull_request:
     types:
       - closed

From 301aa4c0df640d3fdd48b83e954ee8346592f019 Mon Sep 17 00:00:00 2001
From: Felix Hekhorn <felix.hekhorn@uni-tuebingen.de>
Date: Fri, 31 Mar 2023 18:42:03 +0200
Subject: [PATCH 14/15] Restrict LHA bot

---
 .github/workflows/lha_bot.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/lha_bot.yml b/.github/workflows/lha_bot.yml
index 78e1d5b5b..d0689c4a6 100644
--- a/.github/workflows/lha_bot.yml
+++ b/.github/workflows/lha_bot.yml
@@ -38,4 +38,5 @@ jobs:
         run: pip install poethepoet
       - name: Run benchmark
         run: |
-          poe lha
+          poe lha -m "nnlo and sv"
+          poe lha -m "ffns_pol and sv"

From b5af1f003c239c9c475316eb01e3f14dcc96555a Mon Sep 17 00:00:00 2001
From: Alessandro Candido <candido.ale@gmail.com>
Date: Fri, 31 Mar 2023 18:58:07 +0200
Subject: [PATCH 15/15] Ignore all branches push

---
 .github/workflows/lha_bot.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/lha_bot.yml b/.github/workflows/lha_bot.yml
index d0689c4a6..d70c4a948 100644
--- a/.github/workflows/lha_bot.yml
+++ b/.github/workflows/lha_bot.yml
@@ -3,6 +3,8 @@ name: LHA Benchmarks
 
 on:
   push:
+    branches-ignore:
+      - "*"
     tags:
   pull_request:
     types: