Prevent Thread Oversubscription with threadpoolclt

In a cluster setting, thread oversubscription can lead to significant
performance degradation and resource contention for running the deconvolution
with scipy.optimize.  This commit addresses this issue by utilizing
the `threadpoolclt` library to limit the number of threads to 1.

This change ensures that each process uses only the allocated resources,
 preventing contention and improving overall cluster stability.

lollipop/cli/deconvolute.py

@@ -14,6 +14,7 @@
 import pandas as pd
 import numpy as np
 from tqdm import tqdm, trange
+from threadpoolctl import ThreadpoolController
 
 import lollipop as ll
 
@@ -192,6 +193,10 @@ def _deconvolute_bootstrap(args: DeconvBootstrapsArgsNoSeed) -> List[pd.DataFram
     confint_name = args["confint_name"]
     namefield = args["namefield"]
 
+    # monitor the number of threads, to prevent oversubscription on blas / cluster systmes
+    controller = ThreadpoolController()
+    logging.info(f"Threading configuration:\n {controller.info()}")
+
     # deconvolution results
     deconv = []
 
@@ -277,7 +282,7 @@ def _deconvolute_bootstrap(args: DeconvBootstrapsArgsNoSeed) -> List[pd.DataFram
                 # just run one on everything
                 weights = {}
 
-            # deconvolution
+            # define deconvolution kernel
             t_kdec = ll.KernelDeconv(
                 temp_df2[var_dates["var_dates"][mindate] + ["undetermined"]],
                 temp_df2["frac"],
@@ -287,7 +292,11 @@ def _deconvolute_bootstrap(args: DeconvBootstrapsArgsNoSeed) -> List[pd.DataFram
                 confint=confint(**confint_params),
                 **weights,
             )
-            t_kdec = t_kdec.deconv_all(**deconv_params)
+            # limit the number of threads, to prevent oversubscription on blas / cluster systmes
+            with controller.limit(limits=1, user_api="blas"):
+                # do the deconvolution
+                t_kdec = t_kdec.deconv_all(**deconv_params)
+
             if have_confint:
                 # with conf int
                 res = t_kdec.fitted.copy()

pyproject.toml

@@ -23,6 +23,7 @@ strictyaml = { version = ">=1.7", optional = true }
 tqdm = { version = ">=4.64", optional = true }
 click = { version = "^8.0", optional = true }
 click-option-group = { version = "^0.5", optional = true }
+threadpoolctl = "^3.5.0"
 
 [tool.poetry.extras]
 cli = [ "zstandard", "ruamel.yaml", "strictyaml", "tqdm", "click", "click-option-group" ]