Fix, update and improve configuration and scripts (AliceO2Group#417)

.mega-linter.yml

@@ -17,14 +17,18 @@ DISABLE_LINTERS:
   - YAML_V8R
   - YAML_PRETTIER
   - REPOSITORY_DEVSKIM
+  - REPOSITORY_SECRETLINT
   - REPOSITORY_TRIVY
 DISABLE_ERRORS_LINTERS: # If errors are found by these linters, they will be considered as non blocking.
-  - PYTHON_BANDIT
-FILTER_REGEX_EXCLUDE: (codeQA/)
+  - PYTHON_BANDIT # The bandit check is overly broad and complains about subprocess usage.
 SHOW_ELAPSED_TIME: true
 FILEIO_REPORTER: false
 GITHUB_COMMENT_REPORTER: false
 UPDATED_SOURCES_REPORTER: false
 PRINT_ALPACA: false # Don't print ASCII alpaca in the log
 PRINT_ALL_FILES: true # Print all processed files
 FLAVOR_SUGGESTIONS: false # Don't show suggestions about different MegaLinter flavors
+PYTHON_ISORT_CONFIG_FILE: pyproject.toml
+PYTHON_PYRIGHT_CONFIG_FILE: pyproject.toml
+PYTHON_RUFF_CONFIG_FILE: pyproject.toml
+FILTER_REGEX_EXCLUDE: (codeQA/)

FirstAnalysis/homogenize_output.py

@@ -9,11 +9,11 @@
 import time
 from multiprocessing import Pool
 
-from ROOT import TFile
+from ROOT import TFile, gROOT  # pylint: disable=import-error
 
 g_verbose = False
-g_out_path = False
-g_base_dir = False
+g_out_path = ""
+g_base_dir = ""
 
 
 def split_file(input_name):
@@ -79,6 +79,8 @@ def main(input_files, verbose=True, base_dir="TF_", out_path="", jobs=20):
 
 
 if __name__ == "__main__":
+    gROOT.SetBatch(True)
+
     parser = argparse.ArgumentParser(description="Omogenizer for ML processing")
     parser.add_argument("input_files", type=str, nargs="+", help="Input files")
     parser.add_argument(
@@ -87,12 +89,8 @@ def main(input_files, verbose=True, base_dir="TF_", out_path="", jobs=20):
         default="TF_",
         help="Name of the base directory, usually `TF_` or `DF_`",
     )
-    parser.add_argument(
-        "--out_dir", "-o", type=str, default="./tmp/", help="Name of the output path"
-    )
-    parser.add_argument(
-        "--jobs", "-j", type=int, default=5, help="Number of parallel jobs"
-    )
+    parser.add_argument("--out_dir", "-o", type=str, default="./tmp/", help="Name of the output path")
+    parser.add_argument("--jobs", "-j", type=int, default=5, help="Number of parallel jobs")
     parser.add_argument("-v", action="store_true", help="Verbose mode")
     args = parser.parse_args()
     main(args.input_files, verbose=args.v, base_dir=args.base_dir, jobs=args.jobs)

codeHF/compare.py

@@ -2,62 +2,103 @@
 
 """
 Comparison script.
-Comparing different files with same structure and same plot names.
+Comparing different files with same structure and same histogram names.
 To run your comparison between AnalysisResults1.root AnalysisResults2.root you can use:
 ./compare.py AnalysisResults1.root AnalysisResults2.root -b
 """
 
 import argparse
 
-from ROOT import TH1, TCanvas, TColor, TFile, TLegend, gPad
+from ROOT import (  # pylint: disable=import-error
+    TH1,
+    TCanvas,
+    TColor,
+    TFile,
+    TLegend,
+    gPad,
+    gROOT,
+)
 
 # import itertools
 
 
-def compare(objs, add_leg_title=True, normalize=True):
+def compare(dict_obj, add_leg_title=True, normalize=True):
     print("Comparing")
-    cols = ["#e41a1c", "#377eb8", "#4daf4a"]
-    colors = {}
-    drawn = {}
-    for i in objs:
-        print("Entry", len(colors), i)
-        colors[i] = TColor.GetColor(cols[len(colors)])
+    list_colors = ["#e41a1c", "#377eb8", "#4daf4a"]
+    list_markers = [21, 20, 34]
+    dict_colors = {}
+    dict_markers = {}
+    dict_list_canvas = {}
+    for key_file in dict_obj:
+        print("Entry", len(dict_colors), key_file)
+        dict_colors[key_file] = TColor.GetColor(list_colors[len(dict_colors)])
+        dict_markers[key_file] = list_markers[len(dict_markers)]
     # Drawing objects
-    for i in objs:
-        for j in objs[i]:
-            obj = objs[i][j]
-            opt = ""
-            if drawn.setdefault(j, None) is None:
-                drawn[j] = [TCanvas(j, j)]
+    is_first_file = True
+    key_file_first = ""
+    for key_file in dict_obj:
+        if is_first_file:
+            key_file_first = key_file
+        for key_obj in dict_obj[key_file]:
+            obj = dict_obj[key_file][key_obj]
+            # FIXME
+            if "TDirectory" in obj.ClassName():
+                continue
+            opt = "LP"
+            if dict_list_canvas.setdefault(key_obj, None) is None:
+                dict_list_canvas[key_obj] = [TCanvas(key_obj, key_obj), TCanvas(f"{key_obj}_ratio", f"{key_obj}_ratio")]
             else:
                 opt += "same"
-                drawn[j][0].cd()
-            print("Drawing", obj, "with opt", opt, "on canvas", gPad.GetName())
-            obj.SetLineColor(colors[i])
+            dict_list_canvas[key_obj][0].cd()
+            print(f'Drawing {obj.GetName()} with opt "{opt}" on canvas {gPad.GetName()}')
+            obj.SetLineColor(dict_colors[key_file])
+            obj.SetMarkerStyle(dict_markers[key_file])
+            obj.SetMarkerColor(dict_colors[key_file])
             obj.SetBit(TH1.kNoTitle)
             obj.SetBit(TH1.kNoStats)
-            obj.SetTitle(i)
+            obj.SetTitle(key_file)
             if normalize:
-                drawn[j].append(obj.DrawNormalized(opt))
+                dict_list_canvas[key_obj].append(obj.DrawNormalized(opt))
             else:
-                drawn[j].append(obj.DrawClone(opt))
-    for i in drawn:
-        d = drawn[i]
-        can = d[0]
+                dict_list_canvas[key_obj].append(obj.DrawClone(opt))
+            # Ratio
+            if not is_first_file:
+                dict_list_canvas[key_obj][1].cd()
+                print(f'Drawing {obj.GetName()} with opt "{opt}" on canvas {gPad.GetName()}')
+                # line_1 = TLine(obj.GetXaxis().GetXmin(), 1, obj.GetXaxis().GetXmax(), 1)
+                obj_ratio = obj.Clone(f"{obj.GetName()}_ratio")
+                obj_ratio.Divide(dict_obj[key_file_first][key_obj])
+                dict_list_canvas[key_obj].append(obj_ratio.DrawClone(opt))
+                # dict_list_canvas[key_obj].append(line_1.Draw())
+        is_first_file = False
+    for key_obj in dict_list_canvas:
+        list_canvas = dict_list_canvas[key_obj]
+        can = list_canvas[0]
         can.cd()
-        gPad.SetLogy()
+        # gPad.SetLogy()
         leg = TLegend(0.1, 0.9, 0.9, 0.99, can.GetName())
         leg.SetNColumns(2)
-        d.append(leg)
-        for j in can.GetListOfPrimitives():
-            leg.AddEntry(j)
+        list_canvas.append(leg)
+        for prim in can.GetListOfPrimitives():
+            leg.AddEntry(prim)
         leg.Draw()
-    return drawn
+        # Ratio
+        can_ratio = list_canvas[1]
+        can_ratio.cd()
+        # gPad.SetLogy()
+        leg_ratio = TLegend(0.1, 0.9, 0.9, 0.99, can_ratio.GetName())
+        leg_ratio.SetNColumns(2)
+        list_canvas.append(leg_ratio)
+        for prim in can_ratio.GetListOfPrimitives():
+            leg_ratio.AddEntry(prim)
+        leg_ratio.Draw()
+    return dict_list_canvas
 
 
 def main(files, th1=True, th2=False, th3=False):
-    f = [TFile(i) for i in files]
-    h = {}
+    gROOT.SetBatch(True)
+    list_files = [TFile(i) for i in files]
+    dict_obj = {}
 
     def extract(directory):
         def accept_obj(entry):
@@ -69,50 +110,52 @@ def accept_obj(entry):
                 return False
             return True
 
-        o = []
-        print("Dir", directory)
-        for i in directory.GetListOfKeys():
-            obj = directory.Get(i.GetName())
+        list_names = []
+        print(f"Directory {directory.GetName()}")
+        for key in directory.GetListOfKeys():
+            obj = directory.Get(key.GetName())
             if not accept_obj(obj):
                 continue
             if "TDirectory" in obj.ClassName():
-                for j in obj.GetListOfKeys():
-                    if not accept_obj(obj.Get(j.GetName())):
+                for key_sub in obj.GetListOfKeys():
+                    if not accept_obj(obj.Get(key_sub.GetName())):
                         continue
-                    o.append(f"{directory.GetName()}/{i.GetName()}/{j.GetName()}")
+                    list_names.append(f"{directory.GetName()}/{key.GetName()}/{key_sub.GetName()}")
                 continue
-            o.append(f"{directory.GetName()}/{i.GetName()}")
-        return o
+            list_names.append(f"{directory.GetName()}/{key.GetName()}")
+        return list_names
 
-    for i in f:
-        fn = i.GetName()
-        fn = fn.replace(".root", "")
-        fn = fn.replace("AnalysisResults_O2_Run5_", "")
-        fn = fn.split("/")[-1]
-        h[fn] = {}
-        lk = i.GetListOfKeys()
-        for j in lk:
+    for file in list_files:
+        name_file = file.GetName()
+        name_file = name_file.replace(".root", "")
+        name_file = name_file.replace("AnalysisResults_O2_Run5_", "")
+        name_file = name_file.split("/")[-1]
+        dict_obj[name_file] = {}
+        list_keys = file.GetListOfKeys()
+        for key in list_keys:
             # h[fn] = list(itertools.chain(*extract(i.Get(j.GetName()))))
-            o = extract(i.Get(j.GetName()))
-            for k in o:
-                h[fn][k] = i.Get(k)
-    drawn = compare(h)
+            list_obj_names = extract(file.Get(key.GetName()))
+            for name_obj in list_obj_names:
+                dict_obj[name_file][name_obj] = file.Get(name_obj)
+    dict_list_canvas = compare(dict_obj, normalize=False)
     first = True
-    for i in drawn:
-        obj = drawn[i][0]
-        print(i)
+    for key_obj in dict_list_canvas:
+        can = dict_list_canvas[key_obj][0]
+        can_rat = dict_list_canvas[key_obj][1]
+        print(key_obj)
         if first:
-            first_obj = obj
-            obj.SaveAs("Comparison.pdf[")
-        obj.SaveAs("Comparison.pdf")
-        first = False
-    first_obj.SaveAs("Comparison.pdf]")
-    fout = TFile("Comparison.root", "RECREATE")
-    for i in drawn:
-        obj = drawn[i][0]
-        print("Writing", obj.GetName())
-        obj.Write(obj.GetName().replace("/", "_folder_"))
-    fout.Close()
+            can_first = can
+            can_first.SaveAs("Comparison.pdf[")
+            first = False
+        can.SaveAs("Comparison.pdf")
+        can_rat.SaveAs("Comparison.pdf")
+    can_first.SaveAs("Comparison.pdf]")
+    # file_out = TFile("Comparison.root", "RECREATE")
+    # for key_obj in dict_list_canvas:
+    #     can = dict_list_canvas[key_obj][0]
+    #     print("Writing", can.GetName())
+    #     can.Write(can.GetName().replace("/", "_folder_"))
+    # file_out.Close()
 
 
 if __name__ == "__main__":

codeHF/config_tasks.sh

@@ -13,6 +13,12 @@
 
 ####################################################################################################
 
+# Here you can select the AliPhysics and O2Physics branches to load.
+# BRANCH_ALI="master"
+# ENV_ALI="alienv setenv AliPhysics/latest-${BRANCH_ALI}-o2 -c"
+# BRANCH_O2="master"
+# ENV_O2="alienv setenv O2Physics/latest-${BRANCH_O2}-o2 -c"
+
 # Steps
 DOCLEAN=1           # Delete created files (before and after running tasks).
 DOCONVERT=1         # Convert AliESDs.root to AO2D.root.
@@ -89,20 +95,22 @@ DOO2_TREE_XICC=0    # hf-tree-creator-xicc-to-p-k-pi-pi
 DOO2_TREE_CHIC=0    # hf-tree-creator-chic-to-jpsi-gamma
 DOO2_TREE_BPLUS=0   # hf-tree-creator-bplus-to-d0-pi
 # Correlations
-DOO2_D0D0BAR_DATA=0       # hf-correlator-d0-d0bar
-DOO2_D0D0BAR_MCREC=0      # hf-correlator-d0-d0bar-mc-rec
-DOO2_D0D0BAR_MCGEN=0      # hf-correlator-d0-d0bar-mc-gen
-DOO2_DPLUSDMINUS_DATA=0   # hf-correlator-dplus-dminus
-DOO2_DPLUSDMINUS_MCREC=0  # hf-correlator-dplus-dminus-mc-rec
-DOO2_DPLUSDMINUS_MCGEN=0  # hf-correlator-dplus-dminus-mc-gen
-DOO2_CORRELATOR_D0HADRON=0    # hf-correlator-d0-hadrons
-DOO2_TASK_D0HADRON=0          # hf-task-correlation-d0-hadrons
-DOO2_TASK_FLOW=0    # hf-task-flow
-# Other
-DOO2_MCCONV=0       # mc-converter
-DOO2_FDDCONV=0      # fdd-converter
-DOO2_COLLCONV=0     # collision-converter
-DOO2_ZDCCONV=1      # zdc-converter
+DOO2_CORR_D0D0BAR_DATA=0       # hf-correlator-d0-d0bar
+DOO2_CORR_D0D0BAR_MCREC=0      # hf-correlator-d0-d0bar-mc-rec
+DOO2_CORR_D0D0BAR_MCGEN=0      # hf-correlator-d0-d0bar-mc-gen
+DOO2_CORR_DPLUSDMINUS_DATA=0   # hf-correlator-dplus-dminus
+DOO2_CORR_DPLUSDMINUS_MCREC=0  # hf-correlator-dplus-dminus-mc-rec
+DOO2_CORR_DPLUSDMINUS_MCGEN=0  # hf-correlator-dplus-dminus-mc-gen
+DOO2_CORR_D0HADRON=0           # hf-correlator-d0-hadrons
+DOO2_CORR_DPLUSHADRON=0        # hf-correlator-dplus-hadrons
+DOO2_CORR_DSHADRON=0           # hf-correlator-ds-hadrons
+DOO2_TASK_D0HADRON=0           # hf-task-correlation-d0-hadrons
+DOO2_TASK_FLOW=0               # hf-task-flow
+# Converters
+DOO2_CONV_MC=0       # mc-converter
+DOO2_CONV_FDD=0      # fdd-converter
+DOO2_CONV_COLL=0     # collision-converter
+DOO2_CONV_ZDC=1      # zdc-converter
 
 # Selection cuts
 APPLYCUTS_D0=1      # Apply D0 selection cuts.
@@ -396,14 +404,16 @@ function MakeScriptO2 {
   [ $DOO2_TASK_BPLUS -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-task-bplus"
   # Correlations
   WF_CORR=""
-  [ $DOO2_D0D0BAR_DATA -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-d0-d0bar o2-analysis-hf-task-correlation-d-dbar"
-  [ $DOO2_D0D0BAR_MCREC -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-d0-d0bar-mc-rec o2-analysis-hf-task-correlation-d-dbar-mc-rec"
-  [ $DOO2_D0D0BAR_MCGEN -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-d0-d0bar-mc-gen o2-analysis-hf-task-correlation-d-dbar-mc-gen"
-  [ $DOO2_DPLUSDMINUS_DATA -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-dplus-dminus o2-analysis-hf-task-correlation-d-dbar"
-  [ $DOO2_DPLUSDMINUS_MCREC -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-dplus-dminus-mc-rec o2-analysis-hf-task-correlation-d-dbar-mc-rec"
-  [ $DOO2_DPLUSDMINUS_MCGEN -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-dplus-dminus-mc-gen o2-analysis-hf-task-correlation-d-dbar-mc-gen"
+  [ $DOO2_CORR_D0D0BAR_DATA -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-d0-d0bar o2-analysis-hf-task-correlation-d-dbar"
+  [ $DOO2_CORR_D0D0BAR_MCREC -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-d0-d0bar-mc-rec o2-analysis-hf-task-correlation-d-dbar-mc-rec"
+  [ $DOO2_CORR_D0D0BAR_MCGEN -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-d0-d0bar-mc-gen o2-analysis-hf-task-correlation-d-dbar-mc-gen"
+  [ $DOO2_CORR_DPLUSDMINUS_DATA -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-dplus-dminus o2-analysis-hf-task-correlation-d-dbar"
+  [ $DOO2_CORR_DPLUSDMINUS_MCREC -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-dplus-dminus-mc-rec o2-analysis-hf-task-correlation-d-dbar-mc-rec"
+  [ $DOO2_CORR_DPLUSDMINUS_MCGEN -eq 1 ] && WF_CORR="o2-analysis-hf-correlator-dplus-dminus-mc-gen o2-analysis-hf-task-correlation-d-dbar-mc-gen"
   [ "$WF_CORR" ] && WORKFLOWS+=" $WF_CORR"
-  [ $DOO2_CORRELATOR_D0HADRON -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-correlator-d0-hadrons"
+  [ $DOO2_CORR_D0HADRON -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-correlator-d0-hadrons"
+  [ $DOO2_CORR_DPLUSHADRON -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-correlator-dplus-hadrons"
+  [ $DOO2_CORR_DSHADRON -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-correlator-ds-hadrons"
   [ $DOO2_TASK_D0HADRON -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-task-correlation-d0-hadrons"
   [ $DOO2_TASK_FLOW -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-task-flow"
   # Tree creators
@@ -414,11 +424,11 @@ function MakeScriptO2 {
   [ $DOO2_TREE_XICC -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-tree-creator-xicc-to-p-k-pi-pi"
   [ $DOO2_TREE_CHIC -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-tree-creator-chic-to-jpsi-gamma"
   [ $DOO2_TREE_BPLUS -eq 1 ] && WORKFLOWS+=" o2-analysis-hf-tree-creator-bplus-to-d0-pi"
-  # Other
-  [ $DOO2_MCCONV -eq 1 ] && WORKFLOWS+=" o2-analysis-mc-converter"
-  [ $DOO2_FDDCONV -eq 1 ] && WORKFLOWS+=" o2-analysis-fdd-converter"
-  [ $DOO2_COLLCONV -eq 1 ] && WORKFLOWS+=" o2-analysis-collision-converter"
-  [ $DOO2_ZDCCONV -eq 1 ] && WORKFLOWS+=" o2-analysis-zdc-converter"
+  # Converters
+  [ $DOO2_CONV_MC -eq 1 ] && WORKFLOWS+=" o2-analysis-mc-converter"
+  [ $DOO2_CONV_FDD -eq 1 ] && WORKFLOWS+=" o2-analysis-fdd-converter"
+  [ $DOO2_CONV_COLL -eq 1 ] && WORKFLOWS+=" o2-analysis-collision-converter"
+  [ $DOO2_CONV_ZDC -eq 1 ] && WORKFLOWS+=" o2-analysis-zdc-converter"
 
   # Translate options into arguments of the generating script.
   OPT_MAKECMD=""