Merge pull request #673 from casparvl/map_tests_to_software_builds

Map tests to software builds

create_tarball.sh

@@ -35,6 +35,7 @@ if [ ! -d ${software_dir_overlay} ]; then
     exit 3
 fi
 
+current_workdir=${PWD}
 cd ${overlay_upper_dir}/versions/
 echo ">> Collecting list of files/directories to include in tarball via ${PWD}..."
 
@@ -88,6 +89,9 @@ echo "wrote file list to ${files_list}"
 echo "wrote module file list to ${module_files_list}"
 [ -r ${module_files_list} ] && cat ${module_files_list}
 
+# Copy the module files list to current workindg dir for later use in the test step
+cp ${module_files_list} ${current_workdir}/module_files.list.txt
+
 topdir=${cvmfs_repo}/versions/
 
 echo ">> Creating tarball ${target_tgz} from ${topdir}..."

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml

@@ -6,4 +6,4 @@ easyconfigs:
         from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc
   - gnuplot-5.4.6-GCCcore-12.2.0.eb
   - h5py-3.8.0-foss-2022b.eb
-  - MDAnalysis-2.4.2-foss-2022b.eb 
+  - MDAnalysis-2.4.2-foss-2022b.eb

test_suite.sh

@@ -195,8 +195,22 @@ else
     fatal_error "Failed to run 'reframe --version'"
 fi
 
+# Get the subset of test names based on the test mapping and tags (e.g. CI, 1_node)
+module_list="module_files.list.txt"
+mapping_config="tests/eessi_test_mapping/software_to_tests.yml"
+# Run with --debug for easier debugging in case there are issues:
+python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug
+REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}")
+test_selection_exit_code=$?
+if [[ ${test_selection_exit_code} -eq 0 ]]; then
+    echo_green "Succesfully extracted names of tests to run: ${REFRAME_NAME_ARGS}"
+else
+    fatal_error "Failed to extract names of tests to run: ${REFRAME_NAME_ARGS}"
+    exit ${test_selection_exit_code}
+fi
+export REFRAME_ARGS="--tag CI --tag 1_node --nocolor ${REFRAME_NAME_ARGS}"
+
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node --nocolor'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

tests/eessi_test_mapping/map_software_to_test.py

@@ -0,0 +1,90 @@
+import yaml
+import re
+import os
+import argparse
+
+def load_mappings(file_path):
+    """Load the YAML mappings from a file."""
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"Error: {file_path} does not exist.")
+    with open(file_path, 'r') as file:
+        config = yaml.safe_load(file)
+    return config['mappings']
+
+def read_software_names(file_path):
+    """Read software names from the module_files.list.txt file."""
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"Error: {file_path} does not exist.")
+    with open(file_path, 'r') as file:
+        software_names = [line.strip() for line in file if line.strip()]
+    return software_names
+
+def get_tests_for_software(software_name, mappings):
+    """Get the list of tests for a given software name based on the first matching regex pattern."""
+
+    # Iterate over patterns in the order they appear in the YAML file
+    for pattern, tests in mappings.items():
+        if re.match(pattern, software_name):
+            return tests
+
+    # If no matches are found, return the default tests if they exist
+    if 'default_tests' in mappings:
+        return mappings['default_tests']
+
+    return []
+
+def main(yaml_file, module_file, debug):
+    """Main function to process software names and their tests."""
+    mappings = load_mappings(yaml_file)
+    if debug:
+        print(f"Loaded mappings from '{yaml_file}'")
+
+    software_names = read_software_names(module_file)
+    if debug:
+        print(f"Read software names from '{module_file}'")
+
+    tests_to_run = []
+    arg_string = ""
+    # For each module name, get the relevant set of tests
+    for software_name in software_names:
+        additional_tests = get_tests_for_software(software_name, mappings)
+        for test in additional_tests:
+            if test not in tests_to_run:
+                tests_to_run.append(test)
+
+        if additional_tests and debug:
+            print(f"Software: {software_name} -> Tests: {additional_tests}")
+        elif debug:
+            print(f"Software: {software_name} -> No tests found")
+
+    # Always add the default set of tests, if default_tests is specified
+    if 'default_tests' in mappings:
+        additional_tests = mappings['default_tests']
+        for test in additional_tests:
+            if test not in tests_to_run:
+                tests_to_run.append(test)
+
+        if additional_tests and debug:
+            print(f"Adding default set of tests: {additional_tests}")
+
+    # Create argument string out of the list of tests to run
+    if tests_to_run:
+        arg_string = " ".join([f"-n {test_name}" for test_name in tests_to_run])
+
+    # Print final lists & argument string
+    if debug:
+        print(f"Full list of tests to run: {tests_to_run}")
+        print(f"Argument string: {arg_string}")
+    else:
+        # This is the only thing this script should print, unless run with --debug
+        print(f"{arg_string}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Map software names to their tests based on a YAML configuration.")
+    parser.add_argument('--mapping-file', type=str, help='Path to the YAML file containing the test mappings.')
+    parser.add_argument('--module-list', type=str, help='Path to the file containing the list of software names.')
+    parser.add_argument('--debug', action='store_true', help='Enable debug output.')
+
+    args = parser.parse_args()
+
+    main(args.mapping_file, args.module_list, args.debug)

tests/eessi_test_mapping/software_to_tests.yml

@@ -0,0 +1,35 @@
+# This file creates a mapping between (regular expressions for) module names and test names from the EESSI test suite
+# If a module name matches one of the regular expressions, the listed set of tests will be run in the test step
+# For a given module name, the test list for the first matching regular expression is returned
+# E.g. for
+# mappings:
+#   foo-v1:
+#     - bar
+#   foo-*
+#     - bar2
+# only the bar test will be run for foo-v1 (even though it also matches the pattern (foo-*)
+# If a module name does not match anything, the default_tests will be run
+# Note that to list all available tests by name, one can do execute
+# reframe -R -c /path/to/eessi/test-suite/ --list | grep -Po "\bEESSI_\S+?(?=[\s'])" | uniq
+# Note that this regular expression is a bit sensitive to changes in the structure of ReFrame's output,
+# but is confirmed to work for ReFrame version 4.6.1
+mappings:
+  PyTorch-Bundle/*:
+    - EESSI_PyTorch_torchvision
+  QuantumESPRESSO/*:
+    - EESSI_QuantumESPRESSO
+  CP2K/*:
+    - EESSI_CP2K
+  ESPResSo/*:
+    - EESSI_ESPRESSO
+  LAMMPS/*:
+    - EESSI_LAMMPS
+  OSU-Micro-Benchmarks/*:
+    - EESSI_OSU_Micro_Benchmarks
+  GROMACS/*:
+    - EESSI_GROMACS
+  default_tests:
+    # Low level tests
+    - EESSI_OSU_Micro_Benchmarks
+    # A very quick-to-run high level application test
+    - EESSI_LAMMPS