v3 exp

experiments/issue1105/v3.py

@@ -0,0 +1,98 @@
+#! /usr/bin/env python3
+
+import itertools
+import os
+from pathlib import Path
+
+from lab.environments import LocalEnvironment, BaselSlurmEnvironment
+from lab.reports import Attribute, geometric_mean
+
+from downward.reports.compare import ComparativeReport
+
+import common_setup
+from common_setup import IssueConfig, IssueExperiment
+
+ARCHIVE_PATH = "ai/downward/issue1105"
+DIR = os.path.dirname(os.path.abspath(__file__))
+REPO_DIR = os.environ["DOWNWARD_REPO"]
+BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"]
+BASELINE_REV = "issue1105-base-v2"
+REVISION = "issue1105-v3"
+REVISIONS = [BASELINE_REV, REVISION]
+BUILDS = ["release"]
+CONFIG_NICKS = [
+    ('sccs-dfp-b50k', ['--search', 'astar(merge_and_shrink(shrink_strategy=shrink_bisimulation(greedy=false),merge_strategy=merge_sccs(order_of_sccs=topological,merge_selector=score_based_filtering(scoring_functions=[goal_relevance(),dfp(),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
+    ('sccs-sbmiasm-b50k', ['--search', 'astar(merge_and_shrink(shrink_strategy=shrink_bisimulation(greedy=false),merge_strategy=merge_sccs(order_of_sccs=topological,merge_selector=score_based_filtering(scoring_functions=[sf_miasm(use_caching=true,shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']),
+]
+CONFIGS = [
+    IssueConfig(
+        config_nick,
+        config,
+        build_options=[build],
+        driver_options=["--build", build])
+    for build in BUILDS
+    for config_nick, config in CONFIG_NICKS
+]
+
+SUITE = common_setup.DEFAULT_OPTIMAL_SUITE
+ENVIRONMENT = BaselSlurmEnvironment(
+    partition="infai_3",
+    email="[email protected]",
+    memory_per_cpu="3940M",
+    cpus_per_task=2,
+    export=["PATH"],
+    # paths obtained via:
+    # module purge
+    # module -q load Python/3.10.4-GCCcore-11.3.0
+    # module -q load GCC/11.3.0
+    # module -q load CMake/3.23.1-GCCcore-11.3.0
+    # echo $PATH
+    # echo $LD_LIBRARY_PATH
+    setup='export PATH=/scicore/soft/apps/CMake/3.23.1-GCCcore-11.3.0/bin:/scicore/soft/apps/libarchive/3.6.1-GCCcore-11.3.0/bin:/scicore/soft/apps/cURL/7.83.0-GCCcore-11.3.0/bin:/scicore/soft/apps/Python/3.10.4-GCCcore-11.3.0/bin:/scicore/soft/apps/OpenSSL/1.1/bin:/scicore/soft/apps/XZ/5.2.5-GCCcore-11.3.0/bin:/scicore/soft/apps/SQLite/3.38.3-GCCcore-11.3.0/bin:/scicore/soft/apps/Tcl/8.6.12-GCCcore-11.3.0/bin:/scicore/soft/apps/ncurses/6.3-GCCcore-11.3.0/bin:/scicore/soft/apps/bzip2/1.0.8-GCCcore-11.3.0/bin:/scicore/soft/apps/binutils/2.38-GCCcore-11.3.0/bin:/scicore/soft/apps/GCCcore/11.3.0/bin:/infai/sieverss/repos/bin:/infai/sieverss/local:/export/soft/lua_lmod/centos7/lmod/lmod/libexec:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:$PATH\nexport LD_LIBRARY_PATH=/scicore/soft/apps/libarchive/3.6.1-GCCcore-11.3.0/lib:/scicore/soft/apps/cURL/7.83.0-GCCcore-11.3.0/lib:/scicore/soft/apps/Python/3.10.4-GCCcore-11.3.0/lib:/scicore/soft/apps/OpenSSL/1.1/lib:/scicore/soft/apps/libffi/3.4.2-GCCcore-11.3.0/lib64:/scicore/soft/apps/GMP/6.2.1-GCCcore-11.3.0/lib:/scicore/soft/apps/XZ/5.2.5-GCCcore-11.3.0/lib:/scicore/soft/apps/SQLite/3.38.3-GCCcore-11.3.0/lib:/scicore/soft/apps/Tcl/8.6.12-GCCcore-11.3.0/lib:/scicore/soft/apps/libreadline/8.1.2-GCCcore-11.3.0/lib:/scicore/soft/apps/ncurses/6.3-GCCcore-11.3.0/lib:/scicore/soft/apps/bzip2/1.0.8-GCCcore-11.3.0/lib:/scicore/soft/apps/binutils/2.38-GCCcore-11.3.0/lib:/scicore/soft/apps/zlib/1.2.12-GCCcore-11.3.0/lib:/scicore/soft/apps/GCCcore/11.3.0/lib64')
+
+if common_setup.is_test_run():
+    SUITE = IssueExperiment.DEFAULT_TEST_SUITE
+    ENVIRONMENT = LocalEnvironment(processes=4)
+
+exp = IssueExperiment(
+    REPO_DIR,
+    revisions=REVISIONS,
+    configs=CONFIGS,
+    environment=ENVIRONMENT,
+)
+exp.add_suite(BENCHMARKS_DIR, SUITE)
+
+exp.add_parser(exp.EXITCODE_PARSER)
+exp.add_parser(exp.TRANSLATOR_PARSER)
+exp.add_parser(exp.SINGLE_SEARCH_PARSER)
+exp.add_parser(exp.PLANNER_PARSER)
+exp.add_parser('ms-parser.py')
+
+exp.add_step('build', exp.build)
+exp.add_step('start', exp.start_runs)
+exp.add_fetcher(name='fetch')
+
+extra_attributes = [
+    # planner outcome attributes
+    Attribute('perfect_heuristic', absolute=True, min_wins=False),
+
+    # m&s attributes
+    Attribute('ms_construction_time', absolute=False, min_wins=True, function=geometric_mean),
+    Attribute('score_ms_construction_time', min_wins=False, digits=4),
+    Attribute('ms_atomic_construction_time', absolute=False, min_wins=True, function=geometric_mean),
+    Attribute('ms_abstraction_constructed', absolute=True, min_wins=False),
+    Attribute('ms_atomic_fts_constructed', absolute=True, min_wins=False),
+    Attribute('ms_out_of_memory', absolute=True, min_wins=True),
+    Attribute('ms_out_of_time', absolute=True, min_wins=True),
+    Attribute('ms_reached_time_limit', absolute=False, min_wins=True),
+    Attribute('search_out_of_memory', absolute=True, min_wins=True),
+    Attribute('search_out_of_time', absolute=True, min_wins=True),
+]
+attributes = exp.DEFAULT_TABLE_ATTRIBUTES
+attributes.extend(extra_attributes)
+
+exp.add_comparison_table_step(attributes=attributes)
+
+exp.add_archive_step(ARCHIVE_PATH)
+
+exp.run_steps()