From cd60bf68b3d3029216f09b1fe63146033191d45c Mon Sep 17 00:00:00 2001 From: Silvan Sievers Date: Mon, 29 Jan 2024 12:24:43 +0100 Subject: [PATCH] v3 exp --- experiments/issue1105/v3.py | 98 +++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100755 experiments/issue1105/v3.py diff --git a/experiments/issue1105/v3.py b/experiments/issue1105/v3.py new file mode 100755 index 0000000000..e4249a1fba --- /dev/null +++ b/experiments/issue1105/v3.py @@ -0,0 +1,98 @@ +#! /usr/bin/env python3 + +import itertools +import os +from pathlib import Path + +from lab.environments import LocalEnvironment, BaselSlurmEnvironment +from lab.reports import Attribute, geometric_mean + +from downward.reports.compare import ComparativeReport + +import common_setup +from common_setup import IssueConfig, IssueExperiment + +ARCHIVE_PATH = "ai/downward/issue1105" +DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_DIR = os.environ["DOWNWARD_REPO"] +BENCHMARKS_DIR = os.environ["DOWNWARD_BENCHMARKS"] +BASELINE_REV = "issue1105-base-v2" +REVISION = "issue1105-v3" +REVISIONS = [BASELINE_REV, REVISION] +BUILDS = ["release"] +CONFIG_NICKS = [ + ('sccs-dfp-b50k', ['--search', 'astar(merge_and_shrink(shrink_strategy=shrink_bisimulation(greedy=false),merge_strategy=merge_sccs(order_of_sccs=topological,merge_selector=score_based_filtering(scoring_functions=[goal_relevance(),dfp(),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']), + ('sccs-sbmiasm-b50k', ['--search', 'astar(merge_and_shrink(shrink_strategy=shrink_bisimulation(greedy=false),merge_strategy=merge_sccs(order_of_sccs=topological,merge_selector=score_based_filtering(scoring_functions=[sf_miasm(use_caching=true,shrink_strategy=shrink_bisimulation(greedy=false),max_states=50000,threshold_before_merge=1),total_order(atomic_ts_order=reverse_level,product_ts_order=new_to_old,atomic_before_product=false)])),label_reduction=exact(before_shrinking=true,before_merging=false),max_states=50000,threshold_before_merge=1))']), +] +CONFIGS = [ + IssueConfig( + config_nick, + config, + build_options=[build], + driver_options=["--build", build]) + for build in BUILDS + for config_nick, config in CONFIG_NICKS +] + +SUITE = common_setup.DEFAULT_OPTIMAL_SUITE +ENVIRONMENT = BaselSlurmEnvironment( + partition="infai_3", + email="silvan.sievers@unibas.ch", + memory_per_cpu="3940M", + cpus_per_task=2, + export=["PATH"], + # paths obtained via: + # module purge + # module -q load Python/3.10.4-GCCcore-11.3.0 + # module -q load GCC/11.3.0 + # module -q load CMake/3.23.1-GCCcore-11.3.0 + # echo $PATH + # echo $LD_LIBRARY_PATH + setup='export PATH=/scicore/soft/apps/CMake/3.23.1-GCCcore-11.3.0/bin:/scicore/soft/apps/libarchive/3.6.1-GCCcore-11.3.0/bin:/scicore/soft/apps/cURL/7.83.0-GCCcore-11.3.0/bin:/scicore/soft/apps/Python/3.10.4-GCCcore-11.3.0/bin:/scicore/soft/apps/OpenSSL/1.1/bin:/scicore/soft/apps/XZ/5.2.5-GCCcore-11.3.0/bin:/scicore/soft/apps/SQLite/3.38.3-GCCcore-11.3.0/bin:/scicore/soft/apps/Tcl/8.6.12-GCCcore-11.3.0/bin:/scicore/soft/apps/ncurses/6.3-GCCcore-11.3.0/bin:/scicore/soft/apps/bzip2/1.0.8-GCCcore-11.3.0/bin:/scicore/soft/apps/binutils/2.38-GCCcore-11.3.0/bin:/scicore/soft/apps/GCCcore/11.3.0/bin:/infai/sieverss/repos/bin:/infai/sieverss/local:/export/soft/lua_lmod/centos7/lmod/lmod/libexec:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:$PATH\nexport LD_LIBRARY_PATH=/scicore/soft/apps/libarchive/3.6.1-GCCcore-11.3.0/lib:/scicore/soft/apps/cURL/7.83.0-GCCcore-11.3.0/lib:/scicore/soft/apps/Python/3.10.4-GCCcore-11.3.0/lib:/scicore/soft/apps/OpenSSL/1.1/lib:/scicore/soft/apps/libffi/3.4.2-GCCcore-11.3.0/lib64:/scicore/soft/apps/GMP/6.2.1-GCCcore-11.3.0/lib:/scicore/soft/apps/XZ/5.2.5-GCCcore-11.3.0/lib:/scicore/soft/apps/SQLite/3.38.3-GCCcore-11.3.0/lib:/scicore/soft/apps/Tcl/8.6.12-GCCcore-11.3.0/lib:/scicore/soft/apps/libreadline/8.1.2-GCCcore-11.3.0/lib:/scicore/soft/apps/ncurses/6.3-GCCcore-11.3.0/lib:/scicore/soft/apps/bzip2/1.0.8-GCCcore-11.3.0/lib:/scicore/soft/apps/binutils/2.38-GCCcore-11.3.0/lib:/scicore/soft/apps/zlib/1.2.12-GCCcore-11.3.0/lib:/scicore/soft/apps/GCCcore/11.3.0/lib64') + +if common_setup.is_test_run(): + SUITE = IssueExperiment.DEFAULT_TEST_SUITE + ENVIRONMENT = LocalEnvironment(processes=4) + +exp = IssueExperiment( + REPO_DIR, + revisions=REVISIONS, + configs=CONFIGS, + environment=ENVIRONMENT, +) +exp.add_suite(BENCHMARKS_DIR, SUITE) + +exp.add_parser(exp.EXITCODE_PARSER) +exp.add_parser(exp.TRANSLATOR_PARSER) +exp.add_parser(exp.SINGLE_SEARCH_PARSER) +exp.add_parser(exp.PLANNER_PARSER) +exp.add_parser('ms-parser.py') + +exp.add_step('build', exp.build) +exp.add_step('start', exp.start_runs) +exp.add_fetcher(name='fetch') + +extra_attributes = [ + # planner outcome attributes + Attribute('perfect_heuristic', absolute=True, min_wins=False), + + # m&s attributes + Attribute('ms_construction_time', absolute=False, min_wins=True, function=geometric_mean), + Attribute('score_ms_construction_time', min_wins=False, digits=4), + Attribute('ms_atomic_construction_time', absolute=False, min_wins=True, function=geometric_mean), + Attribute('ms_abstraction_constructed', absolute=True, min_wins=False), + Attribute('ms_atomic_fts_constructed', absolute=True, min_wins=False), + Attribute('ms_out_of_memory', absolute=True, min_wins=True), + Attribute('ms_out_of_time', absolute=True, min_wins=True), + Attribute('ms_reached_time_limit', absolute=False, min_wins=True), + Attribute('search_out_of_memory', absolute=True, min_wins=True), + Attribute('search_out_of_time', absolute=True, min_wins=True), +] +attributes = exp.DEFAULT_TABLE_ATTRIBUTES +attributes.extend(extra_attributes) + +exp.add_comparison_table_step(attributes=attributes) + +exp.add_archive_step(ARCHIVE_PATH) + +exp.run_steps()