Allow incremental run to work with step-by-step

JetBrains-Research · Oct 1, 2024 · 6ccc4e4 · 6ccc4e4
1 parent b08dfbb
commit 6ccc4e4
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 32 deletions.
diff --git a/gui/src/helpers.rs b/gui/src/helpers.rs
@@ -59,11 +59,6 @@ fn add_common_arguments<'a>(
     token: &str,
     settings: &Settings,
 ) -> &'a mut Command {
-    let bench_type = if settings.incremental_run {
-        String::from("validating")
-    } else {
-        settings.bench_type.to_string()
-    };
     if settings.do_filter {
         cmd.args(["--filter-by-ext", &settings.filter_by_ext]);
     }
@@ -72,7 +67,7 @@ fn add_common_arguments<'a>(
         .args(["--insert-conditions-mode", "llm-single-step"])
         .args(["--llm-profile", settings.llm_profile.as_grazie()])
         .args(["--grazie-token", token])
-        .args(["--bench-type", &bench_type])
+        .args(["--bench-type", &settings.bench_type.to_string()])
         .args(["--tries", &make_tries(&settings.tries)])
         .args(["--retries", &make_retries(&settings.retries)])
         .args(["--verifier-timeout", &make_timeout(&settings.timeout)])

diff --git a/gui/src/ui/settings.rs b/gui/src/ui/settings.rs
@@ -37,22 +37,16 @@ impl AppState {
                 self.settings.file_mode = FileMode::Directory;
             }
             ui.add_space(2.0);
-            if !self.settings.incremental_run {
-                ui.horizontal(|ui| {
-                    ui.label("Bench mode:");
-                    egui::ComboBox::from_id_source("bench-mode-select")
-                        .selected_text(format!("{}", self.settings.bench_type.name()))
-                        .show_ui(ui, |ui| {
-                            for mode in BenchMode::all() {
-                                ui.selectable_value(
-                                    &mut self.settings.bench_type,
-                                    *mode,
-                                    mode.name(),
-                                );
-                            }
-                        });
-                });
-            }
+            ui.horizontal(|ui| {
+                ui.label("Bench mode:");
+                egui::ComboBox::from_id_source("bench-mode-select")
+                    .selected_text(format!("{}", self.settings.bench_type.name()))
+                    .show_ui(ui, |ui| {
+                        for mode in BenchMode::all() {
+                            ui.selectable_value(&mut self.settings.bench_type, *mode, mode.name());
+                        }
+                    });
+            });
 
             ui.separator();
             ui.horizontal(|ui| {

diff --git a/verified_cogen/experiments/incremental_run.py b/verified_cogen/experiments/incremental_run.py
@@ -10,12 +10,10 @@
     extension_from_file_list,
     register_output_handler,
 )
-from verified_cogen.runners.invariants import InvariantRunner
 from verified_cogen.runners.languages import register_basic_languages
-from verified_cogen.runners.languages.language import LanguageDatabase
-from verified_cogen.runners.validating import ValidatingRunner
 from verified_cogen.tools.modes import Mode
 from verified_cogen.tools.verifier import Verifier
+from verified_cogen.main import make_runner_cls
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +25,6 @@ def main():
     mode = Mode(args.insert_conditions_mode)
     assert mode != Mode.REGEX
     assert args.dir is not None
-    assert args.bench_type == "validating", args.bench_type
     assert args.runs == 1
     assert args.retries == 0
 
@@ -49,7 +46,6 @@ def main():
     assert len(files) > 0, "No files found in the directory"
     files.sort()
 
-    language = LanguageDatabase().get(extension_from_file_list(files))
     verifier = Verifier(args.shell, args.verifier_command)
 
     for file in files:
@@ -59,11 +55,9 @@ def main():
             args.prompts_directory,
             args.temperature,
         )
-        runner = ValidatingRunner(
-            wrapping=InvariantRunner(llm, logger, verifier),
-            language=language,
-            log_tries=log_tries,
-        )
+        runner = make_runner_cls(
+            args.bench_type, extension_from_file_list([file]), log_tries
+        )(llm, logger, verifier)
         display_name = rename_file(file)
         marker_name = str(file.relative_to(directory))
         if marker_name in results and isinstance(results[marker_name], int):

diff --git a/verified_cogen/main.py b/verified_cogen/main.py
@@ -72,6 +72,8 @@ def run_once(
         if tries is not None:
             cnt[name] = tries
 
+        llm.dump_history()
+
     if is_once:
         if args.output_style == "full":
             success_zero_tries_tabbed = tabulate_list(success_zero_tries)