From be975ef544e3a1ede4513b79598df7fc4f799ff6 Mon Sep 17 00:00:00 2001 From: Jeffrey Martin Date: Wed, 24 Apr 2024 16:13:31 -0500 Subject: [PATCH] portable temp file remove for 3.10+ Remove temp files manually for OS portablity in python 3.10+ `delete_on_close` for temp files is added in [3.12](https://github.com/python/cpython/pull/97015) Signed-off-by: Jeffrey Martin --- tests/buffs/test_buff_config.py | 6 +++-- tests/test_config.py | 42 +++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/tests/buffs/test_buff_config.py b/tests/buffs/test_buff_config.py index d5d9fd3be..67d746f97 100644 --- a/tests/buffs/test_buff_config.py +++ b/tests/buffs/test_buff_config.py @@ -24,7 +24,7 @@ def test_include_original_prompt(): # https://github.com/python/cpython/pull/97015 to ensure Windows compatibility - with tempfile.NamedTemporaryFile(buffering=0, delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( """--- plugins: @@ -37,6 +37,7 @@ def test_include_original_prompt(): garak.cli.main( f"-m test -p test.Test -b lowercase.Lowercase --config {tmp.name} --report_prefix {prefix}".split() ) + os.remove(tmp.name) prompts = [] with open(f"{prefix}.report.jsonl", "r", encoding="utf-8") as reportfile: @@ -57,7 +58,7 @@ def test_include_original_prompt(): def test_exclude_original_prompt(): - with tempfile.NamedTemporaryFile(buffering=0, delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( """--- plugins: @@ -70,6 +71,7 @@ def test_exclude_original_prompt(): garak.cli.main( f"-m test -p test.Test -b lowercase.Lowercase --config {tmp.name} --report_prefix {prefix}".split() ) + os.remove(tmp.name) prompts = [] with open(f"{prefix}.report.jsonl", "r", encoding="utf-8") as reportfile: diff --git a/tests/test_config.py b/tests/test_config.py index d42db0bac..84fe0de4c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -110,13 +110,14 @@ def test_yaml_param_settings(param): importlib.reload(_config) option, value = param - with tempfile.NamedTemporaryFile(buffering=0, delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write(f"---\n{param_locs[option]}:\n {option}: {value}\n".encode("utf-8")) tmp.close() garak.cli.main( ["--config", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run subconfig = getattr(_config, param_locs[option]) + os.remove(tmp.name) assert getattr(subconfig, option) == value @@ -174,12 +175,13 @@ def test_cli_overrides_run_yaml(): orig_seed = 10101 override_seed = 37176 - with tempfile.NamedTemporaryFile(buffering=0, delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write(f"---\nrun:\n seed: {orig_seed}\n".encode("utf-8")) tmp.close() garak.cli.main( ["--config", tmp.name, "-s", f"{override_seed}", "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) assert _config.run.seed == override_seed @@ -187,7 +189,7 @@ def test_cli_overrides_run_yaml(): def test_probe_options_yaml(capsys): importlib.reload(_config) - with tempfile.NamedTemporaryFile(buffering=0, delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( """ --- @@ -204,6 +206,7 @@ def test_probe_options_yaml(capsys): garak.cli.main( ["--config", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) assert _config.plugins.probes["test.Blank"]["gen_x"] == 37176 @@ -211,7 +214,7 @@ def test_probe_options_yaml(capsys): def test_generator_options_yaml(capsys): importlib.reload(_config) - with tempfile.NamedTemporaryFile(buffering=0, delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( "---\nplugins:\n model_type: test.Blank\n probe_spec: test.Blank\n generators:\n test.Blank:\n gen_x: 37176\n".encode( "utf-8" @@ -221,6 +224,7 @@ def test_generator_options_yaml(capsys): garak.cli.main( ["--config", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) assert _config.plugins.generators["test.Blank"]["gen_x"] == 37176 @@ -228,7 +232,7 @@ def test_generator_options_yaml(capsys): def test_run_from_yaml(capsys): importlib.reload(_config) - with tempfile.NamedTemporaryFile(buffering=0, delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( "---\nrun:\n generations: 10\n\nplugins:\n model_type: test.Blank\n probe_spec: test.Blank\n".encode( "utf-8" @@ -236,6 +240,7 @@ def test_run_from_yaml(capsys): ) tmp.close() garak.cli.main(["--config", tmp.name]) + os.remove(tmp.name) result = capsys.readouterr() output = result.out all_output = "" @@ -256,13 +261,14 @@ def test_cli_generator_options_file(): importlib.reload(_config) # write an options file - with tempfile.NamedTemporaryFile(mode="w+", delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp: json.dump({"test.Blank": {"this_is_a": "generator"}}, tmp) tmp.close() # invoke cli garak.cli.main( ["--generator_option_file", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) # check it was loaded assert _config.plugins.generators["test.Blank"] == {"this_is_a": "generator"} @@ -273,13 +279,14 @@ def test_cli_probe_options_file(): importlib.reload(_config) # write an options file - with tempfile.NamedTemporaryFile(mode="w+", delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp: json.dump({"test.Blank": {"probes_in_this_config": 1}}, tmp) tmp.close() # invoke cli garak.cli.main( ["--probe_option_file", tmp.name, "--list_config"] ) # add list_config as the action so we don't actually run + os.remove(tmp.name) # check it was loaded assert _config.plugins.probes["test.Blank"] == {"probes_in_this_config": 1} @@ -290,14 +297,10 @@ def test_cli_probe_options_overrides_yaml_probe_options(): importlib.reload(_config) # write an options file - with tempfile.NamedTemporaryFile( - mode="w+", delete_on_close=False - ) as probe_json_file: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as probe_json_file: json.dump({"test.Blank": {"goal": "taken from CLI JSON"}}, probe_json_file) probe_json_file.close() - with tempfile.NamedTemporaryFile( - buffering=0, delete_on_close=False - ) as probe_yaml_file: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as probe_yaml_file: probe_yaml_file.write( """ --- @@ -320,6 +323,8 @@ def test_cli_probe_options_overrides_yaml_probe_options(): "--list_config", ] ) # add list_config as the action so we don't actually run + os.remove(probe_json_file.name) + os.remove(probe_yaml_file.name) # check it was loaded assert _config.plugins.probes["test.Blank"]["goal"] == "taken from CLI JSON" @@ -329,9 +334,7 @@ def test_cli_generator_options_overrides_yaml_probe_options(): importlib.reload(_config) cli_generations_count = 9001 - with tempfile.NamedTemporaryFile( - buffering=0, delete_on_close=False - ) as generator_yaml_file: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as generator_yaml_file: generator_yaml_file.write( """ --- @@ -351,6 +354,7 @@ def test_cli_generator_options_overrides_yaml_probe_options(): ] # add list_config as the action so we don't actually run print(args) garak.cli.main(args) + os.remove(generator_yaml_file.name) # check it was loaded assert _config.run.generations == cli_generations_count @@ -361,7 +365,7 @@ def test_blank_probe_instance_loads_yaml_config(): probe_name = "test.Blank" revised_goal = "TEST GOAL make the model forget what to output" - with tempfile.NamedTemporaryFile(buffering=0, delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( f"---\nplugins:\n probes:\n {probe_name}:\n goal: {revised_goal}\n".encode( "utf-8" @@ -369,6 +373,7 @@ def test_blank_probe_instance_loads_yaml_config(): ) tmp.close() garak.cli.main(["--config", tmp.name, "-p", probe_name]) + os.remove(tmp.name) probe = garak._plugins.load_plugin(f"probes.{probe_name}") assert probe.goal == revised_goal @@ -396,7 +401,7 @@ def test_blank_generator_instance_loads_yaml_config(): generator_name = "test.Blank" revised_temp = 0.9001 - with tempfile.NamedTemporaryFile(buffering=0, delete_on_close=False) as tmp: + with tempfile.NamedTemporaryFile(buffering=0, delete=False) as tmp: tmp.write( f"---\nplugins:\n generators:\n {generator_name}:\n temperature: {revised_temp}\n".encode( "utf-8" @@ -406,6 +411,7 @@ def test_blank_generator_instance_loads_yaml_config(): garak.cli.main( ["--config", tmp.name, "--model_type", generator_name, "--probes", "none"] ) + os.remove(tmp.name) gen = garak._plugins.load_plugin(f"generators.{generator_name}") assert gen.temperature == revised_temp