From 2a7435996a58602e40617320ea75dccb488e1135 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Fri, 26 Jul 2024 14:48:40 +0300 Subject: [PATCH 01/15] Bugfixes and improvements --- .../NaginiBench/invariants_removed/Clover_abs.py | 14 -------------- verified_cogen/main.py | 3 +++ verified_cogen/tools/verifier.py | 2 +- 3 files changed, 4 insertions(+), 15 deletions(-) delete mode 100644 benches/NaginiBench/invariants_removed/Clover_abs.py diff --git a/benches/NaginiBench/invariants_removed/Clover_abs.py b/benches/NaginiBench/invariants_removed/Clover_abs.py deleted file mode 100644 index 2784fe4..0000000 --- a/benches/NaginiBench/invariants_removed/Clover_abs.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import List -from nagini_contracts.contracts import * - -def Abs(x : int) -> int: - Ensures(not ((x) >= (0)) or ((x) == (Result()))) - Ensures(not ((x) < (0)) or (((x) + (Result())) == (0))) - y = int(0) # type : int - if (x) < (0): - y = (0) - (x) - return y - elif True: - y = x - return y - diff --git a/verified_cogen/main.py b/verified_cogen/main.py index 4e949d8..71dfaa9 100644 --- a/verified_cogen/main.py +++ b/verified_cogen/main.py @@ -83,6 +83,9 @@ def main(): verifier = Verifier(args.shell, args.verifier_command, args.verifier_timeout) if args.dir is not None: files = list(pathlib.Path(args.dir).glob("[!.]*")) + for file in files: + with open(file) as f: + runner.precheck(f.read(), mode) if args.runs == 1: run_once(files, args, runner, verifier, mode, is_once=True) diff --git a/verified_cogen/tools/verifier.py b/verified_cogen/tools/verifier.py index 8d114de..ea23760 100644 --- a/verified_cogen/tools/verifier.py +++ b/verified_cogen/tools/verifier.py @@ -16,7 +16,7 @@ def __init__(self, shell: str, verifier_cmd: str, timeout: int = 60): def verify(self, file_path: Path) -> Optional[tuple[bool, str, str]]: try: res = subprocess.run( - '{} -i -l -c "{} {}"; exit'.format( + '{} -i -l -c "{} "{}""; exit'.format( self.shell, self.verifier_cmd, file_path ), capture_output=True, From 47c089520a990a249533f5ce05a314b0bc65e8e2 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Mon, 29 Jul 2024 11:23:38 +0300 Subject: [PATCH 02/15] Move RustBench WIPs to other directory --- .../RustBench/{ground_truth => WIP}/WIP: largest_prime_factor.rs | 0 benches/RustBench/{ground_truth => WIP}/WIP: max_segment_sum.rs | 0 benches/RustBench/{ground_truth => WIP}/WIP: mcontained.rs | 0 benches/RustBench/{ground_truth => WIP}/WIP: transpose_matrix.rs | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename benches/RustBench/{ground_truth => WIP}/WIP: largest_prime_factor.rs (100%) rename benches/RustBench/{ground_truth => WIP}/WIP: max_segment_sum.rs (100%) rename benches/RustBench/{ground_truth => WIP}/WIP: mcontained.rs (100%) rename benches/RustBench/{ground_truth => WIP}/WIP: transpose_matrix.rs (100%) diff --git a/benches/RustBench/ground_truth/WIP: largest_prime_factor.rs b/benches/RustBench/WIP/WIP: largest_prime_factor.rs similarity index 100% rename from benches/RustBench/ground_truth/WIP: largest_prime_factor.rs rename to benches/RustBench/WIP/WIP: largest_prime_factor.rs diff --git a/benches/RustBench/ground_truth/WIP: max_segment_sum.rs b/benches/RustBench/WIP/WIP: max_segment_sum.rs similarity index 100% rename from benches/RustBench/ground_truth/WIP: max_segment_sum.rs rename to benches/RustBench/WIP/WIP: max_segment_sum.rs diff --git a/benches/RustBench/ground_truth/WIP: mcontained.rs b/benches/RustBench/WIP/WIP: mcontained.rs similarity index 100% rename from benches/RustBench/ground_truth/WIP: mcontained.rs rename to benches/RustBench/WIP/WIP: mcontained.rs diff --git a/benches/RustBench/ground_truth/WIP: transpose_matrix.rs b/benches/RustBench/WIP/WIP: transpose_matrix.rs similarity index 100% rename from benches/RustBench/ground_truth/WIP: transpose_matrix.rs rename to benches/RustBench/WIP/WIP: transpose_matrix.rs From 5a8698165c2241ecd464516e1cc29edb724ad526 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Mon, 29 Jul 2024 13:17:43 +0300 Subject: [PATCH 03/15] Houdini iterations --- prompts/rust_invariants/add.txt | 10 ++ prompts/rust_invariants_with_examples/add.txt | 13 ++ run_with_houdini.sh | 4 +- verified_cogen/experiments/use_houdini.py | 160 ++++++++++++------ verified_cogen/llm/llm.py | 12 +- 5 files changed, 143 insertions(+), 56 deletions(-) diff --git a/prompts/rust_invariants/add.txt b/prompts/rust_invariants/add.txt index adc90ac..da84320 100644 --- a/prompts/rust_invariants/add.txt +++ b/prompts/rust_invariants/add.txt @@ -1,4 +1,14 @@ Given the following Rust program, and a set of Verus invariants, output the program with invariants inserted into the correct place. +Here's an example of correctly inserted invariants: +```rust +while i < nums.len() + invariant + 0 < i <= nums.len(), + 0 <= idx && idx < i, + forall|j: int| 0 <= j && j < i ==> nums[j] <= nums[idx as int], +``` +Remember to put commas between invariants + The program: {program} – diff --git a/prompts/rust_invariants_with_examples/add.txt b/prompts/rust_invariants_with_examples/add.txt index adc90ac..2c748da 100644 --- a/prompts/rust_invariants_with_examples/add.txt +++ b/prompts/rust_invariants_with_examples/add.txt @@ -1,4 +1,17 @@ Given the following Rust program, and a set of Verus invariants, output the program with invariants inserted into the correct place. +Here's an example of correctly inserted invariants: +```rust +while i < a.len() + invariant + 0 <= i <= a.len(), + forall|k: int, l: int| + #![trigger result[k], result[l]] + 0 <= k && k < l && l < result.len() ==> result[k] < result[l], + forall|k: int| + #![trigger result[k]] + 0 <= k && k < result.len() ==> exists|m: int| 0 <= m < i && result[k] == a[m], +``` + The program: {program} – diff --git a/run_with_houdini.sh b/run_with_houdini.sh index 454b325..4d1d147 100755 --- a/run_with_houdini.sh +++ b/run_with_houdini.sh @@ -5,9 +5,9 @@ source .envrc export PYTHONPATH=PYTHONPATH:"$(pwd)/src" -PYLOG_LEVEL=DEBUG NOFILE=1 python src/experiments/use_houdini.py \ +PYLOG_LEVEL=INFO NOFILE=1 poetry run python verified_cogen/experiments/use_houdini.py \ --grazie-token=$GRAZIE_TOKEN \ - --profile=gpt-4o \ + --profile="anthropic-claude-3.5-sonnet" \ --prompt-dir=prompts/rust_invariants \ --verifier-command="verus --multiple-errors=100" \ --program=$1 diff --git a/verified_cogen/experiments/use_houdini.py b/verified_cogen/experiments/use_houdini.py index 6d7eb3b..ccec73c 100644 --- a/verified_cogen/experiments/use_houdini.py +++ b/verified_cogen/experiments/use_houdini.py @@ -3,43 +3,88 @@ import logging import os -from pathlib import Path from typing import Optional +from verified_cogen.runners import LLM_GENERATED_DIR from verified_cogen.tools.verifier import Verifier from verified_cogen.llm import LLM -logging.basicConfig(level=logging.INFO) log = logging.getLogger(__name__) -EXAMPLES_PROMPT = "Before we start, here are some examples of correctly verified programs using Verus:" - -EXAMPLES = [ - "has_close_elements", - "index_wise_addition", - "smallest_missing_number", - "is_non_prime", - "unique_better", -] - - -def construct_examples() -> str: - result = EXAMPLES_PROMPT + "\n" - for example in EXAMPLES: - with open(f"RustBench/ground_truth/{example}.rs", "r") as f: - result += f.read() + "\n" - return result - INVARIANTS_JSON_PROMPT = """Given the following Rust program, output Verus invariants that should go into the `while` loop. Ensure that the invariants are as comprehensive as they can be. Even if you think some invariant is not totally necessary, better add it than not. +Don't be afraid of using disjunctions if you see that some invariant is not true, for example, at the beginning of the loop. Respond with a JSON array of strings, representing the invariants. Remember that invariants should not contain a word `invariant` in them, just the expression. +Here are some examples of verified functions: +```rust +fn incr_list(l: Vec) -> (result: Vec) + requires + forall|i: int| 0 <= i < l.len() ==> l[i] + 1 <= i32::MAX, // avoid overflow + + ensures + result.len() == l.len(), + forall|i: int| 0 <= i < l.len() ==> #[trigger] result[i] == l[i] + 1, +{ + let mut result = Vec::new(); + for i in 0..l.len() + invariant + forall|i: int| 0 <= i < l.len() ==> l[i] + 1 <= i32::MAX, + result.len() == i, + forall|j: int| 0 <= j < i ==> #[trigger] result[j] == l[j] + 1, + { + result.push(l[i] + 1); + } + result +} +``` + +```rust +#[verifier::loop_isolation(false)] +fn unique(a: &[i32]) -> (result: Vec) + requires + forall|i: int, j: int| + #![trigger a[i], a[j]] + 0 <= i && i < j && j < a.len() ==> a[i] <= a[j], + ensures + forall|i: int, j: int| + #![trigger result[i], result[j]] + 0 <= i && i < j && j < result.len() ==> result[i] < result[j], +{ + let mut result: Vec = Vec::new(); + let mut i = 0; + while i < a.len() + invariant + 0 <= i <= a.len(), + forall|k: int, l: int| + #![trigger result[k], result[l]] + 0 <= k && k < l && l < result.len() ==> result[k] < result[l], + forall|k: int| + #![trigger result[k]] + 0 <= k && k < result.len() ==> exists|m: int| 0 <= m < i && result[k] == a[m], + { + if result.len() == 0 || result[result.len() - 1] != a[i] { + assert(result.len() == 0 || result[result.len() - 1] < a[i as int]); + result.push(a[i]); + } + i = i + 1; + } + result +} +``` The program: {program} """ +HOUDINI_SYS_PROMPT = """ +You are an expert in a Rust verification framework Verus. +Do not provide ANY explanations. Don't include markdown backticks. Respond only in Rust code when not asked otherwise. +You will be working with the following program: +{program} +""" + REMOVE_FAILED_INVARIANTS_PROMPT = """Some of the provided invariants either have syntax errors or failed to verify. Could you please remove the invariants that failed to verify and provide the rest again as a JSON array of strings? @@ -48,32 +93,22 @@ def construct_examples() -> str: """ -def reset_llm(llm: LLM): - llm.had_errors = False - llm.user_prompts = [] - llm.responses = [] - llm.user_prompts.append(construct_examples()) - - def collect_invariants(args, prg: str): result_invariants = [] - for temperature in range(0, 5): + for temperature in [0.0, 0.1, 0.3, 0.4, 0.5, 0.7, 1.0]: llm = LLM( grazie_token=args.grazie_token, profile=args.profile, prompt_dir=args.prompt_dir, - temperature=temperature / 10, + temperature=temperature, ) - reset_llm(llm) - llm.user_prompts.append(INVARIANTS_JSON_PROMPT.format(program=prg)) + llm.user_prompts.append(INVARIANTS_JSON_PROMPT.replace("{program}", prg)) response = llm._make_request() try: invariants = json.loads(response) result_invariants.extend(invariants) - log.info( - f"Got {len(invariants)} invariants at temperature {temperature / 10.0}" - ) + log.debug(f"Got {len(invariants)} invariants at temperature {temperature}") except json.JSONDecodeError: print("Error parsing response as JSON") print(response) @@ -88,7 +123,7 @@ def remove_failed_invariants( response = llm._make_request() try: new_invariants = json.loads(response) - log.info("REMOVED: {}".format(set(invariants).difference(set(new_invariants)))) + log.debug("REMOVED: {}".format(set(invariants).difference(set(new_invariants)))) return new_invariants except json.JSONDecodeError: print("Error parsing response as JSON") @@ -97,16 +132,23 @@ def remove_failed_invariants( def houdini( - llm: LLM, verifier: Verifier, prg: str, invariants: list[str] + args, verifier: Verifier, prg: str, invariants: list[str] ) -> Optional[list[str]]: while len(invariants) > 0: - reset_llm(llm) + llm = LLM( + grazie_token=args.grazie_token, + profile=args.profile, + prompt_dir=args.prompt_dir, + temperature=0.0, + system_prompt=HOUDINI_SYS_PROMPT.replace("{program}", prg), + ) + prg_with_invariants = llm.add(prg, "\n".join(invariants)) - with open("llm-generated/collected.rs", "w") as f: + with open(LLM_GENERATED_DIR / "collected.rs", "w") as f: f.write(prg_with_invariants) - log.info(f"Trying to verify with {json.dumps(invariants, indent=2)}") - ver_result = verifier.verify(Path("llm-generated/collected.rs")) + log.debug(f"Trying to verify with {json.dumps(invariants, indent=2)}") + ver_result = verifier.verify(LLM_GENERATED_DIR / "collected.rs") if ver_result is None: log.info("Verifier timed out") return None @@ -116,11 +158,28 @@ def houdini( return invariants else: log.info("Failed to verify invariants") - log.info("Error: {}".format(err)) + log.debug("Error: {}".format(err)) new_invariants = remove_failed_invariants(llm, invariants, out + err) - if new_invariants is None: + if new_invariants is None or new_invariants == invariants: return None + inv_set = set(invariants) + is_subset = True + for inv in set(new_invariants): + if inv not in inv_set: + is_subset = False + log.error( + f"New invariant {inv} is not a subset of the old invariants" + ) + if not is_subset: + log.error("New invariants are not a subset of the old invariants") + log.error("Old invariants: {}".format(json.dumps(invariants, indent=2))) + log.error( + "New invariants: {}".format(json.dumps(new_invariants, indent=2)) + ) + raise ValueError( + "New invariants are not a subset of the old invariants" + ) invariants = new_invariants @@ -134,25 +193,22 @@ def main(): args = parser.parse_args() + log.info("Running on program: {}".format(args.program)) + with open(args.program, "r") as f: prg = f.read() invariants = collect_invariants(args, prg) log.info("Collected {} invariants".format(len(invariants))) - log.info("Invariants: {}".format(json.dumps(invariants, indent=4))) + log.debug("Invariants: {}".format(json.dumps(invariants, indent=4))) verifier = Verifier(os.environ["SHELL"], args.verifier_command) - llm = LLM( - grazie_token=args.grazie_token, - profile=args.profile, - prompt_dir=args.prompt_dir, - temperature=0.3, - ) - reset_llm(llm) - result = houdini(llm, verifier, prg, invariants) + result = houdini(args, verifier, prg, invariants) if result is not None: log.info("Vefication successful") - print(json.dumps(result, indent=2)) + log.debug(json.dumps(result, indent=2)) + else: + log.error("Verification failed") if __name__ == "__main__": diff --git a/verified_cogen/llm/llm.py b/verified_cogen/llm/llm.py index fd1e695..df2ec1e 100644 --- a/verified_cogen/llm/llm.py +++ b/verified_cogen/llm/llm.py @@ -12,7 +12,12 @@ class LLM: def __init__( - self, grazie_token: str, profile: str, prompt_dir: str, temperature: float + self, + grazie_token: str, + profile: str, + prompt_dir: str, + temperature: float, + system_prompt: Optional[str] = None, ): self.grazie = GrazieApiGatewayClient( url=GrazieApiGatewayUrls.STAGING, @@ -26,11 +31,14 @@ def __init__( self.responses = [] self.had_errors = False self.temperature = temperature + self.system_prompt = ( + system_prompt if system_prompt else prompts.sys_prompt(self.prompt_dir) + ) def _request(self, temperature: Optional[float] = None): if temperature is None: temperature = self.temperature - prompt = ChatPrompt().add_system(prompts.sys_prompt(self.prompt_dir)) + prompt = ChatPrompt().add_system(self.system_prompt) current_prompt_user = 0 current_response = 0 while current_prompt_user < len(self.user_prompts) or current_response < len( From d042073cdc6342b5b8c99bf46d3de3d38d45a3d9 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Tue, 30 Jul 2024 20:11:52 +0400 Subject: [PATCH 04/15] Finish houdini, now runs on all examples --- .../ground_truth/arithmetic_weird.rs | 2 +- .../RustBench/ground_truth/max_dafny_lsp.rs | 2 +- .../RustBench/ground_truth/unique_better.rs | 4 ++-- .../invariants_removed/remove_duplicates.rs | 3 +++ prompts/rust_invariants_with_examples/add.txt | 2 +- prompts/rust_preconditions/add.txt | 14 ++++++++++++++ verified_cogen/experiments/use_houdini.py | 19 +++++++++++++++---- verified_cogen/llm/llm.py | 9 +++++---- 8 files changed, 42 insertions(+), 13 deletions(-) diff --git a/benches/RustBench/ground_truth/arithmetic_weird.rs b/benches/RustBench/ground_truth/arithmetic_weird.rs index b921066..0bb6961 100644 --- a/benches/RustBench/ground_truth/arithmetic_weird.rs +++ b/benches/RustBench/ground_truth/arithmetic_weird.rs @@ -3,7 +3,7 @@ use vstd::prelude::*; verus! { #[verifier::loop_isolation(false)] -fn arithmetic() -> (result: i32) +fn arithmetic_weird() -> (result: i32) ensures result < 10 { diff --git a/benches/RustBench/ground_truth/max_dafny_lsp.rs b/benches/RustBench/ground_truth/max_dafny_lsp.rs index 7634a12..dd8effc 100644 --- a/benches/RustBench/ground_truth/max_dafny_lsp.rs +++ b/benches/RustBench/ground_truth/max_dafny_lsp.rs @@ -3,7 +3,7 @@ use vstd::prelude::*; verus! { #[verifier::loop_isolation(false)] -fn max(a: &[i32]) -> (x: usize) +fn max_dafny_lsp(a: &[i32]) -> (x: usize) requires a.len() > 0, ensures diff --git a/benches/RustBench/ground_truth/unique_better.rs b/benches/RustBench/ground_truth/unique_better.rs index 04ce024..6d09ac9 100644 --- a/benches/RustBench/ground_truth/unique_better.rs +++ b/benches/RustBench/ground_truth/unique_better.rs @@ -3,7 +3,7 @@ use vstd::prelude::*; verus! { #[verifier::loop_isolation(false)] -fn unique(a: &[i32]) -> (result: Vec) +fn unique_better(a: &[i32]) -> (result: Vec) requires forall|i: int, j: int| #![trigger a[i], a[j]] @@ -35,4 +35,4 @@ fn unique(a: &[i32]) -> (result: Vec) } fn main() {} -} \ No newline at end of file +} diff --git a/benches/RustBench/invariants_removed/remove_duplicates.rs b/benches/RustBench/invariants_removed/remove_duplicates.rs index 05845e0..cc9eecd 100644 --- a/benches/RustBench/invariants_removed/remove_duplicates.rs +++ b/benches/RustBench/invariants_removed/remove_duplicates.rs @@ -12,6 +12,9 @@ fn in_array_exec(a: &Vec, x: i32) -> (result: bool) { let mut i = 0; while i < a.len() + invariant + 0 <= i <= a.len(), + forall|k: int| 0 <= k < i ==> a[k] != x, { if a[i] == x { return true; diff --git a/prompts/rust_invariants_with_examples/add.txt b/prompts/rust_invariants_with_examples/add.txt index 2c748da..c4db60d 100644 --- a/prompts/rust_invariants_with_examples/add.txt +++ b/prompts/rust_invariants_with_examples/add.txt @@ -11,7 +11,7 @@ while i < a.len() #![trigger result[k]] 0 <= k && k < result.len() ==> exists|m: int| 0 <= m < i && result[k] == a[m], ``` - +Do not touch any functions other that {function} The program: {program} – diff --git a/prompts/rust_preconditions/add.txt b/prompts/rust_preconditions/add.txt index b71c9c9..c5fb014 100644 --- a/prompts/rust_preconditions/add.txt +++ b/prompts/rust_preconditions/add.txt @@ -1,4 +1,18 @@ Given the following Rust program, and a set of Verus preconditions, output the program with preconditions inserted into the correct place. +Here's an example of correctly inserted invariants: +```rust +while i < a.len() + invariant + 0 <= i <= a.len(), + forall|k: int, l: int| + #![trigger result[k], result[l]] + 0 <= k && k < l && l < result.len() ==> result[k] < result[l], + forall|k: int| + #![trigger result[k]] + 0 <= k && k < result.len() ==> exists|m: int| 0 <= m < i && result[k] == a[m], +``` +Do not touch any functions other that {function} + The program: {program} – diff --git a/verified_cogen/experiments/use_houdini.py b/verified_cogen/experiments/use_houdini.py index ccec73c..31fef28 100644 --- a/verified_cogen/experiments/use_houdini.py +++ b/verified_cogen/experiments/use_houdini.py @@ -5,6 +5,7 @@ from typing import Optional from verified_cogen.runners import LLM_GENERATED_DIR +from verified_cogen.tools import basename from verified_cogen.tools.verifier import Verifier from verified_cogen.llm import LLM @@ -12,7 +13,8 @@ log = logging.getLogger(__name__) -INVARIANTS_JSON_PROMPT = """Given the following Rust program, output Verus invariants that should go into the `while` loop. +INVARIANTS_JSON_PROMPT = """Given the following Rust program, output Verus invariants that should go into the `while` loop +in the function {function}. Ensure that the invariants are as comprehensive as they can be. Even if you think some invariant is not totally necessary, better add it than not. Don't be afraid of using disjunctions if you see that some invariant is not true, for example, at the beginning of the loop. @@ -81,12 +83,14 @@ HOUDINI_SYS_PROMPT = """ You are an expert in a Rust verification framework Verus. Do not provide ANY explanations. Don't include markdown backticks. Respond only in Rust code when not asked otherwise. +Do not touch any functions other that {function} You will be working with the following program: {program} """ REMOVE_FAILED_INVARIANTS_PROMPT = """Some of the provided invariants either have syntax errors or failed to verify. Could you please remove the invariants that failed to verify and provide the rest again as a JSON array of strings? +DO NOT MODIFY THE INVARIANTS OR ADD NEW ONES. Here's an error from the verifier: {error} @@ -94,6 +98,7 @@ def collect_invariants(args, prg: str): + func = basename(args.program)[:-3] result_invariants = [] for temperature in [0.0, 0.1, 0.3, 0.4, 0.5, 0.7, 1.0]: llm = LLM( @@ -103,7 +108,9 @@ def collect_invariants(args, prg: str): temperature=temperature, ) - llm.user_prompts.append(INVARIANTS_JSON_PROMPT.replace("{program}", prg)) + llm.user_prompts.append( + INVARIANTS_JSON_PROMPT.replace("{program}", prg).replace("{function}", func) + ) response = llm._make_request() try: invariants = json.loads(response) @@ -134,16 +141,20 @@ def remove_failed_invariants( def houdini( args, verifier: Verifier, prg: str, invariants: list[str] ) -> Optional[list[str]]: + func = basename(args.program).strip(".rs") + log.info(f"Starting Houdini for {func} in file {args.program}") while len(invariants) > 0: llm = LLM( grazie_token=args.grazie_token, profile=args.profile, prompt_dir=args.prompt_dir, temperature=0.0, - system_prompt=HOUDINI_SYS_PROMPT.replace("{program}", prg), + system_prompt=HOUDINI_SYS_PROMPT.replace("{program}", prg).replace( + "{function}", func + ), ) - prg_with_invariants = llm.add(prg, "\n".join(invariants)) + prg_with_invariants = llm.add(prg, "\n".join(invariants), func) with open(LLM_GENERATED_DIR / "collected.rs", "w") as f: f.write(prg_with_invariants) diff --git a/verified_cogen/llm/llm.py b/verified_cogen/llm/llm.py index df2ec1e..920f7ad 100644 --- a/verified_cogen/llm/llm.py +++ b/verified_cogen/llm/llm.py @@ -68,10 +68,11 @@ def produce(self, prg: str): ) return self._make_request() - def add(self, prg: str, checks: str): - self.user_prompts.append( - prompts.add_prompt(self.prompt_dir).format(program=prg, checks=checks) - ) + def add(self, prg: str, checks: str, function: Optional[str] = None): + prompt = prompts.add_prompt(self.prompt_dir).format(program=prg, checks=checks) + if "{function}" in prompt and function is not None: + prompt = prompt.replace("{function}", function) + self.user_prompts.append(prompt) return self._make_request() def rewrite(self, prg: str): From af2e1f29b9c86fe4dc276ee3866ffc019800ee73 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Tue, 6 Aug 2024 18:26:04 +0400 Subject: [PATCH 05/15] Add generate runner, human eval prompt as a bench --- .../Strongest_Extension.prompt | 16 ++++ benches/HumanEvalPrompts/add.prompt | 6 ++ benches/HumanEvalPrompts/add_elements.prompt | 13 +++ benches/HumanEvalPrompts/all_prefixes.prompt | 4 + benches/HumanEvalPrompts/anti_shuffle.prompt | 12 +++ benches/HumanEvalPrompts/any_int.prompt | 17 ++++ .../HumanEvalPrompts/below_threshold.prompt | 6 ++ benches/HumanEvalPrompts/below_zero.prompt | 8 ++ benches/HumanEvalPrompts/bf.prompt | 15 ++++ benches/HumanEvalPrompts/by_length.prompt | 21 +++++ benches/HumanEvalPrompts/can_arrange.prompt | 9 ++ .../car_race_collision.prompt | 11 +++ benches/HumanEvalPrompts/change_base.prompt | 10 +++ .../HumanEvalPrompts/check_dict_case.prompt | 11 +++ .../check_if_last_char_is_a_letter.prompt | 12 +++ benches/HumanEvalPrompts/choose_num.prompt | 8 ++ .../HumanEvalPrompts/circular_shift.prompt | 8 ++ .../HumanEvalPrompts/closest_integer.prompt | 17 ++++ benches/HumanEvalPrompts/common.prompt | 7 ++ benches/HumanEvalPrompts/compare.prompt | 14 +++ benches/HumanEvalPrompts/compare_one.prompt | 11 +++ benches/HumanEvalPrompts/concatenate.prompt | 6 ++ .../correct_bracketing.prompt | 12 +++ .../count_distinct_characters.prompt | 6 ++ benches/HumanEvalPrompts/count_nums.prompt | 9 ++ benches/HumanEvalPrompts/count_up_to.prompt | 10 +++ benches/HumanEvalPrompts/count_upper.prompt | 8 ++ .../HumanEvalPrompts/cycpattern_check.prompt | 9 ++ .../HumanEvalPrompts/decimal_to_binary.prompt | 11 +++ benches/HumanEvalPrompts/derivative.prompt | 8 ++ benches/HumanEvalPrompts/digitSum.prompt | 12 +++ benches/HumanEvalPrompts/digits.prompt | 7 ++ benches/HumanEvalPrompts/do_algebra.prompt | 24 ++++++ .../double_the_difference.prompt | 11 +++ benches/HumanEvalPrompts/eat.prompt | 28 ++++++ benches/HumanEvalPrompts/encode.prompt | 13 +++ benches/HumanEvalPrompts/encode_cyclic.prompt | 3 + benches/HumanEvalPrompts/encode_shift.prompt | 3 + benches/HumanEvalPrompts/encrypt.prompt | 10 +++ .../HumanEvalPrompts/even_odd_count.prompt | 6 ++ .../even_odd_palindrome.prompt | 22 +++++ benches/HumanEvalPrompts/exchange.prompt | 12 +++ benches/HumanEvalPrompts/f.prompt | 8 ++ benches/HumanEvalPrompts/factorize.prompt | 10 +++ benches/HumanEvalPrompts/fib.prompt | 8 ++ benches/HumanEvalPrompts/fib4.prompt | 14 +++ benches/HumanEvalPrompts/fibfib.prompt | 13 +++ .../HumanEvalPrompts/file_name_check.prompt | 13 +++ .../HumanEvalPrompts/filter_by_prefix.prompt | 6 ++ .../filter_by_substring.prompt | 6 ++ .../HumanEvalPrompts/filter_integers.prompt | 6 ++ .../find_closest_elements.prompt | 7 ++ benches/HumanEvalPrompts/find_max.prompt | 9 ++ benches/HumanEvalPrompts/fix_spaces.prompt | 10 +++ benches/HumanEvalPrompts/fizz_buzz.prompt | 8 ++ benches/HumanEvalPrompts/flip_case.prompt | 4 + .../fruit_distribution.prompt | 12 +++ .../HumanEvalPrompts/generate_integers.prompt | 9 ++ .../HumanEvalPrompts/get_closest_vowel.prompt | 14 +++ .../HumanEvalPrompts/get_max_triples.prompt | 13 +++ .../HumanEvalPrompts/get_odd_collatz.prompt | 16 ++++ benches/HumanEvalPrompts/get_positive.prompt | 6 ++ benches/HumanEvalPrompts/get_row.prompt | 19 +++++ .../greatest_common_divisor.prompt | 6 ++ .../has_close_elements.prompt | 7 ++ benches/HumanEvalPrompts/hex_key.prompt | 17 ++++ benches/HumanEvalPrompts/histogram.prompt | 12 +++ .../HumanEvalPrompts/how_many_times.prompt | 8 ++ benches/HumanEvalPrompts/incr_list.prompt | 6 ++ .../HumanEvalPrompts/int_to_mini_roman.prompt | 10 +++ benches/HumanEvalPrompts/intersection.prompt | 19 +++++ benches/HumanEvalPrompts/intersperse.prompt | 6 ++ benches/HumanEvalPrompts/is_bored.prompt | 11 +++ .../is_equal_to_sum_even.prompt | 6 ++ benches/HumanEvalPrompts/is_happy.prompt | 11 +++ .../HumanEvalPrompts/is_multiply_prime.prompt | 7 ++ benches/HumanEvalPrompts/is_nested.prompt | 12 +++ benches/HumanEvalPrompts/is_palindrome.prompt | 11 +++ benches/HumanEvalPrompts/is_prime.prompt | 16 ++++ .../HumanEvalPrompts/is_simple_power.prompt | 11 +++ benches/HumanEvalPrompts/is_sorted.prompt | 15 ++++ benches/HumanEvalPrompts/iscube.prompt | 12 +++ .../HumanEvalPrompts/largest_divisor.prompt | 4 + .../largest_prime_factor.prompt | 6 ++ .../largest_smallest_integers.prompt | 11 +++ benches/HumanEvalPrompts/longest.prompt | 9 ++ benches/HumanEvalPrompts/make_a_pile.prompt | 13 +++ benches/HumanEvalPrompts/match_parens.prompt | 14 +++ benches/HumanEvalPrompts/max_element.prompt | 6 ++ benches/HumanEvalPrompts/max_fill.prompt | 33 +++++++ benches/HumanEvalPrompts/maximum.prompt | 24 ++++++ .../mean_absolute_derivation.prompt | 8 ++ benches/HumanEvalPrompts/median.prompt | 6 ++ benches/HumanEvalPrompts/minPath.prompt | 29 +++++++ .../HumanEvalPrompts/minSubArraySum.prompt | 7 ++ benches/HumanEvalPrompts/modp.prompt | 12 +++ benches/HumanEvalPrompts/monotonic.prompt | 8 ++ benches/HumanEvalPrompts/move_one_ball.prompt | 26 ++++++ benches/HumanEvalPrompts/multiply.prompt | 9 ++ benches/HumanEvalPrompts/next_smallest.prompt | 10 +++ .../numerical_letter_grade.prompt | 24 ++++++ benches/HumanEvalPrompts/odd_count.prompt | 11 +++ .../HumanEvalPrompts/order_by_points.prompt | 10 +++ .../HumanEvalPrompts/pairs_sum_to_zero.prompt | 15 ++++ benches/HumanEvalPrompts/parse_music.prompt | 12 +++ .../parse_nested_parens.prompt | 7 ++ benches/HumanEvalPrompts/pluck.prompt | 33 +++++++ benches/HumanEvalPrompts/poly.prompt | 4 + benches/HumanEvalPrompts/prime_fib.prompt | 13 +++ benches/HumanEvalPrompts/prime_length.prompt | 8 ++ benches/HumanEvalPrompts/prod_signs.prompt | 11 +++ .../HumanEvalPrompts/remove_duplicates.prompt | 5 ++ benches/HumanEvalPrompts/remove_vowels.prompt | 15 ++++ .../HumanEvalPrompts/rescale_to_unit.prompt | 5 ++ .../HumanEvalPrompts/reverse_delete.prompt | 10 +++ .../right_angle_triangle.prompt | 9 ++ benches/HumanEvalPrompts/rolling_max.prompt | 5 ++ benches/HumanEvalPrompts/rounded_avg.prompt | 10 +++ benches/HumanEvalPrompts/same_chars.prompt | 15 ++++ benches/HumanEvalPrompts/search.prompt | 10 +++ benches/HumanEvalPrompts/select_words.prompt | 12 +++ .../separate_paren_groups.prompt | 7 ++ benches/HumanEvalPrompts/simplify.prompt | 11 +++ benches/HumanEvalPrompts/skjkasdkd.prompt | 11 +++ .../HumanEvalPrompts/smallest_change.prompt | 10 +++ benches/HumanEvalPrompts/solution.prompt | 8 ++ benches/HumanEvalPrompts/solve.prompt | 10 +++ benches/HumanEvalPrompts/sort_array.prompt | 10 +++ benches/HumanEvalPrompts/sort_even.prompt | 8 ++ benches/HumanEvalPrompts/sort_numbers.prompt | 6 ++ benches/HumanEvalPrompts/sort_third.prompt | 8 ++ .../HumanEvalPrompts/sorted_list_sum.prompt | 14 +++ benches/HumanEvalPrompts/specialFilter.prompt | 7 ++ .../HumanEvalPrompts/special_factorial.prompt | 11 +++ benches/HumanEvalPrompts/split_words.prompt | 9 ++ .../HumanEvalPrompts/starts_one_ends.prompt | 4 + .../HumanEvalPrompts/strange_sort_list.prompt | 10 +++ .../HumanEvalPrompts/string_sequence.prompt | 6 ++ benches/HumanEvalPrompts/string_to_md5.prompt | 6 ++ benches/HumanEvalPrompts/string_xor.prompt | 5 ++ benches/HumanEvalPrompts/strlen.prompt | 6 ++ benches/HumanEvalPrompts/sum_product.prompt | 7 ++ benches/HumanEvalPrompts/sum_squares.prompt | 10 +++ benches/HumanEvalPrompts/sum_to_n.prompt | 12 +++ benches/HumanEvalPrompts/total_match.prompt | 13 +++ benches/HumanEvalPrompts/tri.prompt | 16 ++++ benches/HumanEvalPrompts/triangle_area.prompt | 10 +++ .../triples_sum_to_zero.prompt | 16 ++++ benches/HumanEvalPrompts/unique.prompt | 4 + benches/HumanEvalPrompts/unique_digits.prompt | 11 +++ benches/HumanEvalPrompts/valid_date.prompt | 19 +++++ benches/HumanEvalPrompts/vowels_count.prompt | 3 + benches/HumanEvalPrompts/will_it_fly.prompt | 17 ++++ .../HumanEvalPrompts/words_in_sentence.prompt | 19 +++++ benches/HumanEvalPrompts/words_string.prompt | 8 ++ benches/HumanEvalPrompts/x_or_y.prompt | 8 ++ benches/extract_prompt.py | 18 ++++ gui/src/main.rs | 29 ++++++- prompts/dafny_by_prompt/ask_for_fixed.txt | 4 + .../ask_for_fixed_had_errors.txt | 4 + prompts/dafny_by_prompt/rewrite.txt | 6 ++ prompts/dafny_by_prompt/sys.txt | 33 +++++++ prompts/dafny_by_prompt/timeout.txt | 3 + verified_cogen/args.py | 2 +- verified_cogen/llm/llm.py | 27 ++++-- verified_cogen/main.py | 9 +- verified_cogen/runners/__init__.py | 11 ++- verified_cogen/runners/generate.py | 85 +++++++++++++++++++ 168 files changed, 1910 insertions(+), 20 deletions(-) create mode 100644 benches/HumanEvalPrompts/Strongest_Extension.prompt create mode 100644 benches/HumanEvalPrompts/add.prompt create mode 100644 benches/HumanEvalPrompts/add_elements.prompt create mode 100644 benches/HumanEvalPrompts/all_prefixes.prompt create mode 100644 benches/HumanEvalPrompts/anti_shuffle.prompt create mode 100644 benches/HumanEvalPrompts/any_int.prompt create mode 100644 benches/HumanEvalPrompts/below_threshold.prompt create mode 100644 benches/HumanEvalPrompts/below_zero.prompt create mode 100644 benches/HumanEvalPrompts/bf.prompt create mode 100644 benches/HumanEvalPrompts/by_length.prompt create mode 100644 benches/HumanEvalPrompts/can_arrange.prompt create mode 100644 benches/HumanEvalPrompts/car_race_collision.prompt create mode 100644 benches/HumanEvalPrompts/change_base.prompt create mode 100644 benches/HumanEvalPrompts/check_dict_case.prompt create mode 100644 benches/HumanEvalPrompts/check_if_last_char_is_a_letter.prompt create mode 100644 benches/HumanEvalPrompts/choose_num.prompt create mode 100644 benches/HumanEvalPrompts/circular_shift.prompt create mode 100644 benches/HumanEvalPrompts/closest_integer.prompt create mode 100644 benches/HumanEvalPrompts/common.prompt create mode 100644 benches/HumanEvalPrompts/compare.prompt create mode 100644 benches/HumanEvalPrompts/compare_one.prompt create mode 100644 benches/HumanEvalPrompts/concatenate.prompt create mode 100644 benches/HumanEvalPrompts/correct_bracketing.prompt create mode 100644 benches/HumanEvalPrompts/count_distinct_characters.prompt create mode 100644 benches/HumanEvalPrompts/count_nums.prompt create mode 100644 benches/HumanEvalPrompts/count_up_to.prompt create mode 100644 benches/HumanEvalPrompts/count_upper.prompt create mode 100644 benches/HumanEvalPrompts/cycpattern_check.prompt create mode 100644 benches/HumanEvalPrompts/decimal_to_binary.prompt create mode 100644 benches/HumanEvalPrompts/derivative.prompt create mode 100644 benches/HumanEvalPrompts/digitSum.prompt create mode 100644 benches/HumanEvalPrompts/digits.prompt create mode 100644 benches/HumanEvalPrompts/do_algebra.prompt create mode 100644 benches/HumanEvalPrompts/double_the_difference.prompt create mode 100644 benches/HumanEvalPrompts/eat.prompt create mode 100644 benches/HumanEvalPrompts/encode.prompt create mode 100644 benches/HumanEvalPrompts/encode_cyclic.prompt create mode 100644 benches/HumanEvalPrompts/encode_shift.prompt create mode 100644 benches/HumanEvalPrompts/encrypt.prompt create mode 100644 benches/HumanEvalPrompts/even_odd_count.prompt create mode 100644 benches/HumanEvalPrompts/even_odd_palindrome.prompt create mode 100644 benches/HumanEvalPrompts/exchange.prompt create mode 100644 benches/HumanEvalPrompts/f.prompt create mode 100644 benches/HumanEvalPrompts/factorize.prompt create mode 100644 benches/HumanEvalPrompts/fib.prompt create mode 100644 benches/HumanEvalPrompts/fib4.prompt create mode 100644 benches/HumanEvalPrompts/fibfib.prompt create mode 100644 benches/HumanEvalPrompts/file_name_check.prompt create mode 100644 benches/HumanEvalPrompts/filter_by_prefix.prompt create mode 100644 benches/HumanEvalPrompts/filter_by_substring.prompt create mode 100644 benches/HumanEvalPrompts/filter_integers.prompt create mode 100644 benches/HumanEvalPrompts/find_closest_elements.prompt create mode 100644 benches/HumanEvalPrompts/find_max.prompt create mode 100644 benches/HumanEvalPrompts/fix_spaces.prompt create mode 100644 benches/HumanEvalPrompts/fizz_buzz.prompt create mode 100644 benches/HumanEvalPrompts/flip_case.prompt create mode 100644 benches/HumanEvalPrompts/fruit_distribution.prompt create mode 100644 benches/HumanEvalPrompts/generate_integers.prompt create mode 100644 benches/HumanEvalPrompts/get_closest_vowel.prompt create mode 100644 benches/HumanEvalPrompts/get_max_triples.prompt create mode 100644 benches/HumanEvalPrompts/get_odd_collatz.prompt create mode 100644 benches/HumanEvalPrompts/get_positive.prompt create mode 100644 benches/HumanEvalPrompts/get_row.prompt create mode 100644 benches/HumanEvalPrompts/greatest_common_divisor.prompt create mode 100644 benches/HumanEvalPrompts/has_close_elements.prompt create mode 100644 benches/HumanEvalPrompts/hex_key.prompt create mode 100644 benches/HumanEvalPrompts/histogram.prompt create mode 100644 benches/HumanEvalPrompts/how_many_times.prompt create mode 100644 benches/HumanEvalPrompts/incr_list.prompt create mode 100644 benches/HumanEvalPrompts/int_to_mini_roman.prompt create mode 100644 benches/HumanEvalPrompts/intersection.prompt create mode 100644 benches/HumanEvalPrompts/intersperse.prompt create mode 100644 benches/HumanEvalPrompts/is_bored.prompt create mode 100644 benches/HumanEvalPrompts/is_equal_to_sum_even.prompt create mode 100644 benches/HumanEvalPrompts/is_happy.prompt create mode 100644 benches/HumanEvalPrompts/is_multiply_prime.prompt create mode 100644 benches/HumanEvalPrompts/is_nested.prompt create mode 100644 benches/HumanEvalPrompts/is_palindrome.prompt create mode 100644 benches/HumanEvalPrompts/is_prime.prompt create mode 100644 benches/HumanEvalPrompts/is_simple_power.prompt create mode 100644 benches/HumanEvalPrompts/is_sorted.prompt create mode 100644 benches/HumanEvalPrompts/iscube.prompt create mode 100644 benches/HumanEvalPrompts/largest_divisor.prompt create mode 100644 benches/HumanEvalPrompts/largest_prime_factor.prompt create mode 100644 benches/HumanEvalPrompts/largest_smallest_integers.prompt create mode 100644 benches/HumanEvalPrompts/longest.prompt create mode 100644 benches/HumanEvalPrompts/make_a_pile.prompt create mode 100644 benches/HumanEvalPrompts/match_parens.prompt create mode 100644 benches/HumanEvalPrompts/max_element.prompt create mode 100644 benches/HumanEvalPrompts/max_fill.prompt create mode 100644 benches/HumanEvalPrompts/maximum.prompt create mode 100644 benches/HumanEvalPrompts/mean_absolute_derivation.prompt create mode 100644 benches/HumanEvalPrompts/median.prompt create mode 100644 benches/HumanEvalPrompts/minPath.prompt create mode 100644 benches/HumanEvalPrompts/minSubArraySum.prompt create mode 100644 benches/HumanEvalPrompts/modp.prompt create mode 100644 benches/HumanEvalPrompts/monotonic.prompt create mode 100644 benches/HumanEvalPrompts/move_one_ball.prompt create mode 100644 benches/HumanEvalPrompts/multiply.prompt create mode 100644 benches/HumanEvalPrompts/next_smallest.prompt create mode 100644 benches/HumanEvalPrompts/numerical_letter_grade.prompt create mode 100644 benches/HumanEvalPrompts/odd_count.prompt create mode 100644 benches/HumanEvalPrompts/order_by_points.prompt create mode 100644 benches/HumanEvalPrompts/pairs_sum_to_zero.prompt create mode 100644 benches/HumanEvalPrompts/parse_music.prompt create mode 100644 benches/HumanEvalPrompts/parse_nested_parens.prompt create mode 100644 benches/HumanEvalPrompts/pluck.prompt create mode 100644 benches/HumanEvalPrompts/poly.prompt create mode 100644 benches/HumanEvalPrompts/prime_fib.prompt create mode 100644 benches/HumanEvalPrompts/prime_length.prompt create mode 100644 benches/HumanEvalPrompts/prod_signs.prompt create mode 100644 benches/HumanEvalPrompts/remove_duplicates.prompt create mode 100644 benches/HumanEvalPrompts/remove_vowels.prompt create mode 100644 benches/HumanEvalPrompts/rescale_to_unit.prompt create mode 100644 benches/HumanEvalPrompts/reverse_delete.prompt create mode 100644 benches/HumanEvalPrompts/right_angle_triangle.prompt create mode 100644 benches/HumanEvalPrompts/rolling_max.prompt create mode 100644 benches/HumanEvalPrompts/rounded_avg.prompt create mode 100644 benches/HumanEvalPrompts/same_chars.prompt create mode 100644 benches/HumanEvalPrompts/search.prompt create mode 100644 benches/HumanEvalPrompts/select_words.prompt create mode 100644 benches/HumanEvalPrompts/separate_paren_groups.prompt create mode 100644 benches/HumanEvalPrompts/simplify.prompt create mode 100644 benches/HumanEvalPrompts/skjkasdkd.prompt create mode 100644 benches/HumanEvalPrompts/smallest_change.prompt create mode 100644 benches/HumanEvalPrompts/solution.prompt create mode 100644 benches/HumanEvalPrompts/solve.prompt create mode 100644 benches/HumanEvalPrompts/sort_array.prompt create mode 100644 benches/HumanEvalPrompts/sort_even.prompt create mode 100644 benches/HumanEvalPrompts/sort_numbers.prompt create mode 100644 benches/HumanEvalPrompts/sort_third.prompt create mode 100644 benches/HumanEvalPrompts/sorted_list_sum.prompt create mode 100644 benches/HumanEvalPrompts/specialFilter.prompt create mode 100644 benches/HumanEvalPrompts/special_factorial.prompt create mode 100644 benches/HumanEvalPrompts/split_words.prompt create mode 100644 benches/HumanEvalPrompts/starts_one_ends.prompt create mode 100644 benches/HumanEvalPrompts/strange_sort_list.prompt create mode 100644 benches/HumanEvalPrompts/string_sequence.prompt create mode 100644 benches/HumanEvalPrompts/string_to_md5.prompt create mode 100644 benches/HumanEvalPrompts/string_xor.prompt create mode 100644 benches/HumanEvalPrompts/strlen.prompt create mode 100644 benches/HumanEvalPrompts/sum_product.prompt create mode 100644 benches/HumanEvalPrompts/sum_squares.prompt create mode 100644 benches/HumanEvalPrompts/sum_to_n.prompt create mode 100644 benches/HumanEvalPrompts/total_match.prompt create mode 100644 benches/HumanEvalPrompts/tri.prompt create mode 100644 benches/HumanEvalPrompts/triangle_area.prompt create mode 100644 benches/HumanEvalPrompts/triples_sum_to_zero.prompt create mode 100644 benches/HumanEvalPrompts/unique.prompt create mode 100644 benches/HumanEvalPrompts/unique_digits.prompt create mode 100644 benches/HumanEvalPrompts/valid_date.prompt create mode 100644 benches/HumanEvalPrompts/vowels_count.prompt create mode 100644 benches/HumanEvalPrompts/will_it_fly.prompt create mode 100644 benches/HumanEvalPrompts/words_in_sentence.prompt create mode 100644 benches/HumanEvalPrompts/words_string.prompt create mode 100644 benches/HumanEvalPrompts/x_or_y.prompt create mode 100755 benches/extract_prompt.py create mode 100644 prompts/dafny_by_prompt/ask_for_fixed.txt create mode 100644 prompts/dafny_by_prompt/ask_for_fixed_had_errors.txt create mode 100644 prompts/dafny_by_prompt/rewrite.txt create mode 100644 prompts/dafny_by_prompt/sys.txt create mode 100644 prompts/dafny_by_prompt/timeout.txt create mode 100644 verified_cogen/runners/generate.py diff --git a/benches/HumanEvalPrompts/Strongest_Extension.prompt b/benches/HumanEvalPrompts/Strongest_Extension.prompt new file mode 100644 index 0000000..f4eec2c --- /dev/null +++ b/benches/HumanEvalPrompts/Strongest_Extension.prompt @@ -0,0 +1,16 @@ +You will be given the name of a class (a string) and a list of extensions. +The extensions are to be used to load additional classes to the class. The +strength of the extension is as follows: Let CAP be the number of the uppercase +letters in the extension's name, and let SM be the number of lowercase letters +in the extension's name, the strength is given by the fraction CAP - SM. +You should find the strongest extension and return a string in this +format: ClassName.StrongestExtensionName. +If there are two or more extensions with the same strength, you should +choose the one that comes first in the list. +For example, if you are given "Slices" as the class and a list of the +extensions: ['SErviNGSliCes', 'Cheese', 'StuFfed'] then you should +return 'Slices.SErviNGSliCes' since 'SErviNGSliCes' is the strongest extension +(its strength is -1). +Example: +for Strongest_Extension('my_class', ['AA', 'Be', 'CC']) == 'my_class.AA' + diff --git a/benches/HumanEvalPrompts/add.prompt b/benches/HumanEvalPrompts/add.prompt new file mode 100644 index 0000000..d286229 --- /dev/null +++ b/benches/HumanEvalPrompts/add.prompt @@ -0,0 +1,6 @@ +Given a non-empty list of integers lst. add the even elements that are at odd indices.. + + +Examples: + add([4, 2, 6, 7]) ==> 2 + diff --git a/benches/HumanEvalPrompts/add_elements.prompt b/benches/HumanEvalPrompts/add_elements.prompt new file mode 100644 index 0000000..e004868 --- /dev/null +++ b/benches/HumanEvalPrompts/add_elements.prompt @@ -0,0 +1,13 @@ + +Given a non-empty array of integers arr and an integer k, return +the sum of the elements with at most two digits from the first k elements of arr. + +Example: + + Input: arr = [111,21,3,4000,5,6,7,8,9], k = 4 + Output: 24 # sum of 21 + 3 + +Constraints: + 1. 1 <= len(arr) <= 100 + 2. 1 <= k <= len(arr) + diff --git a/benches/HumanEvalPrompts/all_prefixes.prompt b/benches/HumanEvalPrompts/all_prefixes.prompt new file mode 100644 index 0000000..696e484 --- /dev/null +++ b/benches/HumanEvalPrompts/all_prefixes.prompt @@ -0,0 +1,4 @@ + Return list of all prefixes from shortest to longest of the input string +>>> all_prefixes('abc') +['a', 'ab', 'abc'] + diff --git a/benches/HumanEvalPrompts/anti_shuffle.prompt b/benches/HumanEvalPrompts/anti_shuffle.prompt new file mode 100644 index 0000000..fb91de8 --- /dev/null +++ b/benches/HumanEvalPrompts/anti_shuffle.prompt @@ -0,0 +1,12 @@ + +Write a function that takes a string and returns an ordered version of it. +Ordered version of string, is a string where all words (separated by space) +are replaced by a new word where all the characters arranged in +ascending order based on ascii value. +Note: You should keep the order of words and blank spaces in the sentence. + +For example: +anti_shuffle('Hi') returns 'Hi' +anti_shuffle('hello') returns 'ehllo' +anti_shuffle('Hello World!!!') returns 'Hello !!!Wdlor' + diff --git a/benches/HumanEvalPrompts/any_int.prompt b/benches/HumanEvalPrompts/any_int.prompt new file mode 100644 index 0000000..5fa1b35 --- /dev/null +++ b/benches/HumanEvalPrompts/any_int.prompt @@ -0,0 +1,17 @@ + +Create a function that takes 3 numbers. +Returns true if one of the numbers is equal to the sum of the other two, and all numbers are integers. +Returns false in any other cases. + +Examples +any_int(5, 2, 7) ➞ True + +any_int(3, 2, 2) ➞ False + +any_int(3, -2, 1) ➞ True + +any_int(3.6, -2.2, 2) ➞ False + + + + diff --git a/benches/HumanEvalPrompts/below_threshold.prompt b/benches/HumanEvalPrompts/below_threshold.prompt new file mode 100644 index 0000000..3f87584 --- /dev/null +++ b/benches/HumanEvalPrompts/below_threshold.prompt @@ -0,0 +1,6 @@ +Return True if all numbers in the list l are below threshold t. +>>> below_threshold([1, 2, 4, 10], 100) +True +>>> below_threshold([1, 20, 4, 10], 5) +False + diff --git a/benches/HumanEvalPrompts/below_zero.prompt b/benches/HumanEvalPrompts/below_zero.prompt new file mode 100644 index 0000000..a395267 --- /dev/null +++ b/benches/HumanEvalPrompts/below_zero.prompt @@ -0,0 +1,8 @@ + You're given a list of deposit and withdrawal operations on a bank account that starts with +zero balance. Your task is to detect if at any point the balance of account fallls below zero, and +at that point function should return True. Otherwise it should return False. +>>> below_zero([1, 2, 3]) +False +>>> below_zero([1, 2, -4, 5]) +True + diff --git a/benches/HumanEvalPrompts/bf.prompt b/benches/HumanEvalPrompts/bf.prompt new file mode 100644 index 0000000..cb3a20d --- /dev/null +++ b/benches/HumanEvalPrompts/bf.prompt @@ -0,0 +1,15 @@ + +There are eight planets in our solar system: the closerst to the Sun +is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, +Uranus, Neptune. +Write a function that takes two planet names as strings planet1 and planet2. +The function should return a tuple containing all planets whose orbits are +located between the orbit of planet1 and the orbit of planet2, sorted by +the proximity to the sun. +The function should return an empty tuple if planet1 or planet2 +are not correct planet names. +Examples +bf("Jupiter", "Neptune") ==> ("Saturn", "Uranus") +bf("Earth", "Mercury") ==> ("Venus") +bf("Mercury", "Uranus") ==> ("Venus", "Earth", "Mars", "Jupiter", "Saturn") + diff --git a/benches/HumanEvalPrompts/by_length.prompt b/benches/HumanEvalPrompts/by_length.prompt new file mode 100644 index 0000000..7cb42c1 --- /dev/null +++ b/benches/HumanEvalPrompts/by_length.prompt @@ -0,0 +1,21 @@ + +Given an array of integers, sort the integers that are between 1 and 9 inclusive, +reverse the resulting array, and then replace each digit by its corresponding name from +"One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine". + +For example: + arr = [2, 1, 1, 4, 5, 8, 2, 3] + -> sort arr -> [1, 1, 2, 2, 3, 4, 5, 8] + -> reverse arr -> [8, 5, 4, 3, 2, 2, 1, 1] + return ["Eight", "Five", "Four", "Three", "Two", "Two", "One", "One"] + + If the array is empty, return an empty array: + arr = [] + return [] + + If the array has any strange number ignore it: + arr = [1, -1 , 55] + -> sort arr -> [-1, 1, 55] + -> reverse arr -> [55, 1, -1] + return = ['One'] + diff --git a/benches/HumanEvalPrompts/can_arrange.prompt b/benches/HumanEvalPrompts/can_arrange.prompt new file mode 100644 index 0000000..9310a75 --- /dev/null +++ b/benches/HumanEvalPrompts/can_arrange.prompt @@ -0,0 +1,9 @@ +Create a function which returns the largest index of an element which +is not greater than or equal to the element immediately preceding it. If +no such element exists then return -1. The given array will not contain +duplicate values. + +Examples: +can_arrange([1,2,4,3,5]) = 3 +can_arrange([1,2,3]) = -1 + diff --git a/benches/HumanEvalPrompts/car_race_collision.prompt b/benches/HumanEvalPrompts/car_race_collision.prompt new file mode 100644 index 0000000..2449984 --- /dev/null +++ b/benches/HumanEvalPrompts/car_race_collision.prompt @@ -0,0 +1,11 @@ + +Imagine a road that's a perfectly straight infinitely long line. +n cars are driving left to right; simultaneously, a different set of n cars +are driving right to left. The two sets of cars start out being very far from +each other. All cars move in the same speed. Two cars are said to collide +when a car that's moving left to right hits a car that's moving right to left. +However, the cars are infinitely sturdy and strong; as a result, they continue moving +in their trajectory as if they did not collide. + +This function outputs the number of such collisions. + diff --git a/benches/HumanEvalPrompts/change_base.prompt b/benches/HumanEvalPrompts/change_base.prompt new file mode 100644 index 0000000..0123c33 --- /dev/null +++ b/benches/HumanEvalPrompts/change_base.prompt @@ -0,0 +1,10 @@ +Change numerical base of input number x to base. +return string representation after the conversion. +base numbers are less than 10. +>>> change_base(8, 3) +'22' +>>> change_base(8, 2) +'1000' +>>> change_base(7, 2) +'111' + diff --git a/benches/HumanEvalPrompts/check_dict_case.prompt b/benches/HumanEvalPrompts/check_dict_case.prompt new file mode 100644 index 0000000..6025775 --- /dev/null +++ b/benches/HumanEvalPrompts/check_dict_case.prompt @@ -0,0 +1,11 @@ + +Given a dictionary, return True if all keys are strings in lower +case or all keys are strings in upper case, else return False. +The function should return False is the given dictionary is empty. +Examples: +check_dict_case({"a":"apple", "b":"banana"}) should return True. +check_dict_case({"a":"apple", "A":"banana", "B":"banana"}) should return False. +check_dict_case({"a":"apple", 8:"banana", "a":"apple"}) should return False. +check_dict_case({"Name":"John", "Age":"36", "City":"Houston"}) should return False. +check_dict_case({"STATE":"NC", "ZIP":"12345" }) should return True. + diff --git a/benches/HumanEvalPrompts/check_if_last_char_is_a_letter.prompt b/benches/HumanEvalPrompts/check_if_last_char_is_a_letter.prompt new file mode 100644 index 0000000..ffeacdb --- /dev/null +++ b/benches/HumanEvalPrompts/check_if_last_char_is_a_letter.prompt @@ -0,0 +1,12 @@ + +Create a function that returns True if the last character +of a given string is an alphabetical character and is not +a part of a word, and False otherwise. +Note: "word" is a group of characters separated by space. + +Examples: +check_if_last_char_is_a_letter("apple pie") ➞ False +check_if_last_char_is_a_letter("apple pi e") ➞ True +check_if_last_char_is_a_letter("apple pi e ") ➞ False +check_if_last_char_is_a_letter("") ➞ False + diff --git a/benches/HumanEvalPrompts/choose_num.prompt b/benches/HumanEvalPrompts/choose_num.prompt new file mode 100644 index 0000000..410c852 --- /dev/null +++ b/benches/HumanEvalPrompts/choose_num.prompt @@ -0,0 +1,8 @@ +This function takes two positive numbers x and y and returns the +biggest even integer number that is in the range [x, y] inclusive. If +there's no such number, then the function should return -1. + +For example: +choose_num(12, 15) = 14 +choose_num(13, 12) = -1 + diff --git a/benches/HumanEvalPrompts/circular_shift.prompt b/benches/HumanEvalPrompts/circular_shift.prompt new file mode 100644 index 0000000..ee1fd48 --- /dev/null +++ b/benches/HumanEvalPrompts/circular_shift.prompt @@ -0,0 +1,8 @@ +Circular shift the digits of the integer x, shift the digits right by shift +and return the result as a string. +If shift > number of digits, return digits reversed. +>>> circular_shift(12, 1) +"21" +>>> circular_shift(12, 2) +"12" + diff --git a/benches/HumanEvalPrompts/closest_integer.prompt b/benches/HumanEvalPrompts/closest_integer.prompt new file mode 100644 index 0000000..283c1e7 --- /dev/null +++ b/benches/HumanEvalPrompts/closest_integer.prompt @@ -0,0 +1,17 @@ + +Create a function that takes a value (string) representing a number +and returns the closest integer to it. If the number is equidistant +from two integers, round it away from zero. + +Examples +>>> closest_integer("10") +10 +>>> closest_integer("15.3") +15 + +Note: +Rounding away from zero means that if the given number is equidistant +from two integers, the one you should return is the one that is the +farthest from zero. For example closest_integer("14.5") should +return 15 and closest_integer("-14.5") should return -15. + diff --git a/benches/HumanEvalPrompts/common.prompt b/benches/HumanEvalPrompts/common.prompt new file mode 100644 index 0000000..50ff19e --- /dev/null +++ b/benches/HumanEvalPrompts/common.prompt @@ -0,0 +1,7 @@ +Return sorted unique common elements for two lists. +>>> common([1, 4, 3, 34, 653, 2, 5], [5, 7, 1, 5, 9, 653, 121]) +[1, 5, 653] +>>> common([5, 3, 2, 8], [3, 2]) +[2, 3] + + diff --git a/benches/HumanEvalPrompts/compare.prompt b/benches/HumanEvalPrompts/compare.prompt new file mode 100644 index 0000000..489df9a --- /dev/null +++ b/benches/HumanEvalPrompts/compare.prompt @@ -0,0 +1,14 @@ +I think we all remember that feeling when the result of some long-awaited +event is finally known. The feelings and thoughts you have at that moment are +definitely worth noting down and comparing. +Your task is to determine if a person correctly guessed the results of a number of matches. +You are given two arrays of scores and guesses of equal length, where each index shows a match. +Return an array of the same length denoting how far off each guess was. If they have guessed correctly, +the value is 0, and if not, the value is the absolute difference between the guess and the score. + + +example: + +compare([1,2,3,4,5,1],[1,2,3,4,2,-2]) -> [0,0,0,0,3,3] +compare([0,5,0,0,0,4],[4,1,1,0,0,-2]) -> [4,4,1,0,0,6] + diff --git a/benches/HumanEvalPrompts/compare_one.prompt b/benches/HumanEvalPrompts/compare_one.prompt new file mode 100644 index 0000000..770729c --- /dev/null +++ b/benches/HumanEvalPrompts/compare_one.prompt @@ -0,0 +1,11 @@ + +Create a function that takes integers, floats, or strings representing +real numbers, and returns the larger variable in its given variable type. +Return None if the values are equal. +Note: If a real number is represented as a string, the floating point might be . or , + +compare_one(1, 2.5) ➞ 2.5 +compare_one(1, "2,3") ➞ "2,3" +compare_one("5,1", "6") ➞ "6" +compare_one("1", 1) ➞ None + diff --git a/benches/HumanEvalPrompts/concatenate.prompt b/benches/HumanEvalPrompts/concatenate.prompt new file mode 100644 index 0000000..04289ba --- /dev/null +++ b/benches/HumanEvalPrompts/concatenate.prompt @@ -0,0 +1,6 @@ + Concatenate list of strings into a single string +>>> concatenate([]) +'' +>>> concatenate(['a', 'b', 'c']) +'abc' + diff --git a/benches/HumanEvalPrompts/correct_bracketing.prompt b/benches/HumanEvalPrompts/correct_bracketing.prompt new file mode 100644 index 0000000..00a592d --- /dev/null +++ b/benches/HumanEvalPrompts/correct_bracketing.prompt @@ -0,0 +1,12 @@ + brackets is a string of "(" and ")". +return True if every opening bracket has a corresponding closing bracket. + +>>> correct_bracketing("(") +False +>>> correct_bracketing("()") +True +>>> correct_bracketing("(()())") +True +>>> correct_bracketing(")(()") +False + diff --git a/benches/HumanEvalPrompts/count_distinct_characters.prompt b/benches/HumanEvalPrompts/count_distinct_characters.prompt new file mode 100644 index 0000000..b8fc7f2 --- /dev/null +++ b/benches/HumanEvalPrompts/count_distinct_characters.prompt @@ -0,0 +1,6 @@ + Given a string, find out how many distinct characters (regardless of case) does it consist of +>>> count_distinct_characters('xyzXYZ') +3 +>>> count_distinct_characters('Jerry') +4 + diff --git a/benches/HumanEvalPrompts/count_nums.prompt b/benches/HumanEvalPrompts/count_nums.prompt new file mode 100644 index 0000000..2fa8bb1 --- /dev/null +++ b/benches/HumanEvalPrompts/count_nums.prompt @@ -0,0 +1,9 @@ + +Write a function count_nums which takes an array of integers and returns +the number of elements which has a sum of digits > 0. +If a number is negative, then its first signed digit will be negative: +e.g. -123 has signed digits -1, 2, and 3. +>>> count_nums([]) == 0 +>>> count_nums([-1, 11, -11]) == 1 +>>> count_nums([1, 1, 2]) == 3 + diff --git a/benches/HumanEvalPrompts/count_up_to.prompt b/benches/HumanEvalPrompts/count_up_to.prompt new file mode 100644 index 0000000..49c9b06 --- /dev/null +++ b/benches/HumanEvalPrompts/count_up_to.prompt @@ -0,0 +1,10 @@ +Implement a function that takes an non-negative integer and returns an array of the first n +integers that are prime numbers and less than n. +for example: +count_up_to(5) => [2,3] +count_up_to(11) => [2,3,5,7] +count_up_to(0) => [] +count_up_to(20) => [2,3,5,7,11,13,17,19] +count_up_to(1) => [] +count_up_to(18) => [2,3,5,7,11,13,17] + diff --git a/benches/HumanEvalPrompts/count_upper.prompt b/benches/HumanEvalPrompts/count_upper.prompt new file mode 100644 index 0000000..d1b5b8a --- /dev/null +++ b/benches/HumanEvalPrompts/count_upper.prompt @@ -0,0 +1,8 @@ + +Given a string s, count the number of uppercase vowels in even indices. + +For example: +count_upper('aBCdEf') returns 1 +count_upper('abcdefg') returns 0 +count_upper('dBBE') returns 0 + diff --git a/benches/HumanEvalPrompts/cycpattern_check.prompt b/benches/HumanEvalPrompts/cycpattern_check.prompt new file mode 100644 index 0000000..79872a6 --- /dev/null +++ b/benches/HumanEvalPrompts/cycpattern_check.prompt @@ -0,0 +1,9 @@ +You are given 2 words. You need to return True if the second word or any of its rotations is a substring in the first word +cycpattern_check("abcd","abd") => False +cycpattern_check("hello","ell") => True +cycpattern_check("whassup","psus") => False +cycpattern_check("abab","baa") => True +cycpattern_check("efef","eeff") => False +cycpattern_check("himenss","simen") => True + + diff --git a/benches/HumanEvalPrompts/decimal_to_binary.prompt b/benches/HumanEvalPrompts/decimal_to_binary.prompt new file mode 100644 index 0000000..862be04 --- /dev/null +++ b/benches/HumanEvalPrompts/decimal_to_binary.prompt @@ -0,0 +1,11 @@ +You will be given a number in decimal form and your task is to convert it to +binary format. The function should return a string, with each character representing a binary +number. Each character in the string will be '0' or '1'. + +There will be an extra couple of characters 'db' at the beginning and at the end of the string. +The extra characters are there to help with the format. + +Examples: +decimal_to_binary(15) # returns "db1111db" +decimal_to_binary(32) # returns "db100000db" + diff --git a/benches/HumanEvalPrompts/derivative.prompt b/benches/HumanEvalPrompts/derivative.prompt new file mode 100644 index 0000000..7a45082 --- /dev/null +++ b/benches/HumanEvalPrompts/derivative.prompt @@ -0,0 +1,8 @@ + xs represent coefficients of a polynomial. +xs[0] + xs[1] * x + xs[2] * x^2 + .... + Return derivative of this polynomial in the same form. +>>> derivative([3, 1, 2, 4, 5]) +[1, 4, 12, 20] +>>> derivative([1, 2, 3]) +[2, 6] + diff --git a/benches/HumanEvalPrompts/digitSum.prompt b/benches/HumanEvalPrompts/digitSum.prompt new file mode 100644 index 0000000..7de4b06 --- /dev/null +++ b/benches/HumanEvalPrompts/digitSum.prompt @@ -0,0 +1,12 @@ +Task +Write a function that takes a string as input and returns the sum of the upper characters only' +ASCII codes. + +Examples: + digitSum("") => 0 + digitSum("abAB") => 131 + digitSum("abcCd") => 67 + digitSum("helloE") => 69 + digitSum("woArBld") => 131 + digitSum("aAaaaXa") => 153 + diff --git a/benches/HumanEvalPrompts/digits.prompt b/benches/HumanEvalPrompts/digits.prompt new file mode 100644 index 0000000..64aea34 --- /dev/null +++ b/benches/HumanEvalPrompts/digits.prompt @@ -0,0 +1,7 @@ +Given a positive integer n, return the product of the odd digits. +Return 0 if all digits are even. +For example: +digits(1) == 1 +digits(4) == 0 +digits(235) == 15 + diff --git a/benches/HumanEvalPrompts/do_algebra.prompt b/benches/HumanEvalPrompts/do_algebra.prompt new file mode 100644 index 0000000..1818166 --- /dev/null +++ b/benches/HumanEvalPrompts/do_algebra.prompt @@ -0,0 +1,24 @@ + +Given two lists operator, and operand. The first list has basic algebra operations, and +the second list is a list of integers. Use the two given lists to build the algebric +expression and return the evaluation of this expression. + +The basic algebra operations: +Addition ( + ) +Subtraction ( - ) +Multiplication ( * ) +Floor division ( // ) +Exponentiation ( ** ) + +Example: +operator['+', '*', '-'] +array = [2, 3, 4, 5] +result = 2 + 3 * 4 - 5 +=> result = 9 + +Note: + The length of operator list is equal to the length of operand list minus one. + Operand is a list of of non-negative integers. + Operator list has at least one operator, and operand list has at least two operands. + + diff --git a/benches/HumanEvalPrompts/double_the_difference.prompt b/benches/HumanEvalPrompts/double_the_difference.prompt new file mode 100644 index 0000000..982a406 --- /dev/null +++ b/benches/HumanEvalPrompts/double_the_difference.prompt @@ -0,0 +1,11 @@ + +Given a list of numbers, return the sum of squares of the numbers +in the list that are odd. Ignore numbers that are negative or not integers. + +double_the_difference([1, 3, 2, 0]) == 1 + 9 + 0 + 0 = 10 +double_the_difference([-1, -2, 0]) == 0 +double_the_difference([9, -2]) == 81 +double_the_difference([0]) == 0 + +If the input list is empty, return 0. + diff --git a/benches/HumanEvalPrompts/eat.prompt b/benches/HumanEvalPrompts/eat.prompt new file mode 100644 index 0000000..8797b6d --- /dev/null +++ b/benches/HumanEvalPrompts/eat.prompt @@ -0,0 +1,28 @@ + +You're a hungry rabbit, and you already have eaten a certain number of carrots, +but now you need to eat more carrots to complete the day's meals. +you should return an array of [ total number of eaten carrots after your meals, + the number of carrots left after your meals ] +if there are not enough remaining carrots, you will eat all remaining carrots, but will still be hungry. + +Example: +* eat(5, 6, 10) -> [11, 4] +* eat(4, 8, 9) -> [12, 1] +* eat(1, 10, 10) -> [11, 0] +* eat(2, 11, 5) -> [7, 0] + +Variables: +@number : integer + the number of carrots that you have eaten. +@need : integer + the number of carrots that you need to eat. +@remaining : integer + the number of remaining carrots thet exist in stock + +Constrain: +* 0 <= number <= 1000 +* 0 <= need <= 1000 +* 0 <= remaining <= 1000 + +Have fun :) + diff --git a/benches/HumanEvalPrompts/encode.prompt b/benches/HumanEvalPrompts/encode.prompt new file mode 100644 index 0000000..d5b45de --- /dev/null +++ b/benches/HumanEvalPrompts/encode.prompt @@ -0,0 +1,13 @@ + +Write a function that takes a message, and encodes in such a +way that it swaps case of all letters, replaces all vowels in +the message with the letter that appears 2 places ahead of that +vowel in the english alphabet. +Assume only letters. + +Examples: +>>> encode('test') +'TGST' +>>> encode('This is a message') +'tHKS KS C MGSSCGG' + diff --git a/benches/HumanEvalPrompts/encode_cyclic.prompt b/benches/HumanEvalPrompts/encode_cyclic.prompt new file mode 100644 index 0000000..d654dcb --- /dev/null +++ b/benches/HumanEvalPrompts/encode_cyclic.prompt @@ -0,0 +1,3 @@ + +returns encoded string by cycling groups of three characters. + diff --git a/benches/HumanEvalPrompts/encode_shift.prompt b/benches/HumanEvalPrompts/encode_shift.prompt new file mode 100644 index 0000000..abebf47 --- /dev/null +++ b/benches/HumanEvalPrompts/encode_shift.prompt @@ -0,0 +1,3 @@ + +returns encoded string by shifting every character by 5 in the alphabet. + diff --git a/benches/HumanEvalPrompts/encrypt.prompt b/benches/HumanEvalPrompts/encrypt.prompt new file mode 100644 index 0000000..438bf7d --- /dev/null +++ b/benches/HumanEvalPrompts/encrypt.prompt @@ -0,0 +1,10 @@ +Create a function encrypt that takes a string as an argument and +returns a string encrypted with the alphabet being rotated. +The alphabet should be rotated in a manner such that the letters +shift down by two multiplied to two places. +For example: +encrypt('hi') returns 'lm' +encrypt('asdfghjkl') returns 'ewhjklnop' +encrypt('gf') returns 'kj' +encrypt('et') returns 'ix' + diff --git a/benches/HumanEvalPrompts/even_odd_count.prompt b/benches/HumanEvalPrompts/even_odd_count.prompt new file mode 100644 index 0000000..2f5c9f1 --- /dev/null +++ b/benches/HumanEvalPrompts/even_odd_count.prompt @@ -0,0 +1,6 @@ +Given an integer. return a tuple that has the number of even and odd digits respectively. + + Example: + even_odd_count(-12) ==> (1, 1) + even_odd_count(123) ==> (1, 2) + diff --git a/benches/HumanEvalPrompts/even_odd_palindrome.prompt b/benches/HumanEvalPrompts/even_odd_palindrome.prompt new file mode 100644 index 0000000..7b25f99 --- /dev/null +++ b/benches/HumanEvalPrompts/even_odd_palindrome.prompt @@ -0,0 +1,22 @@ + +Given a positive integer n, return a tuple that has the number of even and odd +integer palindromes that fall within the range(1, n), inclusive. + +Example 1: + + Input: 3 + Output: (1, 2) + Explanation: + Integer palindrome are 1, 2, 3. one of them is even, and two of them are odd. + +Example 2: + + Input: 12 + Output: (4, 6) + Explanation: + Integer palindrome are 1, 2, 3, 4, 5, 6, 7, 8, 9, 11. four of them are even, and 6 of them are odd. + +Note: + 1. 1 <= n <= 10^3 + 2. returned tuple has the number of even and odd integer palindromes respectively. + diff --git a/benches/HumanEvalPrompts/exchange.prompt b/benches/HumanEvalPrompts/exchange.prompt new file mode 100644 index 0000000..f8c3f06 --- /dev/null +++ b/benches/HumanEvalPrompts/exchange.prompt @@ -0,0 +1,12 @@ +In this problem, you will implement a function that takes two lists of numbers, +and determines whether it is possible to perform an exchange of elements +between them to make lst1 a list of only even numbers. +There is no limit on the number of exchanged elements between lst1 and lst2. +If it is possible to exchange elements between the lst1 and lst2 to make +all the elements of lst1 to be even, return "YES". +Otherwise, return "NO". +For example: +exchange([1, 2, 3, 4], [1, 2, 3, 4]) => "YES" +exchange([1, 2, 3, 4], [1, 5, 3, 4]) => "NO" +It is assumed that the input lists will be non-empty. + diff --git a/benches/HumanEvalPrompts/f.prompt b/benches/HumanEvalPrompts/f.prompt new file mode 100644 index 0000000..c0d3a03 --- /dev/null +++ b/benches/HumanEvalPrompts/f.prompt @@ -0,0 +1,8 @@ + Implement the function f that takes n as a parameter, +and returns a list of size n, such that the value of the element at index i is the factorial of i if i is even +or the sum of numbers from 1 to i otherwise. +i starts from 1. +the factorial of i is the multiplication of the numbers from 1 to i (1 * 2 * ... * i). +Example: +f(5) == [1, 2, 6, 24, 15] + diff --git a/benches/HumanEvalPrompts/factorize.prompt b/benches/HumanEvalPrompts/factorize.prompt new file mode 100644 index 0000000..c78de3f --- /dev/null +++ b/benches/HumanEvalPrompts/factorize.prompt @@ -0,0 +1,10 @@ + Return list of prime factors of given integer in the order from smallest to largest. +Each of the factors should be listed number of times corresponding to how many times it appeares in factorization. +Input number should be equal to the product of all factors +>>> factorize(8) +[2, 2, 2] +>>> factorize(25) +[5, 5] +>>> factorize(70) +[2, 5, 7] + diff --git a/benches/HumanEvalPrompts/fib.prompt b/benches/HumanEvalPrompts/fib.prompt new file mode 100644 index 0000000..164a898 --- /dev/null +++ b/benches/HumanEvalPrompts/fib.prompt @@ -0,0 +1,8 @@ +Return n-th Fibonacci number. +>>> fib(10) +55 +>>> fib(1) +1 +>>> fib(8) +21 + diff --git a/benches/HumanEvalPrompts/fib4.prompt b/benches/HumanEvalPrompts/fib4.prompt new file mode 100644 index 0000000..87c8cbd --- /dev/null +++ b/benches/HumanEvalPrompts/fib4.prompt @@ -0,0 +1,14 @@ +The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: +fib4(0) -> 0 +fib4(1) -> 0 +fib4(2) -> 2 +fib4(3) -> 0 +fib4(n) -> fib4(n-1) + fib4(n-2) + fib4(n-3) + fib4(n-4). +Please write a function to efficiently compute the n-th element of the fib4 number sequence. Do not use recursion. +>>> fib4(5) +4 +>>> fib4(6) +8 +>>> fib4(7) +14 + diff --git a/benches/HumanEvalPrompts/fibfib.prompt b/benches/HumanEvalPrompts/fibfib.prompt new file mode 100644 index 0000000..fb9d6a5 --- /dev/null +++ b/benches/HumanEvalPrompts/fibfib.prompt @@ -0,0 +1,13 @@ +The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: +fibfib(0) == 0 +fibfib(1) == 0 +fibfib(2) == 1 +fibfib(n) == fibfib(n-1) + fibfib(n-2) + fibfib(n-3). +Please write a function to efficiently compute the n-th element of the fibfib number sequence. +>>> fibfib(1) +0 +>>> fibfib(5) +4 +>>> fibfib(8) +24 + diff --git a/benches/HumanEvalPrompts/file_name_check.prompt b/benches/HumanEvalPrompts/file_name_check.prompt new file mode 100644 index 0000000..fd7fd1b --- /dev/null +++ b/benches/HumanEvalPrompts/file_name_check.prompt @@ -0,0 +1,13 @@ +Create a function which takes a string representing a file's name, and returns +'Yes' if the the file's name is valid, and returns 'No' otherwise. +A file's name is considered to be valid if and only if all the following conditions +are met: +- There should not be more than three digits ('0'-'9') in the file's name. +- The file's name contains exactly one dot '.' +- The substring before the dot should not be empty, and it starts with a letter from +the latin alphapet ('a'-'z' and 'A'-'Z'). +- The substring after the dot should be one of these: ['txt', 'exe', 'dll'] +Examples: +file_name_check("example.txt") # => 'Yes' +file_name_check("1example.dll") # => 'No' (the name should start with a latin alphapet letter) + diff --git a/benches/HumanEvalPrompts/filter_by_prefix.prompt b/benches/HumanEvalPrompts/filter_by_prefix.prompt new file mode 100644 index 0000000..fbbf8eb --- /dev/null +++ b/benches/HumanEvalPrompts/filter_by_prefix.prompt @@ -0,0 +1,6 @@ + Filter an input list of strings only for ones that start with a given prefix. +>>> filter_by_prefix([], 'a') +[] +>>> filter_by_prefix(['abc', 'bcd', 'cde', 'array'], 'a') +['abc', 'array'] + diff --git a/benches/HumanEvalPrompts/filter_by_substring.prompt b/benches/HumanEvalPrompts/filter_by_substring.prompt new file mode 100644 index 0000000..b99fe0c --- /dev/null +++ b/benches/HumanEvalPrompts/filter_by_substring.prompt @@ -0,0 +1,6 @@ + Filter an input list of strings only for ones that contain given substring +>>> filter_by_substring([], 'a') +[] +>>> filter_by_substring(['abc', 'bacd', 'cde', 'array'], 'a') +['abc', 'bacd', 'array'] + diff --git a/benches/HumanEvalPrompts/filter_integers.prompt b/benches/HumanEvalPrompts/filter_integers.prompt new file mode 100644 index 0000000..31c3792 --- /dev/null +++ b/benches/HumanEvalPrompts/filter_integers.prompt @@ -0,0 +1,6 @@ + Filter given list of any python values only for integers +>>> filter_integers(['a', 3.14, 5]) +[5] +>>> filter_integers([1, 2, 3, 'abc', {}, []]) +[1, 2, 3] + diff --git a/benches/HumanEvalPrompts/find_closest_elements.prompt b/benches/HumanEvalPrompts/find_closest_elements.prompt new file mode 100644 index 0000000..07e9688 --- /dev/null +++ b/benches/HumanEvalPrompts/find_closest_elements.prompt @@ -0,0 +1,7 @@ + From a supplied list of numbers (of length at least two) select and return two that are the closest to each +other and return them in order (smaller number, larger number). +>>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) +(2.0, 2.2) +>>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.0]) +(2.0, 2.0) + diff --git a/benches/HumanEvalPrompts/find_max.prompt b/benches/HumanEvalPrompts/find_max.prompt new file mode 100644 index 0000000..099b6d3 --- /dev/null +++ b/benches/HumanEvalPrompts/find_max.prompt @@ -0,0 +1,9 @@ +Write a function that accepts a list of strings. +The list contains different words. Return the word with maximum number +of unique characters. If multiple strings have maximum number of unique +characters, return the one which comes first in lexicographical order. + +find_max(["name", "of", "string"]) == "string" +find_max(["name", "enam", "game"]) == "enam" +find_max(["aaaaaaa", "bb" ,"cc"]) == ""aaaaaaa" + diff --git a/benches/HumanEvalPrompts/fix_spaces.prompt b/benches/HumanEvalPrompts/fix_spaces.prompt new file mode 100644 index 0000000..68387b1 --- /dev/null +++ b/benches/HumanEvalPrompts/fix_spaces.prompt @@ -0,0 +1,10 @@ + +Given a string text, replace all spaces in it with underscores, +and if a string has more than 2 consecutive spaces, +then replace all consecutive spaces with - + +fix_spaces("Example") == "Example" +fix_spaces("Example 1") == "Example_1" +fix_spaces(" Example 2") == "_Example_2" +fix_spaces(" Example 3") == "_Example-3" + diff --git a/benches/HumanEvalPrompts/fizz_buzz.prompt b/benches/HumanEvalPrompts/fizz_buzz.prompt new file mode 100644 index 0000000..df48aa2 --- /dev/null +++ b/benches/HumanEvalPrompts/fizz_buzz.prompt @@ -0,0 +1,8 @@ +Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. +>>> fizz_buzz(50) +0 +>>> fizz_buzz(78) +2 +>>> fizz_buzz(79) +3 + diff --git a/benches/HumanEvalPrompts/flip_case.prompt b/benches/HumanEvalPrompts/flip_case.prompt new file mode 100644 index 0000000..a8cae0b --- /dev/null +++ b/benches/HumanEvalPrompts/flip_case.prompt @@ -0,0 +1,4 @@ + For a given string, flip lowercase characters to uppercase and uppercase to lowercase. +>>> flip_case('Hello') +'hELLO' + diff --git a/benches/HumanEvalPrompts/fruit_distribution.prompt b/benches/HumanEvalPrompts/fruit_distribution.prompt new file mode 100644 index 0000000..ada4e53 --- /dev/null +++ b/benches/HumanEvalPrompts/fruit_distribution.prompt @@ -0,0 +1,12 @@ + +In this task, you will be given a string that represents a number of apples and oranges +that are distributed in a basket of fruit this basket contains +apples, oranges, and mango fruits. Given the string that represents the total number of +the oranges and apples and an integer that represent the total number of the fruits +in the basket return the number of the mango fruits in the basket. +for examble: +fruit_distribution("5 apples and 6 oranges", 19) ->19 - 5 - 6 = 8 +fruit_distribution("0 apples and 1 oranges",3) -> 3 - 0 - 1 = 2 +fruit_distribution("2 apples and 3 oranges", 100) -> 100 - 2 - 3 = 95 +fruit_distribution("100 apples and 1 oranges",120) -> 120 - 100 - 1 = 19 + diff --git a/benches/HumanEvalPrompts/generate_integers.prompt b/benches/HumanEvalPrompts/generate_integers.prompt new file mode 100644 index 0000000..ac8b57d --- /dev/null +++ b/benches/HumanEvalPrompts/generate_integers.prompt @@ -0,0 +1,9 @@ + +Given two positive integers a and b, return the even digits between a +and b, in ascending order. + +For example: +generate_integers(2, 8) => [2, 4, 6, 8] +generate_integers(8, 2) => [2, 4, 6, 8] +generate_integers(10, 14) => [] + diff --git a/benches/HumanEvalPrompts/get_closest_vowel.prompt b/benches/HumanEvalPrompts/get_closest_vowel.prompt new file mode 100644 index 0000000..5c7634d --- /dev/null +++ b/benches/HumanEvalPrompts/get_closest_vowel.prompt @@ -0,0 +1,14 @@ +You are given a word. Your task is to find the closest vowel that stands between +two consonants from the right side of the word (case sensitive). + +Vowels in the beginning and ending doesn't count. Return empty string if you didn't +find any vowel met the above condition. + +You may assume that the given string contains English letter only. + +Example: +get_closest_vowel("yogurt") ==> "u" +get_closest_vowel("FULL") ==> "U" +get_closest_vowel("quick") ==> "" +get_closest_vowel("ab") ==> "" + diff --git a/benches/HumanEvalPrompts/get_max_triples.prompt b/benches/HumanEvalPrompts/get_max_triples.prompt new file mode 100644 index 0000000..4feece7 --- /dev/null +++ b/benches/HumanEvalPrompts/get_max_triples.prompt @@ -0,0 +1,13 @@ + +You are given a positive integer n. You have to create an integer array a of length n. + For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. + Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, +and a[i] + a[j] + a[k] is a multiple of 3. + +Example : + Input: n = 5 + Output: 1 + Explanation: + a = [1, 3, 7, 13, 21] + The only valid triple is (1, 7, 13). + diff --git a/benches/HumanEvalPrompts/get_odd_collatz.prompt b/benches/HumanEvalPrompts/get_odd_collatz.prompt new file mode 100644 index 0000000..fb3a494 --- /dev/null +++ b/benches/HumanEvalPrompts/get_odd_collatz.prompt @@ -0,0 +1,16 @@ + +Given a positive integer n, return a sorted list that has the odd numbers in collatz sequence. + +The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined +as follows: start with any positive integer n. Then each term is obtained from the +previous term as follows: if the previous term is even, the next term is one half of +the previous term. If the previous term is odd, the next term is 3 times the previous +term plus 1. The conjecture is that no matter what value of n, the sequence will always reach 1. + +Note: + 1. Collatz(1) is [1]. + 2. returned list sorted in increasing order. + +For example: +get_odd_collatz(5) returns [1, 5] # The collatz sequence for 5 is [5, 16, 8, 4, 2, 1], so the odd numbers are only 1, and 5. + diff --git a/benches/HumanEvalPrompts/get_positive.prompt b/benches/HumanEvalPrompts/get_positive.prompt new file mode 100644 index 0000000..62cd055 --- /dev/null +++ b/benches/HumanEvalPrompts/get_positive.prompt @@ -0,0 +1,6 @@ +Return only positive numbers in the list. +>>> get_positive([-1, 2, -4, 5, 6]) +[2, 5, 6] +>>> get_positive([5, 3, -5, 2, -3, 3, 9, 0, 123, 1, -10]) +[5, 3, 2, 3, 9, 123, 1] + diff --git a/benches/HumanEvalPrompts/get_row.prompt b/benches/HumanEvalPrompts/get_row.prompt new file mode 100644 index 0000000..901c487 --- /dev/null +++ b/benches/HumanEvalPrompts/get_row.prompt @@ -0,0 +1,19 @@ + +You are given a 2 dimensional data, as a nested lists, +which is similar to matrix, however, unlike matrices, +each row may contain a different number of columns. +Given lst, and integer x, find integers x in the list, +and return list of tuples, [(x1, y1), (x2, y2) ...] such that +each tuple is a coordinate - (row, columns), starting with 0. +Sort coordinates initially by rows in ascending order. +Also, sort coordinates of the row by columns in descending order. + +Examples: +get_row([ + [1,2,3,4,5,6], + [1,2,3,4,1,6], + [1,2,3,4,5,1] +], 1) == [(0, 0), (1, 4), (1, 0), (2, 5), (2, 0)] +get_row([], 1) == [] +get_row([[], [1], [1, 2, 3]], 3) == [(2, 2)] + diff --git a/benches/HumanEvalPrompts/greatest_common_divisor.prompt b/benches/HumanEvalPrompts/greatest_common_divisor.prompt new file mode 100644 index 0000000..c68081c --- /dev/null +++ b/benches/HumanEvalPrompts/greatest_common_divisor.prompt @@ -0,0 +1,6 @@ + Return a greatest common divisor of two integers a and b +>>> greatest_common_divisor(3, 5) +1 +>>> greatest_common_divisor(25, 15) +5 + diff --git a/benches/HumanEvalPrompts/has_close_elements.prompt b/benches/HumanEvalPrompts/has_close_elements.prompt new file mode 100644 index 0000000..d978d35 --- /dev/null +++ b/benches/HumanEvalPrompts/has_close_elements.prompt @@ -0,0 +1,7 @@ + Check if in given list of numbers, are any two numbers closer to each other than +given threshold. +>>> has_close_elements([1.0, 2.0, 3.0], 0.5) +False +>>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3) +True + diff --git a/benches/HumanEvalPrompts/hex_key.prompt b/benches/HumanEvalPrompts/hex_key.prompt new file mode 100644 index 0000000..8455d5d --- /dev/null +++ b/benches/HumanEvalPrompts/hex_key.prompt @@ -0,0 +1,17 @@ +You have been tasked to write a function that receives +a hexadecimal number as a string and counts the number of hexadecimal +digits that are primes (prime number, or a prime, is a natural number +greater than 1 that is not a product of two smaller natural numbers). +Hexadecimal digits are 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, A, B, C, D, E, F. +Prime numbers are 2, 3, 5, 7, 11, 13, 17,... +So you have to determine a number of the following digits: 2, 3, 5, 7, +B (=decimal 11), D (=decimal 13). +Note: you may assume the input is always correct or empty string, +and symbols A,B,C,D,E,F are always uppercase. +Examples: +For num = "AB" the output should be 1. +For num = "1077E" the output should be 2. +For num = "ABED1A33" the output should be 4. +For num = "123456789ABCDEF0" the output should be 6. +For num = "2020" the output should be 2. + diff --git a/benches/HumanEvalPrompts/histogram.prompt b/benches/HumanEvalPrompts/histogram.prompt new file mode 100644 index 0000000..861c4f9 --- /dev/null +++ b/benches/HumanEvalPrompts/histogram.prompt @@ -0,0 +1,12 @@ +Given a string representing a space separated lowercase letters, return a dictionary +of the letter with the most repetition and containing the corresponding count. +If several letters have the same occurrence, return all of them. + +Example: +histogram('a b c') == {'a': 1, 'b': 1, 'c': 1} +histogram('a b b a') == {'a': 2, 'b': 2} +histogram('a b c a b') == {'a': 2, 'b': 2} +histogram('b b b b a') == {'b': 4} +histogram('') == {} + + diff --git a/benches/HumanEvalPrompts/how_many_times.prompt b/benches/HumanEvalPrompts/how_many_times.prompt new file mode 100644 index 0000000..0552d1d --- /dev/null +++ b/benches/HumanEvalPrompts/how_many_times.prompt @@ -0,0 +1,8 @@ + Find how many times a given substring can be found in the original string. Count overlaping cases. +>>> how_many_times('', 'a') +0 +>>> how_many_times('aaa', 'a') +3 +>>> how_many_times('aaaa', 'aa') +3 + diff --git a/benches/HumanEvalPrompts/incr_list.prompt b/benches/HumanEvalPrompts/incr_list.prompt new file mode 100644 index 0000000..fe932c5 --- /dev/null +++ b/benches/HumanEvalPrompts/incr_list.prompt @@ -0,0 +1,6 @@ +Return list with elements incremented by 1. +>>> incr_list([1, 2, 3]) +[2, 3, 4] +>>> incr_list([5, 3, 5, 2, 3, 3, 9, 0, 123]) +[6, 4, 6, 3, 4, 4, 10, 1, 124] + diff --git a/benches/HumanEvalPrompts/int_to_mini_roman.prompt b/benches/HumanEvalPrompts/int_to_mini_roman.prompt new file mode 100644 index 0000000..129e10a --- /dev/null +++ b/benches/HumanEvalPrompts/int_to_mini_roman.prompt @@ -0,0 +1,10 @@ + +Given a positive integer, obtain its roman numeral equivalent as a string, +and return it in lowercase. +Restrictions: 1 <= num <= 1000 + +Examples: +>>> int_to_mini_roman(19) == 'xix' +>>> int_to_mini_roman(152) == 'clii' +>>> int_to_mini_roman(426) == 'cdxxvi' + diff --git a/benches/HumanEvalPrompts/intersection.prompt b/benches/HumanEvalPrompts/intersection.prompt new file mode 100644 index 0000000..40a8541 --- /dev/null +++ b/benches/HumanEvalPrompts/intersection.prompt @@ -0,0 +1,19 @@ +You are given two intervals, +where each interval is a pair of integers. For example, interval = (start, end) = (1, 2). +The given intervals are closed which means that the interval (start, end) +includes both start and end. +For each given interval, it is assumed that its start is less or equal its end. +Your task is to determine whether the length of intersection of these two +intervals is a prime number. +Example, the intersection of the intervals (1, 3), (2, 4) is (2, 3) +which its length is 1, which not a prime number. +If the length of the intersection is a prime number, return "YES", +otherwise, return "NO". +If the two intervals don't intersect, return "NO". + + +[input/output] samples: +intersection((1, 2), (2, 3)) ==> "NO" +intersection((-1, 1), (0, 4)) ==> "NO" +intersection((-3, -1), (-5, 5)) ==> "YES" + diff --git a/benches/HumanEvalPrompts/intersperse.prompt b/benches/HumanEvalPrompts/intersperse.prompt new file mode 100644 index 0000000..ba8dd4f --- /dev/null +++ b/benches/HumanEvalPrompts/intersperse.prompt @@ -0,0 +1,6 @@ + Insert a number 'delimeter' between every two consecutive elements of input list `numbers' +>>> intersperse([], 4) +[] +>>> intersperse([1, 2, 3], 4) +[1, 4, 2, 4, 3] + diff --git a/benches/HumanEvalPrompts/is_bored.prompt b/benches/HumanEvalPrompts/is_bored.prompt new file mode 100644 index 0000000..2d7601e --- /dev/null +++ b/benches/HumanEvalPrompts/is_bored.prompt @@ -0,0 +1,11 @@ + +You'll be given a string of words, and your task is to count the number +of boredoms. A boredom is a sentence that starts with the word "I". +Sentences are delimited by '.', '?' or '!'. + +For example: +>>> is_bored("Hello world") +0 +>>> is_bored("The sky is blue. The sun is shining. I love this weather") +1 + diff --git a/benches/HumanEvalPrompts/is_equal_to_sum_even.prompt b/benches/HumanEvalPrompts/is_equal_to_sum_even.prompt new file mode 100644 index 0000000..860298c --- /dev/null +++ b/benches/HumanEvalPrompts/is_equal_to_sum_even.prompt @@ -0,0 +1,6 @@ +Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers +Example +is_equal_to_sum_even(4) == False +is_equal_to_sum_even(6) == False +is_equal_to_sum_even(8) == True + diff --git a/benches/HumanEvalPrompts/is_happy.prompt b/benches/HumanEvalPrompts/is_happy.prompt new file mode 100644 index 0000000..ebfbd6d --- /dev/null +++ b/benches/HumanEvalPrompts/is_happy.prompt @@ -0,0 +1,11 @@ +You are given a string s. +Your task is to check if the string is happy or not. +A string is happy if its length is at least 3 and every 3 consecutive letters are distinct +For example: +is_happy(a) => False +is_happy(aa) => False +is_happy(abcd) => True +is_happy(aabb) => False +is_happy(adb) => True +is_happy(xyy) => False + diff --git a/benches/HumanEvalPrompts/is_multiply_prime.prompt b/benches/HumanEvalPrompts/is_multiply_prime.prompt new file mode 100644 index 0000000..2ae2d0b --- /dev/null +++ b/benches/HumanEvalPrompts/is_multiply_prime.prompt @@ -0,0 +1,7 @@ +Write a function that returns true if the given number is the multiplication of 3 prime numbers +and false otherwise. +Knowing that (a) is less then 100. +Example: +is_multiply_prime(30) == True +30 = 2 * 3 * 5 + diff --git a/benches/HumanEvalPrompts/is_nested.prompt b/benches/HumanEvalPrompts/is_nested.prompt new file mode 100644 index 0000000..34b467e --- /dev/null +++ b/benches/HumanEvalPrompts/is_nested.prompt @@ -0,0 +1,12 @@ + +Create a function that takes a string as input which contains only square brackets. +The function should return True if and only if there is a valid subsequence of brackets +where at least one bracket in the subsequence is nested. + +is_nested('[[]]') ➞ True +is_nested('[]]]]]]][[[[[]') ➞ False +is_nested('[][]') ➞ False +is_nested('[]') ➞ False +is_nested('[[][]]') ➞ True +is_nested('[[]][[') ➞ True + diff --git a/benches/HumanEvalPrompts/is_palindrome.prompt b/benches/HumanEvalPrompts/is_palindrome.prompt new file mode 100644 index 0000000..ad8cc37 --- /dev/null +++ b/benches/HumanEvalPrompts/is_palindrome.prompt @@ -0,0 +1,11 @@ + +Checks if given string is a palindrome +>>> is_palindrome('') +True +>>> is_palindrome('aba') +True +>>> is_palindrome('aaaaa') +True +>>> is_palindrome('zbcd') +False + diff --git a/benches/HumanEvalPrompts/is_prime.prompt b/benches/HumanEvalPrompts/is_prime.prompt new file mode 100644 index 0000000..b9b3ae6 --- /dev/null +++ b/benches/HumanEvalPrompts/is_prime.prompt @@ -0,0 +1,16 @@ +Return true if a given number is prime, and false otherwise. +>>> is_prime(6) +False +>>> is_prime(101) +True +>>> is_prime(11) +True +>>> is_prime(13441) +True +>>> is_prime(61) +True +>>> is_prime(4) +False +>>> is_prime(1) +False + diff --git a/benches/HumanEvalPrompts/is_simple_power.prompt b/benches/HumanEvalPrompts/is_simple_power.prompt new file mode 100644 index 0000000..5d4d2d6 --- /dev/null +++ b/benches/HumanEvalPrompts/is_simple_power.prompt @@ -0,0 +1,11 @@ +Your task is to write a function that returns true if a number x is a simple +power of n and false in other cases. +x is a simple power of n if n**int=x +For example: +is_simple_power(1, 4) => true +is_simple_power(2, 2) => true +is_simple_power(8, 2) => true +is_simple_power(3, 2) => false +is_simple_power(3, 1) => false +is_simple_power(5, 3) => false + diff --git a/benches/HumanEvalPrompts/is_sorted.prompt b/benches/HumanEvalPrompts/is_sorted.prompt new file mode 100644 index 0000000..ce87866 --- /dev/null +++ b/benches/HumanEvalPrompts/is_sorted.prompt @@ -0,0 +1,15 @@ + +Given a list of numbers, return whether or not they are sorted +in ascending order. If list has more than 1 duplicate of the same +number, return False. Assume no negative numbers and only integers. + +Examples +is_sorted([5]) ➞ True +is_sorted([1, 2, 3, 4, 5]) ➞ True +is_sorted([1, 3, 2, 4, 5]) ➞ False +is_sorted([1, 2, 3, 4, 5, 6]) ➞ True +is_sorted([1, 2, 3, 4, 5, 6, 7]) ➞ True +is_sorted([1, 3, 2, 4, 5, 6, 7]) ➞ False +is_sorted([1, 2, 2, 3, 3, 4]) ➞ True +is_sorted([1, 2, 2, 2, 3, 4]) ➞ False + diff --git a/benches/HumanEvalPrompts/iscube.prompt b/benches/HumanEvalPrompts/iscube.prompt new file mode 100644 index 0000000..7195e12 --- /dev/null +++ b/benches/HumanEvalPrompts/iscube.prompt @@ -0,0 +1,12 @@ + +Write a function that takes an integer a and returns True +if this ingeger is a cube of some integer number. +Note: you may assume the input is always valid. +Examples: +iscube(1) ==> True +iscube(2) ==> False +iscube(-1) ==> True +iscube(64) ==> True +iscube(0) ==> True +iscube(180) ==> False + diff --git a/benches/HumanEvalPrompts/largest_divisor.prompt b/benches/HumanEvalPrompts/largest_divisor.prompt new file mode 100644 index 0000000..2de3e42 --- /dev/null +++ b/benches/HumanEvalPrompts/largest_divisor.prompt @@ -0,0 +1,4 @@ + For a given number n, find the largest number that divides n evenly, smaller than n +>>> largest_divisor(15) +5 + diff --git a/benches/HumanEvalPrompts/largest_prime_factor.prompt b/benches/HumanEvalPrompts/largest_prime_factor.prompt new file mode 100644 index 0000000..bbdd6f8 --- /dev/null +++ b/benches/HumanEvalPrompts/largest_prime_factor.prompt @@ -0,0 +1,6 @@ +Return the largest prime factor of n. Assume n > 1 and is not a prime. +>>> largest_prime_factor(13195) +29 +>>> largest_prime_factor(2048) +2 + diff --git a/benches/HumanEvalPrompts/largest_smallest_integers.prompt b/benches/HumanEvalPrompts/largest_smallest_integers.prompt new file mode 100644 index 0000000..bc1b204 --- /dev/null +++ b/benches/HumanEvalPrompts/largest_smallest_integers.prompt @@ -0,0 +1,11 @@ + +Create a function that returns a tuple (a, b), where 'a' is +the largest of negative integers, and 'b' is the smallest +of positive integers in a list. +If there is no negative or positive integers, return them as None. + +Examples: +largest_smallest_integers([2, 4, 1, 3, 5, 7]) == (None, 1) +largest_smallest_integers([]) == (None, None) +largest_smallest_integers([0]) == (None, None) + diff --git a/benches/HumanEvalPrompts/longest.prompt b/benches/HumanEvalPrompts/longest.prompt new file mode 100644 index 0000000..3ab0b07 --- /dev/null +++ b/benches/HumanEvalPrompts/longest.prompt @@ -0,0 +1,9 @@ + Out of list of strings, return the longest one. Return the first one in case of multiple +strings of the same length. Return None in case the input list is empty. +>>> longest([]) + +>>> longest(['a', 'b', 'c']) +'a' +>>> longest(['a', 'bb', 'ccc']) +'ccc' + diff --git a/benches/HumanEvalPrompts/make_a_pile.prompt b/benches/HumanEvalPrompts/make_a_pile.prompt new file mode 100644 index 0000000..da4c0f8 --- /dev/null +++ b/benches/HumanEvalPrompts/make_a_pile.prompt @@ -0,0 +1,13 @@ + +Given a positive integer n, you have to make a pile of n levels of stones. +The first level has n stones. +The number of stones in the next level is: + - the next odd number if n is odd. + - the next even number if n is even. +Return the number of stones in each level in a list, where element at index +i represents the number of stones in the level (i+1). + +Examples: +>>> make_a_pile(3) +[3, 5, 7] + diff --git a/benches/HumanEvalPrompts/match_parens.prompt b/benches/HumanEvalPrompts/match_parens.prompt new file mode 100644 index 0000000..ebf10fe --- /dev/null +++ b/benches/HumanEvalPrompts/match_parens.prompt @@ -0,0 +1,14 @@ + +You are given a list of two strings, both strings consist of open +parentheses '(' or close parentheses ')' only. +Your job is to check if it is possible to concatenate the two strings in +some order, that the resulting string will be good. +A string S is considered to be good if and only if all parentheses in S +are balanced. For example: the string '(())()' is good, while the string +'())' is not. +Return 'Yes' if there's a way to make a good string, and return 'No' otherwise. + +Examples: +match_parens(['()(', ')']) == 'Yes' +match_parens([')', ')']) == 'No' + diff --git a/benches/HumanEvalPrompts/max_element.prompt b/benches/HumanEvalPrompts/max_element.prompt new file mode 100644 index 0000000..7c1fea5 --- /dev/null +++ b/benches/HumanEvalPrompts/max_element.prompt @@ -0,0 +1,6 @@ +Return maximum element in the list. +>>> max_element([1, 2, 3]) +3 +>>> max_element([5, 3, -5, 2, -3, 3, 9, 0, 123, 1, -10]) +123 + diff --git a/benches/HumanEvalPrompts/max_fill.prompt b/benches/HumanEvalPrompts/max_fill.prompt new file mode 100644 index 0000000..40e4007 --- /dev/null +++ b/benches/HumanEvalPrompts/max_fill.prompt @@ -0,0 +1,33 @@ + +You are given a rectangular grid of wells. Each row represents a single well, +and each 1 in a row represents a single unit of water. +Each well has a corresponding bucket that can be used to extract water from it, +and all buckets have the same capacity. +Your task is to use the buckets to empty the wells. +Output the number of times you need to lower the buckets. + +Example 1: + Input: + grid : [[0,0,1,0], [0,1,0,0], [1,1,1,1]] + bucket_capacity : 1 + Output: 6 + +Example 2: + Input: + grid : [[0,0,1,1], [0,0,0,0], [1,1,1,1], [0,1,1,1]] + bucket_capacity : 2 + Output: 5 + +Example 3: + Input: + grid : [[0,0,0], [0,0,0]] + bucket_capacity : 5 + Output: 0 + +Constraints: + * all wells have the same length + * 1 <= grid.length <= 10^2 + * 1 <= grid[:,1].length <= 10^2 + * grid[i][j] -> 0 | 1 + * 1 <= capacity <= 10 + diff --git a/benches/HumanEvalPrompts/maximum.prompt b/benches/HumanEvalPrompts/maximum.prompt new file mode 100644 index 0000000..2eb6fbc --- /dev/null +++ b/benches/HumanEvalPrompts/maximum.prompt @@ -0,0 +1,24 @@ + +Given an array arr of integers and a positive integer k, return a sorted list +of length k with the maximum k numbers in arr. + +Example 1: + + Input: arr = [-3, -4, 5], k = 3 + Output: [-4, -3, 5] + +Example 2: + + Input: arr = [4, -4, 4], k = 2 + Output: [4, 4] + +Example 3: + + Input: arr = [-3, 2, 1, 2, -1, -2, 1], k = 1 + Output: [2] + +Note: + 1. The length of the array will be in the range of [1, 1000]. + 2. The elements in the array will be in the range of [-1000, 1000]. + 3. 0 <= k <= len(arr) + diff --git a/benches/HumanEvalPrompts/mean_absolute_derivation.prompt b/benches/HumanEvalPrompts/mean_absolute_derivation.prompt new file mode 100644 index 0000000..b93cbee --- /dev/null +++ b/benches/HumanEvalPrompts/mean_absolute_derivation.prompt @@ -0,0 +1,8 @@ + For a given list of input numbers, calculate Mean Absolute Deviation +around the mean of this dataset. +Mean Absolute Deviation is the average absolute difference between each +element and a centerpoint (mean in this case): +MAD = average | x - x_mean | +>>> mean_absolute_deviation([1.0, 2.0, 3.0, 4.0]) +1.0 + diff --git a/benches/HumanEvalPrompts/median.prompt b/benches/HumanEvalPrompts/median.prompt new file mode 100644 index 0000000..d3e3147 --- /dev/null +++ b/benches/HumanEvalPrompts/median.prompt @@ -0,0 +1,6 @@ +Return median of elements in the list l. +>>> median([3, 1, 2, 4, 5]) +3 +>>> median([-10, 4, 6, 1000, 10, 20]) +15.0 + diff --git a/benches/HumanEvalPrompts/minPath.prompt b/benches/HumanEvalPrompts/minPath.prompt new file mode 100644 index 0000000..45282bf --- /dev/null +++ b/benches/HumanEvalPrompts/minPath.prompt @@ -0,0 +1,29 @@ + +Given a grid with N rows and N columns (N >= 2) and a positive integer k, +each cell of the grid contains a value. Every integer in the range [1, N * N] +inclusive appears exactly once on the cells of the grid. + +You have to find the minimum path of length k in the grid. You can start +from any cell, and in each step you can move to any of the neighbor cells, +in other words, you can go to cells which share an edge with you current +cell. +Please note that a path of length k means visiting exactly k cells (not +necessarily distinct). +You CANNOT go off the grid. +A path A (of length k) is considered less than a path B (of length k) if +after making the ordered lists of the values on the cells that A and B go +through (let's call them lst_A and lst_B), lst_A is lexicographically less +than lst_B, in other words, there exist an integer index i (1 <= i <= k) +such that lst_A[i] < lst_B[i] and for any j (1 <= j < i) we have +lst_A[j] = lst_B[j]. +It is guaranteed that the answer is unique. +Return an ordered list of the values on the cells that the minimum path go through. + +Examples: + + Input: grid = [ [1,2,3], [4,5,6], [7,8,9]], k = 3 + Output: [1, 2, 1] + + Input: grid = [ [5,9,3], [4,1,6], [7,8,2]], k = 1 + Output: [1] + diff --git a/benches/HumanEvalPrompts/minSubArraySum.prompt b/benches/HumanEvalPrompts/minSubArraySum.prompt new file mode 100644 index 0000000..5cd75d3 --- /dev/null +++ b/benches/HumanEvalPrompts/minSubArraySum.prompt @@ -0,0 +1,7 @@ + +Given an array of integers nums, find the minimum sum of any non-empty sub-array +of nums. +Example +minSubArraySum([2, 3, 4, 1, 2, 4]) == 1 +minSubArraySum([-1, -2, -3]) == -6 + diff --git a/benches/HumanEvalPrompts/modp.prompt b/benches/HumanEvalPrompts/modp.prompt new file mode 100644 index 0000000..cca2bf1 --- /dev/null +++ b/benches/HumanEvalPrompts/modp.prompt @@ -0,0 +1,12 @@ +Return 2^n modulo p (be aware of numerics). +>>> modp(3, 5) +3 +>>> modp(1101, 101) +2 +>>> modp(0, 101) +1 +>>> modp(3, 11) +8 +>>> modp(100, 101) +1 + diff --git a/benches/HumanEvalPrompts/monotonic.prompt b/benches/HumanEvalPrompts/monotonic.prompt new file mode 100644 index 0000000..b861579 --- /dev/null +++ b/benches/HumanEvalPrompts/monotonic.prompt @@ -0,0 +1,8 @@ +Return True is list elements are monotonically increasing or decreasing. +>>> monotonic([1, 2, 4, 20]) +True +>>> monotonic([1, 20, 4, 10]) +False +>>> monotonic([4, 1, 0, -10]) +True + diff --git a/benches/HumanEvalPrompts/move_one_ball.prompt b/benches/HumanEvalPrompts/move_one_ball.prompt new file mode 100644 index 0000000..d347138 --- /dev/null +++ b/benches/HumanEvalPrompts/move_one_ball.prompt @@ -0,0 +1,26 @@ +We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The +numbers in the array will be randomly ordered. Your task is to determine if +it is possible to get an array sorted in non-decreasing order by performing +the following operation on the given array: + You are allowed to perform right shift operation any number of times. + +One right shift operation means shifting all elements of the array by one +position in the right direction. The last element of the array will be moved to +the starting position in the array i.e. 0th index. + +If it is possible to obtain the sorted array by performing the above operation +then return True else return False. +If the given array is empty then return True. + +Note: The given list is guaranteed to have unique elements. + +For Example: + +move_one_ball([3, 4, 5, 1, 2])==>True +Explanation: By performin 2 right shift operations, non-decreasing order can + be achieved for the given array. +move_one_ball([3, 5, 4, 1, 2])==>False +Explanation:It is not possible to get non-decreasing order for the given + array by performing any number of right shift operations. + + diff --git a/benches/HumanEvalPrompts/multiply.prompt b/benches/HumanEvalPrompts/multiply.prompt new file mode 100644 index 0000000..3c97a91 --- /dev/null +++ b/benches/HumanEvalPrompts/multiply.prompt @@ -0,0 +1,9 @@ +Complete the function that takes two integers and returns +the product of their unit digits. +Assume the input is always valid. +Examples: +multiply(148, 412) should return 16. +multiply(19, 28) should return 72. +multiply(2020, 1851) should return 0. +multiply(14,-15) should return 20. + diff --git a/benches/HumanEvalPrompts/next_smallest.prompt b/benches/HumanEvalPrompts/next_smallest.prompt new file mode 100644 index 0000000..e336ff6 --- /dev/null +++ b/benches/HumanEvalPrompts/next_smallest.prompt @@ -0,0 +1,10 @@ + +You are given a list of integers. +Write a function next_smallest() that returns the 2nd smallest element of the list. +Return None if there is no such element. + +next_smallest([1, 2, 3, 4, 5]) == 2 +next_smallest([5, 1, 4, 3, 2]) == 2 +next_smallest([]) == None +next_smallest([1, 1]) == None + diff --git a/benches/HumanEvalPrompts/numerical_letter_grade.prompt b/benches/HumanEvalPrompts/numerical_letter_grade.prompt new file mode 100644 index 0000000..490294d --- /dev/null +++ b/benches/HumanEvalPrompts/numerical_letter_grade.prompt @@ -0,0 +1,24 @@ +It is the last week of the semester and the teacher has to give the grades +to students. The teacher has been making her own algorithm for grading. +The only problem is, she has lost the code she used for grading. +She has given you a list of GPAs for some students and you have to write +a function that can output a list of letter grades using the following table: + GPA | Letter grade + 4.0 A+ + > 3.7 A + > 3.3 A- + > 3.0 B+ + > 2.7 B + > 2.3 B- + > 2.0 C+ + > 1.7 C + > 1.3 C- + > 1.0 D+ + > 0.7 D + > 0.0 D- + 0.0 E + + +Example: +grade_equation([4.0, 3, 1.7, 2, 3.5]) ==> ['A+', 'B', 'C-', 'C', 'A-'] + diff --git a/benches/HumanEvalPrompts/odd_count.prompt b/benches/HumanEvalPrompts/odd_count.prompt new file mode 100644 index 0000000..4078741 --- /dev/null +++ b/benches/HumanEvalPrompts/odd_count.prompt @@ -0,0 +1,11 @@ +Given a list of strings, where each string consists of only digits, return a list. +Each element i of the output should be "the number of odd elements in the +string i of the input." where all the i's should be replaced by the number +of odd digits in the i'th string of the input. + +>>> odd_count(['1234567']) +["the number of odd elements 4n the str4ng 4 of the 4nput."] +>>> odd_count(['3',"11111111"]) +["the number of odd elements 1n the str1ng 1 of the 1nput.", + "the number of odd elements 8n the str8ng 8 of the 8nput."] + diff --git a/benches/HumanEvalPrompts/order_by_points.prompt b/benches/HumanEvalPrompts/order_by_points.prompt new file mode 100644 index 0000000..1309ffd --- /dev/null +++ b/benches/HumanEvalPrompts/order_by_points.prompt @@ -0,0 +1,10 @@ + +Write a function which sorts the given list of integers +in ascending order according to the sum of their digits. +Note: if there are several items with similar sum of their digits, +order them based on their index in original list. + +For example: +>>> order_by_points([1, 11, -1, -11, -12]) == [-1, -11, 1, -12, 11] +>>> order_by_points([]) == [] + diff --git a/benches/HumanEvalPrompts/pairs_sum_to_zero.prompt b/benches/HumanEvalPrompts/pairs_sum_to_zero.prompt new file mode 100644 index 0000000..d8f33ea --- /dev/null +++ b/benches/HumanEvalPrompts/pairs_sum_to_zero.prompt @@ -0,0 +1,15 @@ + +pairs_sum_to_zero takes a list of integers as an input. +it returns True if there are two distinct elements in the list that +sum to zero, and False otherwise. +>>> pairs_sum_to_zero([1, 3, 5, 0]) +False +>>> pairs_sum_to_zero([1, 3, -2, 1]) +False +>>> pairs_sum_to_zero([1, 2, 3, 7]) +False +>>> pairs_sum_to_zero([2, 4, -5, 3, 5, 7]) +True +>>> pairs_sum_to_zero([1]) +False + diff --git a/benches/HumanEvalPrompts/parse_music.prompt b/benches/HumanEvalPrompts/parse_music.prompt new file mode 100644 index 0000000..b331867 --- /dev/null +++ b/benches/HumanEvalPrompts/parse_music.prompt @@ -0,0 +1,12 @@ + Input to this function is a string representing musical notes in a special ASCII format. +Your task is to parse this string and return list of integers corresponding to how many beats does each +not last. + +Here is a legend: +'o' - whole note, lasts four beats +'o|' - half note, lasts two beats +'.|' - quater note, lasts one beat + +>>> parse_music('o o| .| o| o| .| .| .| .| o o') +[4, 2, 1, 2, 2, 1, 1, 1, 1, 4, 4] + diff --git a/benches/HumanEvalPrompts/parse_nested_parens.prompt b/benches/HumanEvalPrompts/parse_nested_parens.prompt new file mode 100644 index 0000000..98fa834 --- /dev/null +++ b/benches/HumanEvalPrompts/parse_nested_parens.prompt @@ -0,0 +1,7 @@ + Input to this function is a string represented multiple groups for nested parentheses separated by spaces. +For each of the group, output the deepest level of nesting of parentheses. +E.g. (()()) has maximum two levels of nesting while ((())) has three. + +>>> parse_nested_parens('(()()) ((())) () ((())()())') +[2, 3, 1, 3] + diff --git a/benches/HumanEvalPrompts/pluck.prompt b/benches/HumanEvalPrompts/pluck.prompt new file mode 100644 index 0000000..821bb74 --- /dev/null +++ b/benches/HumanEvalPrompts/pluck.prompt @@ -0,0 +1,33 @@ + +"Given an array representing a branch of a tree that has non-negative integer nodes +your task is to pluck one of the nodes and return it. +The plucked node should be the node with the smallest even value. +If multiple nodes with the same smallest even value are found return the node that has smallest index. + +The plucked node should be returned in a list, [ smalest_value, its index ], +If there are no even values or the given array is empty, return []. + +Example 1: + Input: [4,2,3] + Output: [2, 1] + Explanation: 2 has the smallest even value, and 2 has the smallest index. + +Example 2: + Input: [1,2,3] + Output: [2, 1] + Explanation: 2 has the smallest even value, and 2 has the smallest index. + +Example 3: + Input: [] + Output: [] + +Example 4: + Input: [5, 0, 3, 0, 4, 2] + Output: [0, 1] + Explanation: 0 is the smallest value, but there are two zeros, + so we will choose the first zero, which has the smallest index. + +Constraints: + * 1 <= nodes.length <= 10000 + * 0 <= node.value + diff --git a/benches/HumanEvalPrompts/poly.prompt b/benches/HumanEvalPrompts/poly.prompt new file mode 100644 index 0000000..c5afeff --- /dev/null +++ b/benches/HumanEvalPrompts/poly.prompt @@ -0,0 +1,4 @@ + +Evaluates polynomial with coefficients xs at point x. +return xs[0] + xs[1] * x + xs[1] * x^2 + .... xs[n] * x^n + diff --git a/benches/HumanEvalPrompts/prime_fib.prompt b/benches/HumanEvalPrompts/prime_fib.prompt new file mode 100644 index 0000000..41d0344 --- /dev/null +++ b/benches/HumanEvalPrompts/prime_fib.prompt @@ -0,0 +1,13 @@ + +prime_fib returns n-th number that is a Fibonacci number and it's also prime. +>>> prime_fib(1) +2 +>>> prime_fib(2) +3 +>>> prime_fib(3) +5 +>>> prime_fib(4) +13 +>>> prime_fib(5) +89 + diff --git a/benches/HumanEvalPrompts/prime_length.prompt b/benches/HumanEvalPrompts/prime_length.prompt new file mode 100644 index 0000000..c53854e --- /dev/null +++ b/benches/HumanEvalPrompts/prime_length.prompt @@ -0,0 +1,8 @@ +Write a function that takes a string and returns True if the string +length is a prime number or False otherwise +Examples +prime_length('Hello') == True +prime_length('abcdcba') == True +prime_length('kittens') == True +prime_length('orange') == False + diff --git a/benches/HumanEvalPrompts/prod_signs.prompt b/benches/HumanEvalPrompts/prod_signs.prompt new file mode 100644 index 0000000..942c1f1 --- /dev/null +++ b/benches/HumanEvalPrompts/prod_signs.prompt @@ -0,0 +1,11 @@ + +You are given an array arr of integers and you need to return +sum of magnitudes of integers multiplied by product of all signs +of each number in the array, represented by 1, -1 or 0. +Note: return None for empty arr. + +Example: +>>> prod_signs([1, 2, 2, -4]) == -9 +>>> prod_signs([0, 1]) == 0 +>>> prod_signs([]) == None + diff --git a/benches/HumanEvalPrompts/remove_duplicates.prompt b/benches/HumanEvalPrompts/remove_duplicates.prompt new file mode 100644 index 0000000..da8ac67 --- /dev/null +++ b/benches/HumanEvalPrompts/remove_duplicates.prompt @@ -0,0 +1,5 @@ + From a list of integers, remove all elements that occur more than once. +Keep order of elements left the same as in the input. +>>> remove_duplicates([1, 2, 3, 2, 4]) +[1, 3, 4] + diff --git a/benches/HumanEvalPrompts/remove_vowels.prompt b/benches/HumanEvalPrompts/remove_vowels.prompt new file mode 100644 index 0000000..23590cf --- /dev/null +++ b/benches/HumanEvalPrompts/remove_vowels.prompt @@ -0,0 +1,15 @@ + +remove_vowels is a function that takes string and returns string without vowels. +>>> remove_vowels('') +'' +>>> remove_vowels("abcdef\nghijklm") +'bcdf\nghjklm' +>>> remove_vowels('abcdef') +'bcdf' +>>> remove_vowels('aaaaa') +'' +>>> remove_vowels('aaBAA') +'B' +>>> remove_vowels('zbcd') +'zbcd' + diff --git a/benches/HumanEvalPrompts/rescale_to_unit.prompt b/benches/HumanEvalPrompts/rescale_to_unit.prompt new file mode 100644 index 0000000..cef68c6 --- /dev/null +++ b/benches/HumanEvalPrompts/rescale_to_unit.prompt @@ -0,0 +1,5 @@ + Given list of numbers (of at least two elements), apply a linear transform to that list, +such that the smallest number will become 0 and the largest will become 1 +>>> rescale_to_unit([1.0, 2.0, 3.0, 4.0, 5.0]) +[0.0, 0.25, 0.5, 0.75, 1.0] + diff --git a/benches/HumanEvalPrompts/reverse_delete.prompt b/benches/HumanEvalPrompts/reverse_delete.prompt new file mode 100644 index 0000000..46135dd --- /dev/null +++ b/benches/HumanEvalPrompts/reverse_delete.prompt @@ -0,0 +1,10 @@ +Task +We are given two strings s and c, you have to deleted all the characters in s that are equal to any character in c +then check if the result string is palindrome. +A string is called palindrome if it reads the same backward as forward. +You should return a tuple containing the result string and True/False for the check. +Example +For s = "abcde", c = "ae", the result should be ('bcd',False) +For s = "abcdef", c = "b" the result should be ('acdef',False) +For s = "abcdedcba", c = "ab", the result should be ('cdedc',True) + diff --git a/benches/HumanEvalPrompts/right_angle_triangle.prompt b/benches/HumanEvalPrompts/right_angle_triangle.prompt new file mode 100644 index 0000000..3af6271 --- /dev/null +++ b/benches/HumanEvalPrompts/right_angle_triangle.prompt @@ -0,0 +1,9 @@ + +Given the lengths of the three sides of a triangle. Return True if the three +sides form a right-angled triangle, False otherwise. +A right-angled triangle is a triangle in which one angle is right angle or +90 degree. +Example: +right_angle_triangle(3, 4, 5) == True +right_angle_triangle(1, 2, 3) == False + diff --git a/benches/HumanEvalPrompts/rolling_max.prompt b/benches/HumanEvalPrompts/rolling_max.prompt new file mode 100644 index 0000000..6531e28 --- /dev/null +++ b/benches/HumanEvalPrompts/rolling_max.prompt @@ -0,0 +1,5 @@ + From a given list of integers, generate a list of rolling maximum element found until given moment +in the sequence. +>>> rolling_max([1, 2, 3, 2, 3, 4, 2]) +[1, 2, 3, 3, 3, 4, 4] + diff --git a/benches/HumanEvalPrompts/rounded_avg.prompt b/benches/HumanEvalPrompts/rounded_avg.prompt new file mode 100644 index 0000000..acbff8c --- /dev/null +++ b/benches/HumanEvalPrompts/rounded_avg.prompt @@ -0,0 +1,10 @@ +You are given two positive integers n and m, and your task is to compute the +average of the integers from n through m (including n and m). +Round the answer to the nearest integer and convert that to binary. +If n is greater than m, return -1. +Example: +rounded_avg(1, 5) => "0b11" +rounded_avg(7, 5) => -1 +rounded_avg(10, 20) => "0b1111" +rounded_avg(20, 33) => "0b11010" + diff --git a/benches/HumanEvalPrompts/same_chars.prompt b/benches/HumanEvalPrompts/same_chars.prompt new file mode 100644 index 0000000..b440f32 --- /dev/null +++ b/benches/HumanEvalPrompts/same_chars.prompt @@ -0,0 +1,15 @@ + +Check if two words have the same characters. +>>> same_chars('eabcdzzzz', 'dddzzzzzzzddeddabc') +True +>>> same_chars('abcd', 'dddddddabc') +True +>>> same_chars('dddddddabc', 'abcd') +True +>>> same_chars('eabcd', 'dddddddabc') +False +>>> same_chars('abcd', 'dddddddabce') +False +>>> same_chars('eabcdzzzz', 'dddzzzzzzzddddabc') +False + diff --git a/benches/HumanEvalPrompts/search.prompt b/benches/HumanEvalPrompts/search.prompt new file mode 100644 index 0000000..b3e4577 --- /dev/null +++ b/benches/HumanEvalPrompts/search.prompt @@ -0,0 +1,10 @@ + +You are given a non-empty list of positive integers. Return the greatest integer that is greater than +zero, and has a frequency greater than or equal to the value of the integer itself. +The frequency of an integer is the number of times it appears in the list. +If no such a value exist, return -1. +Examples: + search([4, 1, 2, 2, 3, 1]) == 2 + search([1, 2, 2, 3, 3, 3, 4, 4, 4]) == 3 + search([5, 5, 4, 4, 4]) == -1 + diff --git a/benches/HumanEvalPrompts/select_words.prompt b/benches/HumanEvalPrompts/select_words.prompt new file mode 100644 index 0000000..6fc80c8 --- /dev/null +++ b/benches/HumanEvalPrompts/select_words.prompt @@ -0,0 +1,12 @@ +Given a string s and a natural number n, you have been tasked to implement +a function that returns a list of all words from string s that contain exactly +n consonants, in order these words appear in the string s. +If the string s is empty then the function should return an empty list. +Note: you may assume the input string contains only letters and spaces. +Examples: +select_words("Mary had a little lamb", 4) ==> ["little"] +select_words("Mary had a little lamb", 3) ==> ["Mary", "lamb"] +select_words("simple white space", 2) ==> [] +select_words("Hello world", 4) ==> ["world"] +select_words("Uncle sam", 3) ==> ["Uncle"] + diff --git a/benches/HumanEvalPrompts/separate_paren_groups.prompt b/benches/HumanEvalPrompts/separate_paren_groups.prompt new file mode 100644 index 0000000..30b4f34 --- /dev/null +++ b/benches/HumanEvalPrompts/separate_paren_groups.prompt @@ -0,0 +1,7 @@ + Input to this function is a string containing multiple groups of nested parentheses. Your goal is to +separate those group into separate strings and return the list of those. +Separate groups are balanced (each open brace is properly closed) and not nested within each other +Ignore any spaces in the input string. +>>> separate_paren_groups('( ) (( )) (( )( ))') +['()', '(())', '(()())'] + diff --git a/benches/HumanEvalPrompts/simplify.prompt b/benches/HumanEvalPrompts/simplify.prompt new file mode 100644 index 0000000..71b25f6 --- /dev/null +++ b/benches/HumanEvalPrompts/simplify.prompt @@ -0,0 +1,11 @@ +Your task is to implement a function that will simplify the expression +x * n. The function returns True if x * n evaluates to a whole number and False +otherwise. Both x and n, are string representation of a fraction, and have the following format, +/ where both numerator and denominator are positive whole numbers. + +You can assume that x, and n are valid fractions, and do not have zero as denominator. + +simplify("1/5", "5/1") = True +simplify("1/6", "2/1") = False +simplify("7/10", "10/2") = False + diff --git a/benches/HumanEvalPrompts/skjkasdkd.prompt b/benches/HumanEvalPrompts/skjkasdkd.prompt new file mode 100644 index 0000000..c51cac4 --- /dev/null +++ b/benches/HumanEvalPrompts/skjkasdkd.prompt @@ -0,0 +1,11 @@ +You are given a list of integers. +You need to find the largest prime value and return the sum of its digits. + +Examples: +For lst = [0,3,2,1,3,5,7,4,5,5,5,2,181,32,4,32,3,2,32,324,4,3] the output should be 10 +For lst = [1,0,1,8,2,4597,2,1,3,40,1,2,1,2,4,2,5,1] the output should be 25 +For lst = [1,3,1,32,5107,34,83278,109,163,23,2323,32,30,1,9,3] the output should be 13 +For lst = [0,724,32,71,99,32,6,0,5,91,83,0,5,6] the output should be 11 +For lst = [0,81,12,3,1,21] the output should be 3 +For lst = [0,8,1,2,1,7] the output should be 7 + diff --git a/benches/HumanEvalPrompts/smallest_change.prompt b/benches/HumanEvalPrompts/smallest_change.prompt new file mode 100644 index 0000000..6f612c5 --- /dev/null +++ b/benches/HumanEvalPrompts/smallest_change.prompt @@ -0,0 +1,10 @@ + +Given an array arr of integers, find the minimum number of elements that +need to be changed to make the array palindromic. A palindromic array is an array that +is read the same backwards and forwards. In one change, you can change one element to any other element. + +For example: +smallest_change([1,2,3,5,4,7,9,6]) == 4 +smallest_change([1, 2, 3, 4, 3, 2, 2]) == 1 +smallest_change([1, 2, 3, 2, 1]) == 0 + diff --git a/benches/HumanEvalPrompts/solution.prompt b/benches/HumanEvalPrompts/solution.prompt new file mode 100644 index 0000000..8d62797 --- /dev/null +++ b/benches/HumanEvalPrompts/solution.prompt @@ -0,0 +1,8 @@ +Given a non-empty list of integers, return the sum of all of the odd elements that are in even positions. + + +Examples +solution([5, 8, 7, 1]) ==> 12 +solution([3, 3, 3, 3, 3]) ==> 9 +solution([30, 13, 24, 321]) ==>0 + diff --git a/benches/HumanEvalPrompts/solve.prompt b/benches/HumanEvalPrompts/solve.prompt new file mode 100644 index 0000000..54bd9aa --- /dev/null +++ b/benches/HumanEvalPrompts/solve.prompt @@ -0,0 +1,10 @@ +You are given a string s. +if s[i] is a letter, reverse its case from lower to upper or vise versa, +otherwise keep it as it is. +If the string contains no letters, reverse the string. +The function should return the resulted string. +Examples +solve("1234") = "4321" +solve("ab") = "AB" +solve("#a@C") = "#A@c" + diff --git a/benches/HumanEvalPrompts/sort_array.prompt b/benches/HumanEvalPrompts/sort_array.prompt new file mode 100644 index 0000000..25d07ca --- /dev/null +++ b/benches/HumanEvalPrompts/sort_array.prompt @@ -0,0 +1,10 @@ + +In this Kata, you have to sort an array of non-negative integers according to +number of ones in their binary representation in ascending order. +For similar number of ones, sort based on decimal value. + +It must be implemented like this: +>>> sort_array([1, 5, 2, 3, 4]) == [1, 2, 3, 4, 5] +>>> sort_array([-2, -3, -4, -5, -6]) == [-6, -5, -4, -3, -2] +>>> sort_array([1, 0, 2, 3, 4]) [0, 1, 2, 3, 4] + diff --git a/benches/HumanEvalPrompts/sort_even.prompt b/benches/HumanEvalPrompts/sort_even.prompt new file mode 100644 index 0000000..ae2ad01 --- /dev/null +++ b/benches/HumanEvalPrompts/sort_even.prompt @@ -0,0 +1,8 @@ +This function takes a list l and returns a list l' such that +l' is identical to l in the odd indicies, while its values at the even indicies are equal +to the values of the even indicies of l, but sorted. +>>> sort_even([1, 2, 3]) +[1, 2, 3] +>>> sort_even([5, 6, 3, 4]) +[3, 6, 5, 4] + diff --git a/benches/HumanEvalPrompts/sort_numbers.prompt b/benches/HumanEvalPrompts/sort_numbers.prompt new file mode 100644 index 0000000..4097dcf --- /dev/null +++ b/benches/HumanEvalPrompts/sort_numbers.prompt @@ -0,0 +1,6 @@ + Input is a space-delimited string of numberals from 'zero' to 'nine'. +Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. +Return the string with numbers sorted from smallest to largest +>>> sort_numbers('three one five') +'one three five' + diff --git a/benches/HumanEvalPrompts/sort_third.prompt b/benches/HumanEvalPrompts/sort_third.prompt new file mode 100644 index 0000000..09cf1c0 --- /dev/null +++ b/benches/HumanEvalPrompts/sort_third.prompt @@ -0,0 +1,8 @@ +This function takes a list l and returns a list l' such that +l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal +to the values of the corresponding indicies of l, but sorted. +>>> sort_third([1, 2, 3]) +[1, 2, 3] +>>> sort_third([5, 6, 3, 4, 8, 9, 2]) +[2, 6, 3, 4, 8, 9, 5] + diff --git a/benches/HumanEvalPrompts/sorted_list_sum.prompt b/benches/HumanEvalPrompts/sorted_list_sum.prompt new file mode 100644 index 0000000..8b6f706 --- /dev/null +++ b/benches/HumanEvalPrompts/sorted_list_sum.prompt @@ -0,0 +1,14 @@ +Write a function that accepts a list of strings as a parameter, +deletes the strings that have odd lengths from it, +and returns the resulted list with a sorted order, +The list is always a list of strings and never an array of numbers, +and it may contain duplicates. +The order of the list should be ascending by length of each word, and you +should return the list sorted by that rule. +If two words have the same length, sort the list alphabetically. +The function should return a list of strings in sorted order. +You may assume that all words will have the same length. +For example: +assert list_sort(["aa", "a", "aaa"]) => ["aa"] +assert list_sort(["ab", "a", "aaa", "cd"]) => ["ab", "cd"] + diff --git a/benches/HumanEvalPrompts/specialFilter.prompt b/benches/HumanEvalPrompts/specialFilter.prompt new file mode 100644 index 0000000..37dea2b --- /dev/null +++ b/benches/HumanEvalPrompts/specialFilter.prompt @@ -0,0 +1,7 @@ +Write a function that takes an array of numbers as input and returns +the number of elements in the array that are greater than 10 and both +first and last digits of a number are odd (1, 3, 5, 7, 9). +For example: +specialFilter([15, -73, 14, -15]) => 1 +specialFilter([33, -2, -3, 45, 21, 109]) => 2 + diff --git a/benches/HumanEvalPrompts/special_factorial.prompt b/benches/HumanEvalPrompts/special_factorial.prompt new file mode 100644 index 0000000..a73a2ec --- /dev/null +++ b/benches/HumanEvalPrompts/special_factorial.prompt @@ -0,0 +1,11 @@ +The Brazilian factorial is defined as: +brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! +where n > 0 + +For example: +>>> special_factorial(4) +288 + +The function will receive an integer as input and should return the special +factorial of this integer. + diff --git a/benches/HumanEvalPrompts/split_words.prompt b/benches/HumanEvalPrompts/split_words.prompt new file mode 100644 index 0000000..2ab89e7 --- /dev/null +++ b/benches/HumanEvalPrompts/split_words.prompt @@ -0,0 +1,9 @@ + +Given a string of words, return a list of words split on whitespace, if no whitespaces exists in the text you +should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the +alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 +Examples +split_words("Hello world!") ➞ ["Hello", "world!"] +split_words("Hello,world!") ➞ ["Hello", "world!"] +split_words("abcdef") == 3 + diff --git a/benches/HumanEvalPrompts/starts_one_ends.prompt b/benches/HumanEvalPrompts/starts_one_ends.prompt new file mode 100644 index 0000000..6d53568 --- /dev/null +++ b/benches/HumanEvalPrompts/starts_one_ends.prompt @@ -0,0 +1,4 @@ + +Given a positive integer n, return the count of the numbers of n-digit +positive integers that start or end with 1. + diff --git a/benches/HumanEvalPrompts/strange_sort_list.prompt b/benches/HumanEvalPrompts/strange_sort_list.prompt new file mode 100644 index 0000000..e210952 --- /dev/null +++ b/benches/HumanEvalPrompts/strange_sort_list.prompt @@ -0,0 +1,10 @@ + +Given list of integers, return list in strange order. +Strange sorting, is when you start with the minimum value, +then maximum of the remaining integers, then minimum and so on. + +Examples: +strange_sort_list([1, 2, 3, 4]) == [1, 4, 2, 3] +strange_sort_list([5, 5, 5, 5]) == [5, 5, 5, 5] +strange_sort_list([]) == [] + diff --git a/benches/HumanEvalPrompts/string_sequence.prompt b/benches/HumanEvalPrompts/string_sequence.prompt new file mode 100644 index 0000000..d4eb6ee --- /dev/null +++ b/benches/HumanEvalPrompts/string_sequence.prompt @@ -0,0 +1,6 @@ + Return a string containing space-delimited numbers starting from 0 upto n inclusive. +>>> string_sequence(0) +'0' +>>> string_sequence(5) +'0 1 2 3 4 5' + diff --git a/benches/HumanEvalPrompts/string_to_md5.prompt b/benches/HumanEvalPrompts/string_to_md5.prompt new file mode 100644 index 0000000..96fa243 --- /dev/null +++ b/benches/HumanEvalPrompts/string_to_md5.prompt @@ -0,0 +1,6 @@ + +Given a string 'text', return its md5 hash equivalent string. +If 'text' is an empty string, return None. + +>>> string_to_md5('Hello world') == '3e25960a79dbc69b674cd4ec67a72c62' + diff --git a/benches/HumanEvalPrompts/string_xor.prompt b/benches/HumanEvalPrompts/string_xor.prompt new file mode 100644 index 0000000..e9b260c --- /dev/null +++ b/benches/HumanEvalPrompts/string_xor.prompt @@ -0,0 +1,5 @@ + Input are two strings a and b consisting only of 1s and 0s. +Perform binary XOR on these inputs and return result also as a string. +>>> string_xor('010', '110') +'100' + diff --git a/benches/HumanEvalPrompts/strlen.prompt b/benches/HumanEvalPrompts/strlen.prompt new file mode 100644 index 0000000..6b50089 --- /dev/null +++ b/benches/HumanEvalPrompts/strlen.prompt @@ -0,0 +1,6 @@ + Return length of given string +>>> strlen('') +0 +>>> strlen('abc') +3 + diff --git a/benches/HumanEvalPrompts/sum_product.prompt b/benches/HumanEvalPrompts/sum_product.prompt new file mode 100644 index 0000000..69c9cfa --- /dev/null +++ b/benches/HumanEvalPrompts/sum_product.prompt @@ -0,0 +1,7 @@ + For a given list of integers, return a tuple consisting of a sum and a product of all the integers in a list. +Empty sum should be equal to 0 and empty product should be equal to 1. +>>> sum_product([]) +(0, 1) +>>> sum_product([1, 2, 3, 4]) +(10, 24) + diff --git a/benches/HumanEvalPrompts/sum_squares.prompt b/benches/HumanEvalPrompts/sum_squares.prompt new file mode 100644 index 0000000..c47aa96 --- /dev/null +++ b/benches/HumanEvalPrompts/sum_squares.prompt @@ -0,0 +1,10 @@ +" +This function will take a list of integers. For all entries in the list, the function shall square the integer entry if its index is a +multiple of 3 and will cube the integer entry if its index is a multiple of 4 and not a multiple of 3. The function will not +change the entries in the list whose indexes are not a multiple of 3 or 4. The function shall then return the sum of all entries. + +Examples: +For lst = [1,2,3] the output should be 6 +For lst = [] the output should be 0 +For lst = [-1,-5,2,-1,-5] the output should be -126 + diff --git a/benches/HumanEvalPrompts/sum_to_n.prompt b/benches/HumanEvalPrompts/sum_to_n.prompt new file mode 100644 index 0000000..fde4633 --- /dev/null +++ b/benches/HumanEvalPrompts/sum_to_n.prompt @@ -0,0 +1,12 @@ +sum_to_n is a function that sums numbers from 1 to n. +>>> sum_to_n(30) +465 +>>> sum_to_n(100) +5050 +>>> sum_to_n(5) +15 +>>> sum_to_n(10) +55 +>>> sum_to_n(1) +1 + diff --git a/benches/HumanEvalPrompts/total_match.prompt b/benches/HumanEvalPrompts/total_match.prompt new file mode 100644 index 0000000..52f9b28 --- /dev/null +++ b/benches/HumanEvalPrompts/total_match.prompt @@ -0,0 +1,13 @@ + +Write a function that accepts two lists of strings and returns the list that has +total number of chars in the all strings of the list less than the other list. + +if the two lists have the same number of chars, return the first list. + +Examples +total_match([], []) ➞ [] +total_match(['hi', 'admin'], ['hI', 'Hi']) ➞ ['hI', 'Hi'] +total_match(['hi', 'admin'], ['hi', 'hi', 'admin', 'project']) ➞ ['hi', 'admin'] +total_match(['hi', 'admin'], ['hI', 'hi', 'hi']) ➞ ['hI', 'hi', 'hi'] +total_match(['4'], ['1', '2', '3', '4', '5']) ➞ ['4'] + diff --git a/benches/HumanEvalPrompts/tri.prompt b/benches/HumanEvalPrompts/tri.prompt new file mode 100644 index 0000000..3357f59 --- /dev/null +++ b/benches/HumanEvalPrompts/tri.prompt @@ -0,0 +1,16 @@ +Everyone knows Fibonacci sequence, it was studied deeply by mathematicians in +the last couple centuries. However, what people don't know is Tribonacci sequence. +Tribonacci sequence is defined by the recurrence: +tri(1) = 3 +tri(n) = 1 + n / 2, if n is even. +tri(n) = tri(n - 1) + tri(n - 2) + tri(n + 1), if n is odd. +For example: +tri(2) = 1 + (2 / 2) = 2 +tri(4) = 3 +tri(3) = tri(2) + tri(1) + tri(4) + = 2 + 3 + 3 = 8 +You are given a non-negative integer number n, you have to a return a list of the +first n + 1 numbers of the Tribonacci sequence. +Examples: +tri(3) = [1, 3, 2, 8] + diff --git a/benches/HumanEvalPrompts/triangle_area.prompt b/benches/HumanEvalPrompts/triangle_area.prompt new file mode 100644 index 0000000..e80f79b --- /dev/null +++ b/benches/HumanEvalPrompts/triangle_area.prompt @@ -0,0 +1,10 @@ + +Given the lengths of the three sides of a triangle. Return the area of +the triangle rounded to 2 decimal points if the three sides form a valid triangle. +Otherwise return -1 +Three sides make a valid triangle when the sum of any two sides is greater +than the third side. +Example: +triangle_area(3, 4, 5) == 6.00 +triangle_area(1, 2, 10) == -1 + diff --git a/benches/HumanEvalPrompts/triples_sum_to_zero.prompt b/benches/HumanEvalPrompts/triples_sum_to_zero.prompt new file mode 100644 index 0000000..bdcfd0c --- /dev/null +++ b/benches/HumanEvalPrompts/triples_sum_to_zero.prompt @@ -0,0 +1,16 @@ + +triples_sum_to_zero takes a list of integers as an input. +it returns True if there are three distinct elements in the list that +sum to zero, and False otherwise. + +>>> triples_sum_to_zero([1, 3, 5, 0]) +False +>>> triples_sum_to_zero([1, 3, -2, 1]) +True +>>> triples_sum_to_zero([1, 2, 3, 7]) +False +>>> triples_sum_to_zero([2, 4, -5, 3, 9, 7]) +True +>>> triples_sum_to_zero([1]) +False + diff --git a/benches/HumanEvalPrompts/unique.prompt b/benches/HumanEvalPrompts/unique.prompt new file mode 100644 index 0000000..dfcfe05 --- /dev/null +++ b/benches/HumanEvalPrompts/unique.prompt @@ -0,0 +1,4 @@ +Return sorted unique elements in a list +>>> unique([5, 3, 5, 2, 3, 3, 9, 0, 123]) +[0, 2, 3, 5, 9, 123] + diff --git a/benches/HumanEvalPrompts/unique_digits.prompt b/benches/HumanEvalPrompts/unique_digits.prompt new file mode 100644 index 0000000..6c6e4a7 --- /dev/null +++ b/benches/HumanEvalPrompts/unique_digits.prompt @@ -0,0 +1,11 @@ +Given a list of positive integers x. return a sorted list of all +elements that hasn't any even digit. + +Note: Returned list should be sorted in increasing order. + +For example: +>>> unique_digits([15, 33, 1422, 1]) +[1, 15, 33] +>>> unique_digits([152, 323, 1422, 10]) +[] + diff --git a/benches/HumanEvalPrompts/valid_date.prompt b/benches/HumanEvalPrompts/valid_date.prompt new file mode 100644 index 0000000..b89a7a2 --- /dev/null +++ b/benches/HumanEvalPrompts/valid_date.prompt @@ -0,0 +1,19 @@ +You have to write a function which validates a given date string and +returns True if the date is valid otherwise False. +The date is valid if all of the following rules are satisfied: +1. The date string is not empty. +2. The number of days is not less than 1 or higher than 31 days for months 1,3,5,7,8,10,12. And the number of days is not less than 1 or higher than 30 days for months 4,6,9,11. And, the number of days is not less than 1 or higher than 29 for the month 2. +3. The months should not be less than 1 or higher than 12. +4. The date should be in the format: mm-dd-yyyy + +for example: +valid_date('03-11-2000') => True + +valid_date('15-01-2012') => False + +valid_date('04-0-2040') => False + +valid_date('06-04-2020') => True + +valid_date('06/04/2020') => False + diff --git a/benches/HumanEvalPrompts/vowels_count.prompt b/benches/HumanEvalPrompts/vowels_count.prompt new file mode 100644 index 0000000..79445a7 --- /dev/null +++ b/benches/HumanEvalPrompts/vowels_count.prompt @@ -0,0 +1,3 @@ + +Add more test cases. + diff --git a/benches/HumanEvalPrompts/will_it_fly.prompt b/benches/HumanEvalPrompts/will_it_fly.prompt new file mode 100644 index 0000000..1aa991b --- /dev/null +++ b/benches/HumanEvalPrompts/will_it_fly.prompt @@ -0,0 +1,17 @@ + +Write a function that returns True if the object q will fly, and False otherwise. +The object q will fly if it's balanced (it is a palindromic list) and the sum of its elements is less than or equal the maximum possible weight w. + +Example: +will_it_fly([1, 2], 5) ➞ False +# 1+2 is less than the maximum possible weight, but it's unbalanced. + +will_it_fly([3, 2, 3], 1) ➞ False +# it's balanced, but 3+2+3 is more than the maximum possible weight. + +will_it_fly([3, 2, 3], 9) ➞ True +# 3+2+3 is less than the maximum possible weight, and it's balanced. + +will_it_fly([3], 5) ➞ True +# 3 is less than the maximum possible weight, and it's balanced. + diff --git a/benches/HumanEvalPrompts/words_in_sentence.prompt b/benches/HumanEvalPrompts/words_in_sentence.prompt new file mode 100644 index 0000000..5ad27b9 --- /dev/null +++ b/benches/HumanEvalPrompts/words_in_sentence.prompt @@ -0,0 +1,19 @@ + +You are given a string representing a sentence, +the sentence contains some words separated by a space, +and you have to return a string that contains the words from the original sentence, +whose lengths are prime numbers, +the order of the words in the new string should be the same as the original one. + +Example 1: + Input: sentence = "This is a test" + Output: "is" + +Example 2: + Input: sentence = "lets go for swimming" + Output: "go for" + +Constraints: + * 1 <= len(sentence) <= 100 + * sentence contains only letters + diff --git a/benches/HumanEvalPrompts/words_string.prompt b/benches/HumanEvalPrompts/words_string.prompt new file mode 100644 index 0000000..0ab08a6 --- /dev/null +++ b/benches/HumanEvalPrompts/words_string.prompt @@ -0,0 +1,8 @@ + +You will be given a string of words separated by commas or spaces. Your task is +to split the string into words and return an array of the words. + +For example: +words_string("Hi, my name is John") == ["Hi", "my", "name", "is", "John"] +words_string("One, two, three, four, five, six") == ["One", "two", "three", "four", "five", "six"] + diff --git a/benches/HumanEvalPrompts/x_or_y.prompt b/benches/HumanEvalPrompts/x_or_y.prompt new file mode 100644 index 0000000..6f54b81 --- /dev/null +++ b/benches/HumanEvalPrompts/x_or_y.prompt @@ -0,0 +1,8 @@ +A simple program which should return the value of x if n is +a prime number and should return the value of y otherwise. + +Examples: +for x_or_y(7, 34, 12) == 34 +for x_or_y(15, 8, 5) == 5 + + diff --git a/benches/extract_prompt.py b/benches/extract_prompt.py new file mode 100755 index 0000000..dc64e06 --- /dev/null +++ b/benches/extract_prompt.py @@ -0,0 +1,18 @@ +#! /usr/bin/env python3 + +from sys import stdin +from textwrap import dedent + +full_test = stdin.read() + +prompt_start = full_test.find('"""') +if prompt_start == -1: + prompt_start = full_test.find("'''") + if prompt_start == -1: + raise ValueError("No prompt found") + prompt_end = full_test.find("'''", prompt_start + 3) +else: + prompt_end = full_test.find('"""', prompt_start + 3) + + +print(dedent(full_test[prompt_start + 3 : prompt_end].replace('\n ', '\n'))) diff --git a/gui/src/main.rs b/gui/src/main.rs index 20d9f44..317fe9b 100644 --- a/gui/src/main.rs +++ b/gui/src/main.rs @@ -72,6 +72,23 @@ enum BenchMode { #[default] Invariants, Generic, + Generate, +} + +impl BenchMode { + fn llm_generated_path(&self, path: &str) -> PathBuf { + let name = match self { + BenchMode::Invariants | BenchMode::Generic => basename(path).to_string(), + BenchMode::Generate => { + let base = basename(path); + base.chars() + .take(base.len() - 7) + .chain(".dfy".chars()) + .collect::() + } + }; + APP_DIRS.cache_dir().join("llm-generated").join(name) + } } impl Display for BenchMode { @@ -79,6 +96,7 @@ impl Display for BenchMode { match self { BenchMode::Invariants => write!(f, "invariants"), BenchMode::Generic => write!(f, "generic"), + BenchMode::Generate => write!(f, "generate"), } } } @@ -234,10 +252,8 @@ impl AppState { *output = Some(py_output); } - let llm_generated_path = APP_DIRS - .cache_dir() - .join("llm-generated") - .join(basename(path)); + let llm_generated_path = + settings.bench_type.llm_generated_path(path); let llm_code = std::fs::read_to_string(llm_generated_path).ok(); if let Ok(mut last_verified_code) = last_verified_code.write() { *last_verified_code = llm_code; @@ -465,6 +481,11 @@ impl AppState { "Invariants", ); ui.radio_value(&mut self.settings.bench_type, BenchMode::Generic, "Generic"); + ui.radio_value( + &mut self.settings.bench_type, + BenchMode::Generate, + "Generate", + ); }); ui.horizontal(|ui| { let max_rect = ui.max_rect(); diff --git a/prompts/dafny_by_prompt/ask_for_fixed.txt b/prompts/dafny_by_prompt/ask_for_fixed.txt new file mode 100644 index 0000000..7162bdf --- /dev/null +++ b/prompts/dafny_by_prompt/ask_for_fixed.txt @@ -0,0 +1,4 @@ +The following errors occurred during verification: +{error} + +Please fix the error by modifying the program accordingly. diff --git a/prompts/dafny_by_prompt/ask_for_fixed_had_errors.txt b/prompts/dafny_by_prompt/ask_for_fixed_had_errors.txt new file mode 100644 index 0000000..5ee549a --- /dev/null +++ b/prompts/dafny_by_prompt/ask_for_fixed_had_errors.txt @@ -0,0 +1,4 @@ +There are still some errors: +{error} + +Could you please fix them? If some postconditions fail repeatedly, try removing them for now. diff --git a/prompts/dafny_by_prompt/rewrite.txt b/prompts/dafny_by_prompt/rewrite.txt new file mode 100644 index 0000000..451b386 --- /dev/null +++ b/prompts/dafny_by_prompt/rewrite.txt @@ -0,0 +1,6 @@ +Following is a prompt that you should follow to generate a verified Dafny program. +Ensure that the program is syntactically correct and compiles without errors. +Keep the pre and post conditions simple and easy to vefify. + +Prompt: +{program} diff --git a/prompts/dafny_by_prompt/sys.txt b/prompts/dafny_by_prompt/sys.txt new file mode 100644 index 0000000..3ea69f5 --- /dev/null +++ b/prompts/dafny_by_prompt/sys.txt @@ -0,0 +1,33 @@ +You are an expert in Dafny. +You will be given tasks generating Dafny programs including precise docstrings and specifications. +Do not provide explanations. Respond only in dafny code. +Here is an example of a verified Dafny program: + +```dafny +predicate sorted(a: array) + reads a +{ + forall j, k :: 0 <= j < k < a.Length ==> a[j] <= a[k] +} +method BinarySearch(a: array, value: int) returns (index: int) + requires 0 <= a.Length && sorted(a) + ensures 0 <= index ==> index < a.Length && a[index] == value + ensures index < 0 ==> forall k :: 0 <= k < a.Length ==> a[k] != value +{ + var low, high := 0, a.Length; + while low < high + invariant 0 <= low <= high <= a.Length + invariant forall i :: + 0 <= i < a.Length && !(low <= i < high) ==> a[i] != value + { + var mid := (low + high) / 2; + if a[mid] < value { + low := mid + 1; + } else if value < a[mid] { + high := mid; + } else { + return mid; + } + } + return -1; +}``` diff --git a/prompts/dafny_by_prompt/timeout.txt b/prompts/dafny_by_prompt/timeout.txt new file mode 100644 index 0000000..f69fd69 --- /dev/null +++ b/prompts/dafny_by_prompt/timeout.txt @@ -0,0 +1,3 @@ +The verifier timed out during the verification. +This usually means that the provided invariants were too broad or were difficult to check. +Could you please try to improve the invariants and try again? diff --git a/verified_cogen/args.py b/verified_cogen/args.py index 0257ac2..77cf220 100644 --- a/verified_cogen/args.py +++ b/verified_cogen/args.py @@ -17,7 +17,7 @@ def get_args(): ) parser.add_argument( "--bench-type", - help="benchmark type, available: {invariants, generic}", + help="benchmark type, available: {invariants, generic, generate}", default="invariants", ) parser.add_argument("--temperature", help="model temperature", default=0, type=int) diff --git a/verified_cogen/llm/llm.py b/verified_cogen/llm/llm.py index 920f7ad..b7dbe21 100644 --- a/verified_cogen/llm/llm.py +++ b/verified_cogen/llm/llm.py @@ -1,3 +1,5 @@ +import logging +from http.client import RemoteDisconnected from typing import Optional from grazie.api.client.chat.prompt import ChatPrompt @@ -6,8 +8,11 @@ from grazie.api.client.llm_parameters import LLMParameters from grazie.api.client.parameters import Parameters from grazie.api.client.profiles import Profile -from verified_cogen.tools import extract_code_from_llm_output + import verified_cogen.llm.prompts as prompts +from verified_cogen.tools import extract_code_from_llm_output + +logger = logging.getLogger(__name__) class LLM: @@ -35,7 +40,9 @@ def __init__( system_prompt if system_prompt else prompts.sys_prompt(self.prompt_dir) ) - def _request(self, temperature: Optional[float] = None): + def _request(self, temperature: Optional[float] = None, tries: int = 5): + if tries == 0: + raise Exception("Exhausted tries to get response from Grazie API") if temperature is None: temperature = self.temperature prompt = ChatPrompt().add_system(self.system_prompt) @@ -51,11 +58,17 @@ def _request(self, temperature: Optional[float] = None): prompt = prompt.add_assistant(self.responses[current_response]) current_response += 1 - return self.grazie.chat( - chat=prompt, - profile=self.profile, - parameters={LLMParameters.Temperature: Parameters.FloatValue(temperature)}, - ) + try: + return self.grazie.chat( + chat=prompt, + profile=self.profile, + parameters={ + LLMParameters.Temperature: Parameters.FloatValue(temperature) + }, + ) + except RemoteDisconnected: + logger.warning("Grazie API is down, retrying...") + return self._request(temperature, tries - 1) def _make_request(self): response = self._request().content diff --git a/verified_cogen/main.py b/verified_cogen/main.py index 71dfaa9..5e47a77 100644 --- a/verified_cogen/main.py +++ b/verified_cogen/main.py @@ -4,6 +4,7 @@ from tqdm import tqdm from verified_cogen.llm import LLM +from verified_cogen.runners.generate import GenerateRunner from verified_cogen.runners.generic import GenericRunner from verified_cogen.runners.invariants import InvariantRunner from verified_cogen.tools import pprint_stat, rename_file, tabulate_list @@ -17,7 +18,7 @@ def run_once(files, args, runner, verifier, mode, is_once) -> tuple[int, int, int]: success, success_zero_tries, failed = [], [], [] - for file in tqdm(files): + for file in files: llm = LLM( args.grazie_token, args.llm_profile, @@ -78,7 +79,11 @@ def main(): if args.input is None and args.dir is None: args.input = input("Input file: ").strip() - runner = InvariantRunner if args.bench_type == "invariants" else GenericRunner + runner = { + "invariants": InvariantRunner, + "generic": GenericRunner, + "generate": GenerateRunner, + }[args.bench_type] verifier = Verifier(args.shell, args.verifier_command, args.verifier_timeout) if args.dir is not None: diff --git a/verified_cogen/runners/__init__.py b/verified_cogen/runners/__init__.py index f1c4750..75cbe67 100644 --- a/verified_cogen/runners/__init__.py +++ b/verified_cogen/runners/__init__.py @@ -32,18 +32,17 @@ def precheck(cls, prg: str, mode: Mode): pass @classmethod - def __invoke(cls, logger: Logger, llm: LLM, prg: str, mode: Mode) -> str: + def invoke(cls, logger: Logger, llm: LLM, prg: str, mode: Mode) -> str: logger.info("Invoking LLM") if mode.is_singlestep: inv_prg = cls.rewrite(llm, prg) else: - inv = cls.produce(llm, prg) - inv_prg = cls.insert(llm, prg, inv, mode) + raise ValueError(f"Unexpected mode: {mode}") logger.info("Invocation done") return inv_prg @classmethod - def __try_fixing( + def try_fixing( cls, logger: Logger, verifier: Verifier, @@ -99,7 +98,7 @@ def run_on_file( with open(file, "r") as f: prg = f.read() cls.precheck(prg, mode) - inv_prg = cls.__invoke(logger, llm, prg, mode) - return cls.__try_fixing( + inv_prg = cls.invoke(logger, llm, prg, mode) + return cls.try_fixing( logger, verifier, llm, total_tries, inv_prg, basename(file) ) diff --git a/verified_cogen/runners/generate.py b/verified_cogen/runners/generate.py new file mode 100644 index 0000000..1333ab4 --- /dev/null +++ b/verified_cogen/runners/generate.py @@ -0,0 +1,85 @@ +import logging +from typing import Optional +import pathlib + +from verified_cogen.tools.modes import Mode + +from verified_cogen.llm import LLM +from verified_cogen.runners import Runner +from verified_cogen.tools.verifier import Verifier +from verified_cogen.tools import basename, get_cache_dir + + +LLM_GENERATED_DIR = pathlib.Path(get_cache_dir()) / "llm-generated" + +logger = logging.getLogger(__name__) + + +class GenerateRunner(Runner): + @classmethod + def rewrite(cls, llm: LLM, prg: str) -> str: + return llm.rewrite(prg) + + @classmethod + def produce(cls, llm: LLM, prg: str) -> str: + raise ValueError("Produce not supported for generate") + + @classmethod + def insert(cls, llm: LLM, prg: str, checks: str, mode: Mode) -> str: + raise ValueError("Insert not supported for generate") + + @classmethod + def try_fixing( + cls, + logger: logging.Logger, + verifier: Verifier, + llm: LLM, + total_tries: int, + inv_prg: str, + name: str, + ) -> Optional[int]: + tries = total_tries + while tries > 0: + LLM_GENERATED_DIR.mkdir(parents=True, exist_ok=True) + output = LLM_GENERATED_DIR / f"{name[:-7]}.dfy" + with open(output, "w") as f: + f.write(inv_prg) + verification_result = verifier.verify(output) + if verification_result is None: + logger.info("Verification timed out") + tries -= 1 + if tries > 0: + inv_prg = llm.ask_for_timeout() + else: + verified_inv, out_inv, err_inv = verification_result + if verified_inv: + return total_tries - tries + 1 + else: + logger.info("Verification failed:") + logger.info(out_inv) + logger.info(err_inv) + logger.info("Retrying...") + tries -= 1 + if tries > 0: + inv_prg = llm.ask_for_fixed(out_inv + err_inv) + return None + + @classmethod + def run_on_file( + cls, + logger: logging.Logger, + verifier: Verifier, + mode: Mode, + llm: LLM, + total_tries: int, + file: str, + ) -> Optional[int]: + logger.info(f"Running on {file}") + + with open(file, "r") as f: + prg = f.read() + cls.precheck(prg, mode) + inv_prg = cls.invoke(logger, llm, prg, mode) + return cls.try_fixing( + logger, verifier, llm, total_tries, inv_prg, basename(file) + ) From c394415af43b8c51ea3da62a7a581c4df92daa89 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Tue, 6 Aug 2024 19:04:22 +0400 Subject: [PATCH 06/15] Get rid of tqdm for good --- poetry.lock | 22 +--------------------- pyproject.toml | 1 - verified_cogen/main.py | 2 -- 3 files changed, 1 insertion(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index ec12b33..152c79b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -764,26 +764,6 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -[[package]] -name = "tqdm" -version = "4.66.4" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tqdm-4.66.4-py3-none-any.whl", hash = "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644"}, - {file = "tqdm-4.66.4.tar.gz", hash = "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] -notebook = ["ipywidgets (>=6)"] -slack = ["slack-sdk"] -telegram = ["requests"] - [[package]] name = "typing-extensions" version = "4.12.2" @@ -815,4 +795,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f1796c097ac45ae8c9b50fc122b41ca5cb62b53beda0391fa807d13f9d193878" +content-hash = "b50c36e9a7de2af832b4fc2e6188aa6baef3879effa9096cc82f0e41c91c7d41" diff --git a/pyproject.toml b/pyproject.toml index cc462c4..fcb2748 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,6 @@ priority = "supplemental" [tool.poetry.dependencies] python = "^3.9" -tqdm = "^4.66.4" grazie-api-gateway-client = "^0.1.3" appdirs = "^1.4.4" diff --git a/verified_cogen/main.py b/verified_cogen/main.py index 5e47a77..abfc367 100644 --- a/verified_cogen/main.py +++ b/verified_cogen/main.py @@ -1,8 +1,6 @@ import logging import pathlib -from tqdm import tqdm - from verified_cogen.llm import LLM from verified_cogen.runners.generate import GenerateRunner from verified_cogen.runners.generic import GenericRunner From f679fb23f14b12f19bd16e561677d78ba093e0b9 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Wed, 7 Aug 2024 18:44:22 +0300 Subject: [PATCH 07/15] Add prompts with python solutions --- .../Strongest_Extension.prompt | 35 ++++++++++ .../HumanEvalPromptsWithPythonSol/add.prompt | 23 +++++++ .../add_elements.prompt | 23 +++++++ .../all_prefixes.prompt | 19 ++++++ .../anti_shuffle.prompt | 23 +++++++ .../any_int.prompt | 34 ++++++++++ .../below_threshold.prompt | 18 +++++ .../below_zero.prompt | 29 ++++++++ .../HumanEvalPromptsWithPythonSol/bf.prompt | 34 ++++++++++ .../by_length.prompt | 52 +++++++++++++++ .../can_arrange.prompt | 25 +++++++ .../car_race_collision.prompt | 16 +++++ .../change_base.prompt | 24 +++++++ .../check_dict_case.prompt | 45 +++++++++++++ .../check_if_last_char_is_a_letter.prompt | 28 ++++++++ .../choose_num.prompt | 24 +++++++ .../circular_shift.prompt | 23 +++++++ .../closest_integer.prompt | 47 +++++++++++++ .../common.prompt | 21 ++++++ .../compare.prompt | 24 +++++++ .../compare_one.prompt | 29 ++++++++ .../concatenate.prompt | 18 +++++ .../correct_bracketing.prompt | 47 +++++++++++++ .../count_distinct_characters.prompt | 15 +++++ .../count_nums.prompt | 28 ++++++++ .../count_up_to.prompt | 33 ++++++++++ .../count_upper.prompt | 25 +++++++ .../cycpattern_check.prompt | 32 +++++++++ .../decimal_to_binary.prompt | 23 +++++++ .../derivative.prompt | 17 +++++ .../digitSum.prompt | 30 +++++++++ .../digits.prompt | 28 ++++++++ .../do_algebra.prompt | 32 +++++++++ .../double_the_difference.prompt | 23 +++++++ .../HumanEvalPromptsWithPythonSol/eat.prompt | 46 +++++++++++++ .../encode.prompt | 27 ++++++++ .../encode_cyclic.prompt | 19 ++++++ .../encode_shift.prompt | 15 +++++ .../encrypt.prompt | 29 ++++++++ .../even_odd_count.prompt | 25 +++++++ .../even_odd_palindrome.prompt | 43 ++++++++++++ .../exchange.prompt | 33 ++++++++++ .../HumanEvalPromptsWithPythonSol/f.prompt | 26 ++++++++ .../factorize.prompt | 35 ++++++++++ .../HumanEvalPromptsWithPythonSol/fib.prompt | 22 +++++++ .../HumanEvalPromptsWithPythonSol/fib4.prompt | 32 +++++++++ .../fibfib.prompt | 29 ++++++++ .../file_name_check.prompt | 36 ++++++++++ .../filter_by_prefix.prompt | 18 +++++ .../filter_by_substring.prompt | 19 ++++++ .../filter_integers.prompt | 18 +++++ .../find_closest_elements.prompt | 34 ++++++++++ .../find_max.prompt | 20 ++++++ .../fix_spaces.prompt | 42 ++++++++++++ .../fizz_buzz.prompt | 26 ++++++++ .../flip_case.prompt | 12 ++++ .../fruit_distribution.prompt | 28 ++++++++ .../generate_integers.prompt | 26 ++++++++ .../get_closest_vowel.prompt | 34 ++++++++++ .../get_max_triples.prompt | 28 ++++++++ .../get_odd_collatz.prompt | 38 +++++++++++ .../get_positive.prompt | 15 +++++ .../get_row.prompt | 37 +++++++++++ .../greatest_common_divisor.prompt | 17 +++++ .../has_close_elements.prompt | 29 ++++++++ .../hex_key.prompt | 36 ++++++++++ .../histogram.prompt | 38 +++++++++++ .../how_many_times.prompt | 24 +++++++ .../incr_list.prompt | 15 +++++ .../int_to_mini_roman.prompt | 34 ++++++++++ .../intersection.prompt | 45 +++++++++++++ .../intersperse.prompt | 25 +++++++ .../is_bored.prompt | 23 +++++++ .../is_equal_to_sum_even.prompt | 16 +++++ .../is_happy.prompt | 32 +++++++++ .../is_multiply_prime.prompt | 28 ++++++++ .../is_nested.prompt | 44 +++++++++++++ .../is_palindrome.prompt | 45 +++++++++++++ .../is_prime.prompt | 35 ++++++++++ .../is_simple_power.prompt | 31 +++++++++ .../is_sorted.prompt | 42 ++++++++++++ .../iscube.prompt | 28 ++++++++ .../largest_divisor.prompt | 14 ++++ .../largest_prime_factor.prompt | 26 ++++++++ .../largest_smallest_integers.prompt | 26 ++++++++ .../longest.prompt | 28 ++++++++ .../make_a_pile.prompt | 24 +++++++ .../match_parens.prompt | 63 ++++++++++++++++++ .../max_element.prompt | 19 ++++++ .../max_fill.prompt | 47 +++++++++++++ .../maximum.prompt | 39 +++++++++++ .../mean_absolute_derivation.prompt | 17 +++++ .../median.prompt | 19 ++++++ .../minPath.prompt | 66 +++++++++++++++++++ .../minSubArraySum.prompt | 27 ++++++++ .../HumanEvalPromptsWithPythonSol/modp.prompt | 27 ++++++++ .../monotonic.prompt | 20 ++++++ .../move_one_ball.prompt | 48 ++++++++++++++ .../multiply.prompt | 21 ++++++ .../next_smallest.prompt | 25 +++++++ .../numerical_letter_grade.prompt | 65 ++++++++++++++++++ .../odd_count.prompt | 27 ++++++++ .../order_by_points.prompt | 26 ++++++++ .../pairs_sum_to_zero.prompt | 30 +++++++++ .../parse_music.prompt | 24 +++++++ .../parse_nested_parens.prompt | 31 +++++++++ .../pluck.prompt | 51 ++++++++++++++ .../HumanEvalPromptsWithPythonSol/poly.prompt | 34 ++++++++++ .../prime_fib.prompt | 40 +++++++++++ .../prime_length.prompt | 26 ++++++++ .../prod_signs.prompt | 25 +++++++ .../remove_duplicates.prompt | 18 +++++ .../remove_vowels.prompt | 28 ++++++++ .../rescale_to_unit.prompt | 18 +++++ .../reverse_delete.prompt | 21 ++++++ .../right_angle_triangle.prompt | 19 ++++++ .../rolling_max.prompt | 27 ++++++++ .../rounded_avg.prompt | 28 ++++++++ .../same_chars.prompt | 28 ++++++++ .../search.prompt | 30 +++++++++ .../select_words.prompt | 35 ++++++++++ .../separate_paren_groups.prompt | 35 ++++++++++ .../simplify.prompt | 28 ++++++++ .../skjkasdkd.prompt | 44 +++++++++++++ .../smallest_change.prompt | 27 ++++++++ .../solution.prompt | 20 ++++++ .../solve.prompt | 37 +++++++++++ .../sort_array.prompt | 36 ++++++++++ .../sort_even.prompt | 25 +++++++ .../sort_numbers.prompt | 29 ++++++++ .../sort_third.prompt | 19 ++++++ .../sorted_list_sum.prompt | 29 ++++++++ .../specialFilter.prompt | 26 ++++++++ .../special_factorial.prompt | 25 +++++++ .../split_words.prompt | 24 +++++++ .../starts_one_ends.prompt | 10 +++ .../strange_sort_list.prompt | 28 ++++++++ .../string_sequence.prompt | 15 +++++ .../string_to_md5.prompt | 18 +++++ .../string_xor.prompt | 22 +++++++ .../strlen.prompt | 15 +++++ .../sum_product.prompt | 26 ++++++++ .../sum_squares.prompt | 45 +++++++++++++ .../sum_to_n.prompt | 24 +++++++ .../total_match.prompt | 40 +++++++++++ .../HumanEvalPromptsWithPythonSol/tri.prompt | 33 ++++++++++ .../triangle_area.prompt | 30 +++++++++ .../triples_sum_to_zero.prompt | 32 +++++++++ .../unique.prompt | 12 ++++ .../unique_digits.prompt | 29 ++++++++ .../valid_date.prompt | 47 +++++++++++++ .../vowels_count.prompt | 23 +++++++ .../will_it_fly.prompt | 38 +++++++++++ .../words_in_sentence.prompt | 39 +++++++++++ .../words_string.prompt | 31 +++++++++ .../x_or_y.prompt | 25 +++++++ 156 files changed, 4519 insertions(+) create mode 100644 benches/HumanEvalPromptsWithPythonSol/Strongest_Extension.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/add.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/add_elements.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/all_prefixes.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/anti_shuffle.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/any_int.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/below_threshold.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/below_zero.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/bf.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/by_length.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/can_arrange.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/car_race_collision.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/change_base.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/check_dict_case.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/check_if_last_char_is_a_letter.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/choose_num.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/circular_shift.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/closest_integer.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/common.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/compare.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/compare_one.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/concatenate.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/correct_bracketing.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/count_distinct_characters.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/count_nums.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/count_up_to.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/count_upper.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/cycpattern_check.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/decimal_to_binary.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/derivative.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/digitSum.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/digits.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/do_algebra.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/double_the_difference.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/eat.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/encode.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/encode_cyclic.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/encode_shift.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/encrypt.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/even_odd_count.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/even_odd_palindrome.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/exchange.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/f.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/factorize.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/fib.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/fib4.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/fibfib.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/file_name_check.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/filter_by_prefix.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/filter_by_substring.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/filter_integers.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/find_closest_elements.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/find_max.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/fix_spaces.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/fizz_buzz.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/flip_case.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/fruit_distribution.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/generate_integers.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/get_closest_vowel.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/get_max_triples.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/get_odd_collatz.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/get_positive.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/get_row.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/greatest_common_divisor.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/has_close_elements.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/hex_key.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/histogram.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/how_many_times.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/incr_list.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/int_to_mini_roman.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/intersection.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/intersperse.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/is_bored.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/is_equal_to_sum_even.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/is_happy.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/is_multiply_prime.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/is_nested.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/is_palindrome.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/is_prime.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/is_simple_power.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/is_sorted.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/iscube.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/largest_divisor.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/largest_prime_factor.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/largest_smallest_integers.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/longest.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/make_a_pile.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/match_parens.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/max_element.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/max_fill.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/maximum.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/mean_absolute_derivation.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/median.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/minPath.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/minSubArraySum.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/modp.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/monotonic.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/move_one_ball.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/multiply.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/next_smallest.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/numerical_letter_grade.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/odd_count.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/order_by_points.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/pairs_sum_to_zero.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/parse_music.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/parse_nested_parens.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/pluck.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/poly.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/prime_fib.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/prime_length.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/prod_signs.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/remove_duplicates.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/remove_vowels.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/rescale_to_unit.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/reverse_delete.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/right_angle_triangle.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/rolling_max.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/rounded_avg.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/same_chars.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/search.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/select_words.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/separate_paren_groups.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/simplify.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/skjkasdkd.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/smallest_change.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/solution.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/solve.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/sort_array.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/sort_even.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/sort_numbers.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/sort_third.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/sorted_list_sum.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/specialFilter.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/special_factorial.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/split_words.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/starts_one_ends.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/strange_sort_list.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/string_sequence.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/string_to_md5.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/string_xor.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/strlen.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/sum_product.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/sum_squares.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/sum_to_n.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/total_match.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/tri.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/triangle_area.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/triples_sum_to_zero.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/unique.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/unique_digits.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/valid_date.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/vowels_count.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/will_it_fly.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/words_in_sentence.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/words_string.prompt create mode 100644 benches/HumanEvalPromptsWithPythonSol/x_or_y.prompt diff --git a/benches/HumanEvalPromptsWithPythonSol/Strongest_Extension.prompt b/benches/HumanEvalPromptsWithPythonSol/Strongest_Extension.prompt new file mode 100644 index 0000000..3839c6b --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/Strongest_Extension.prompt @@ -0,0 +1,35 @@ +You will be given the name of a class (a string) and a list of extensions. +The extensions are to be used to load additional classes to the class. The +strength of the extension is as follows: Let CAP be the number of the uppercase +letters in the extension's name, and let SM be the number of lowercase letters +in the extension's name, the strength is given by the fraction CAP - SM. +You should find the strongest extension and return a string in this +format: ClassName.StrongestExtensionName. +If there are two or more extensions with the same strength, you should +choose the one that comes first in the list. +For example, if you are given "Slices" as the class and a list of the +extensions: ['SErviNGSliCes', 'Cheese', 'StuFfed'] then you should +return 'Slices.SErviNGSliCes' since 'SErviNGSliCes' is the strongest extension +(its strength is -1). +Example: +for Strongest_Extension('my_class', ['AA', 'Be', 'CC']) == 'my_class.AA' + +Here's a correct unverified solution in python: +```python +def Strongest_Extension(class_name, extensions): + strong = extensions[0] + my_val = len([x for x in extensions[0] if x.isalpha() and x.isupper()]) - len([x for x in extensions[0] if x.isalpha() and x.islower()]) + for s in extensions: + val = len([x for x in s if x.isalpha() and x.isupper()]) - len([x for x in s if x.isalpha() and x.islower()]) + if val > my_val: + strong = s + my_val = val + + ans = class_name + "." + strong + return ans + +def check(Strongest_Extension): + # Check some simple cases + assert Strongest_Extension('my_class', ['AA', 'Be', 'CC']) == 'my_class.AA' +check(Strongest_Extension) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/add.prompt b/benches/HumanEvalPromptsWithPythonSol/add.prompt new file mode 100644 index 0000000..327e3a7 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/add.prompt @@ -0,0 +1,23 @@ +Given a non-empty list of integers lst. add the even elements that are at odd indices.. + + +Examples: + add([4, 2, 6, 7]) ==> 2 + +Here's a correct unverified solution in python: +```python +def add(x: int, y: int): + return x + y +def check(add): + import random + assert add(2, 3) == 5 + assert add(5, 7) == 12 +check(add) +def add(lst): + return sum([lst[i] for i in range(1, len(lst), 2) if lst[i]%2 == 0]) +def check(add): + # Check some simple cases + assert add([4, 2, 6, 7]) == 2 + # Check some edge cases that are easy to work out by hand. +check(add) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/add_elements.prompt b/benches/HumanEvalPromptsWithPythonSol/add_elements.prompt new file mode 100644 index 0000000..09ff811 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/add_elements.prompt @@ -0,0 +1,23 @@ + +Given a non-empty array of integers arr and an integer k, return +the sum of the elements with at most two digits from the first k elements of arr. + +Example: + + Input: arr = [111,21,3,4000,5,6,7,8,9], k = 4 + Output: 24 # sum of 21 + 3 + +Constraints: + 1. 1 <= len(arr) <= 100 + 2. 1 <= k <= len(arr) + +Here's a correct unverified solution in python: +```python +def add_elements(arr, k): + return sum(elem for elem in arr[:k] if len(str(elem)) <= 2) +def check(add_elements): + # Check some simple cases + assert add_elements([111,21,3,4000,5,6,7,8,9], 4) == 24, "This prints if this assert fails 1 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. +check(add_elements) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/all_prefixes.prompt b/benches/HumanEvalPromptsWithPythonSol/all_prefixes.prompt new file mode 100644 index 0000000..762c89d --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/all_prefixes.prompt @@ -0,0 +1,19 @@ + Return list of all prefixes from shortest to longest of the input string +>>> all_prefixes('abc') +['a', 'ab', 'abc'] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def all_prefixes(string: str) -> List[str]: + result = [] + + for i in range(len(string)): + result.append(string[:i+1]) + return result +def check(all_prefixes): + assert all_prefixes('abc') == ['a', 'ab', 'abc'] +check(all_prefixes) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/anti_shuffle.prompt b/benches/HumanEvalPromptsWithPythonSol/anti_shuffle.prompt new file mode 100644 index 0000000..4e95f98 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/anti_shuffle.prompt @@ -0,0 +1,23 @@ + +Write a function that takes a string and returns an ordered version of it. +Ordered version of string, is a string where all words (separated by space) +are replaced by a new word where all the characters arranged in +ascending order based on ascii value. +Note: You should keep the order of words and blank spaces in the sentence. + +For example: +anti_shuffle('Hi') returns 'Hi' +anti_shuffle('hello') returns 'ehllo' +anti_shuffle('Hello World!!!') returns 'Hello !!!Wdlor' + +Here's a correct unverified solution in python: +```python +def anti_shuffle(s): + return ' '.join([''.join(sorted(list(i))) for i in s.split(' ')]) +def check(anti_shuffle): + # Check some simple cases + assert anti_shuffle('Hi') == 'Hi' + assert anti_shuffle('hello') == 'ehllo' + assert anti_shuffle('Hello World!!!') == 'Hello !!!Wdlor' +check(anti_shuffle) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/any_int.prompt b/benches/HumanEvalPromptsWithPythonSol/any_int.prompt new file mode 100644 index 0000000..cc8eba7 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/any_int.prompt @@ -0,0 +1,34 @@ + +Create a function that takes 3 numbers. +Returns true if one of the numbers is equal to the sum of the other two, and all numbers are integers. +Returns false in any other cases. + +Examples +any_int(5, 2, 7) ➞ True + +any_int(3, 2, 2) ➞ False + +any_int(3, -2, 1) ➞ True + +any_int(3.6, -2.2, 2) ➞ False + + + + +Here's a correct unverified solution in python: +```python +def any_int(x, y, z): + + if isinstance(x,int) and isinstance(y,int) and isinstance(z,int): + if (x+y==z) or (x+z==y) or (y+z==x): + return True + return False + return False +def check(any_int): + # Check some simple cases + assert any_int(5, 2, 7)==True, "This prints if this assert fails 1 (good for debugging!)" + assert any_int(3, 2, 2)==False, "This prints if this assert fails 2 (good for debugging!)" + assert any_int(3, -2, 1)==True, "This prints if this assert fails 5 (good for debugging!)" + assert any_int(3.6, -2.2, 2)==False, "This prints if this assert fails 6 (good for debugging!)" +check(any_int) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/below_threshold.prompt b/benches/HumanEvalPromptsWithPythonSol/below_threshold.prompt new file mode 100644 index 0000000..647babb --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/below_threshold.prompt @@ -0,0 +1,18 @@ +Return True if all numbers in the list l are below threshold t. +>>> below_threshold([1, 2, 4, 10], 100) +True +>>> below_threshold([1, 20, 4, 10], 5) +False + +Here's a correct unverified solution in python: +```python +def below_threshold(l: list, t: int): + for e in l: + if e >= t: + return False + return True +def check(below_threshold): + assert below_threshold([1, 2, 4, 10], 100) + assert not below_threshold([1, 20, 4, 10], 5) +check(below_threshold) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/below_zero.prompt b/benches/HumanEvalPromptsWithPythonSol/below_zero.prompt new file mode 100644 index 0000000..647bc56 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/below_zero.prompt @@ -0,0 +1,29 @@ + You're given a list of deposit and withdrawal operations on a bank account that starts with +zero balance. Your task is to detect if at any point the balance of account fallls below zero, and +at that point function should return True. Otherwise it should return False. +>>> below_zero([1, 2, 3]) +False +>>> below_zero([1, 2, -4, 5]) +True + +Here's a correct unverified solution in python: +```python +from typing import List + + +def below_zero(operations: List[int]) -> bool: + balance = 0 + + for op in operations: + balance += op + if balance < 0: + return True + + return False + + +def check(below_zero): + assert below_zero([1, 2, 3]) == False + assert below_zero([1, 2, -4, 5]) == True +check(below_zero) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/bf.prompt b/benches/HumanEvalPromptsWithPythonSol/bf.prompt new file mode 100644 index 0000000..cdeb9d0 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/bf.prompt @@ -0,0 +1,34 @@ + +There are eight planets in our solar system: the closerst to the Sun +is Mercury, the next one is Venus, then Earth, Mars, Jupiter, Saturn, +Uranus, Neptune. +Write a function that takes two planet names as strings planet1 and planet2. +The function should return a tuple containing all planets whose orbits are +located between the orbit of planet1 and the orbit of planet2, sorted by +the proximity to the sun. +The function should return an empty tuple if planet1 or planet2 +are not correct planet names. +Examples +bf("Jupiter", "Neptune") ==> ("Saturn", "Uranus") +bf("Earth", "Mercury") ==> ("Venus") +bf("Mercury", "Uranus") ==> ("Venus", "Earth", "Mars", "Jupiter", "Saturn") + +Here's a correct unverified solution in python: +```python +def bf(planet1, planet2): + planet_names = ("Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune") + if planet1 not in planet_names or planet2 not in planet_names or planet1 == planet2: + return () + planet1_index = planet_names.index(planet1) + planet2_index = planet_names.index(planet2) + if planet1_index < planet2_index: + return (planet_names[planet1_index + 1: planet2_index]) + else: + return (planet_names[planet2_index + 1 : planet1_index]) +def check(bf): + # Check some simple cases + assert bf("Jupiter", "Neptune") == ("Saturn", "Uranus"), "First test error: " + str(len(bf("Jupiter", "Neptune"))) + assert bf("Earth", "Mercury") == ("Venus",), "Second test error: " + str(bf("Earth", "Mercury")) + assert bf("Mercury", "Uranus") == ("Venus", "Earth", "Mars", "Jupiter", "Saturn"), "Third test error: " + str(bf("Mercury", "Uranus")) +check(bf) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/by_length.prompt b/benches/HumanEvalPromptsWithPythonSol/by_length.prompt new file mode 100644 index 0000000..3fe9bea --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/by_length.prompt @@ -0,0 +1,52 @@ + +Given an array of integers, sort the integers that are between 1 and 9 inclusive, +reverse the resulting array, and then replace each digit by its corresponding name from +"One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine". + +For example: + arr = [2, 1, 1, 4, 5, 8, 2, 3] + -> sort arr -> [1, 1, 2, 2, 3, 4, 5, 8] + -> reverse arr -> [8, 5, 4, 3, 2, 2, 1, 1] + return ["Eight", "Five", "Four", "Three", "Two", "Two", "One", "One"] + + If the array is empty, return an empty array: + arr = [] + return [] + + If the array has any strange number ignore it: + arr = [1, -1 , 55] + -> sort arr -> [-1, 1, 55] + -> reverse arr -> [55, 1, -1] + return = ['One'] + +Here's a correct unverified solution in python: +```python +def by_length(arr): + dic = { + 1: "One", + 2: "Two", + 3: "Three", + 4: "Four", + 5: "Five", + 6: "Six", + 7: "Seven", + 8: "Eight", + 9: "Nine", + } + sorted_arr = sorted(arr, reverse=True) + new_arr = [] + for var in sorted_arr: + try: + new_arr.append(dic[var]) + except: + pass + return new_arr +def check(by_length): + # Check some simple cases + assert True, "This prints if this assert fails 1 (good for debugging!)" + assert by_length([2, 1, 1, 4, 5, 8, 2, 3]) == ["Eight", "Five", "Four", "Three", "Two", "Two", "One", "One"], "Error" + assert by_length([]) == [], "Error" + assert by_length([1, -1 , 55]) == ['One'], "Error" + # Check some edge cases that are easy to work out by hand. +check(by_length) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/can_arrange.prompt b/benches/HumanEvalPromptsWithPythonSol/can_arrange.prompt new file mode 100644 index 0000000..12ce1df --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/can_arrange.prompt @@ -0,0 +1,25 @@ +Create a function which returns the largest index of an element which +is not greater than or equal to the element immediately preceding it. If +no such element exists then return -1. The given array will not contain +duplicate values. + +Examples: +can_arrange([1,2,4,3,5]) = 3 +can_arrange([1,2,3]) = -1 + +Here's a correct unverified solution in python: +```python +def can_arrange(arr): + ind=-1 + i=1 + while i>> change_base(8, 3) +'22' +>>> change_base(8, 2) +'1000' +>>> change_base(7, 2) +'111' + +Here's a correct unverified solution in python: +```python +def change_base(x: int, base: int): + ret = "" + while x > 0: + ret = str(x % base) + ret + x //= base + return ret +def check(change_base): + assert change_base(8, 3) == "22" + assert change_base(8, 2) == "1000" + assert change_base(7, 2) == "111" +check(change_base) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/check_dict_case.prompt b/benches/HumanEvalPromptsWithPythonSol/check_dict_case.prompt new file mode 100644 index 0000000..f75110e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/check_dict_case.prompt @@ -0,0 +1,45 @@ + +Given a dictionary, return True if all keys are strings in lower +case or all keys are strings in upper case, else return False. +The function should return False is the given dictionary is empty. +Examples: +check_dict_case({"a":"apple", "b":"banana"}) should return True. +check_dict_case({"a":"apple", "A":"banana", "B":"banana"}) should return False. +check_dict_case({"a":"apple", 8:"banana", "a":"apple"}) should return False. +check_dict_case({"Name":"John", "Age":"36", "City":"Houston"}) should return False. +check_dict_case({"STATE":"NC", "ZIP":"12345" }) should return True. + +Here's a correct unverified solution in python: +```python +def check_dict_case(dict): + if len(dict.keys()) == 0: + return False + else: + state = "start" + for key in dict.keys(): + + if isinstance(key, str) == False: + state = "mixed" + break + if state == "start": + if key.isupper(): + state = "upper" + elif key.islower(): + state = "lower" + else: + break + elif (state == "upper" and not key.isupper()) or (state == "lower" and not key.islower()): + state = "mixed" + break + else: + break + return state == "upper" or state == "lower" +def check(check_dict_case): + # Check some simple cases + assert check_dict_case({"p":"pineapple", "b":"banana"}) == True, "First test error: " + str(check_dict_case({"p":"pineapple", "b":"banana"})) + assert check_dict_case({"p":"pineapple", "A":"banana", "B":"banana"}) == False, "Second test error: " + str(check_dict_case({"p":"pineapple", "A":"banana", "B":"banana"})) + assert check_dict_case({"p":"pineapple", 8:"banana", "a":"apple"}) == False, "Third test error: " + str(check_dict_case({"p":"pineapple", 5:"banana", "a":"apple"})) + assert check_dict_case({"Name":"John", "Age":"36", "City":"Houston"}) == False, "Fourth test error: " + str(check_dict_case({"Name":"John", "Age":"36", "City":"Houston"})) + assert check_dict_case({"STATE":"NC", "ZIP":"12345" }) == True, "Fifth test error: " + str(check_dict_case({"STATE":"NC", "ZIP":"12345" })) +check(check_dict_case) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/check_if_last_char_is_a_letter.prompt b/benches/HumanEvalPromptsWithPythonSol/check_if_last_char_is_a_letter.prompt new file mode 100644 index 0000000..2e22055 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/check_if_last_char_is_a_letter.prompt @@ -0,0 +1,28 @@ + +Create a function that returns True if the last character +of a given string is an alphabetical character and is not +a part of a word, and False otherwise. +Note: "word" is a group of characters separated by space. + +Examples: +check_if_last_char_is_a_letter("apple pie") ➞ False +check_if_last_char_is_a_letter("apple pi e") ➞ True +check_if_last_char_is_a_letter("apple pi e ") ➞ False +check_if_last_char_is_a_letter("") ➞ False + +Here's a correct unverified solution in python: +```python +def check_if_last_char_is_a_letter(txt): + + check = txt.split(' ')[-1] + return True if len(check) == 1 and (97 <= ord(check.lower()) <= 122) else False +def check(check_if_last_char_is_a_letter): + # Check some simple cases + assert check_if_last_char_is_a_letter("apple pi e") == True + assert check_if_last_char_is_a_letter("") == False + assert check_if_last_char_is_a_letter("apple pie") == False + assert check_if_last_char_is_a_letter("apple pi e ") == False + # Check some edge cases that are easy to work out by hand. + assert True +check(check_if_last_char_is_a_letter) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/choose_num.prompt b/benches/HumanEvalPromptsWithPythonSol/choose_num.prompt new file mode 100644 index 0000000..de27c44 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/choose_num.prompt @@ -0,0 +1,24 @@ +This function takes two positive numbers x and y and returns the +biggest even integer number that is in the range [x, y] inclusive. If +there's no such number, then the function should return -1. + +For example: +choose_num(12, 15) = 14 +choose_num(13, 12) = -1 + +Here's a correct unverified solution in python: +```python +def choose_num(x, y): + if x > y: + return -1 + if y % 2 == 0: + return y + if x == y: + return -1 + return y - 1 +def check(choose_num): + # Check some simple cases + assert choose_num(12, 15) == 14 + assert choose_num(13, 12) == -1 +check(choose_num) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/circular_shift.prompt b/benches/HumanEvalPromptsWithPythonSol/circular_shift.prompt new file mode 100644 index 0000000..082d58f --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/circular_shift.prompt @@ -0,0 +1,23 @@ +Circular shift the digits of the integer x, shift the digits right by shift +and return the result as a string. +If shift > number of digits, return digits reversed. +>>> circular_shift(12, 1) +"21" +>>> circular_shift(12, 2) +"12" + +Here's a correct unverified solution in python: +```python +def circular_shift(x, shift): + s = str(x) + if shift > len(s): + return s[::-1] + else: + return s[len(s) - shift:] + s[:len(s) - shift] +def check(circular_shift): + # Check some simple cases + assert circular_shift(12, 2) == "12" + assert circular_shift(12, 1) == "21", "This prints if this assert fails 1 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. +check(circular_shift) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/closest_integer.prompt b/benches/HumanEvalPromptsWithPythonSol/closest_integer.prompt new file mode 100644 index 0000000..0c96927 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/closest_integer.prompt @@ -0,0 +1,47 @@ + +Create a function that takes a value (string) representing a number +and returns the closest integer to it. If the number is equidistant +from two integers, round it away from zero. + +Examples +>>> closest_integer("10") +10 +>>> closest_integer("15.3") +15 + +Note: +Rounding away from zero means that if the given number is equidistant +from two integers, the one you should return is the one that is the +farthest from zero. For example closest_integer("14.5") should +return 15 and closest_integer("-14.5") should return -15. + +Here's a correct unverified solution in python: +```python +def closest_integer(value): + from math import ceil, floor + + if value.count('.') == 1: + # remove trailing zeros + while (value[-1] == '0'): + value = value[:-1] + + num = float(value) + if value[-2:] == '.5': + if num > 0: + res = ceil(num) + else: + res = floor(num) + elif len(value) > 0: + res = int(round(num)) + else: + res = 0 + + return res + +def check(closest_integer): + # Check some simple cases + assert closest_integer("10") == 10, "Test 1" + assert closest_integer("15.3") == 15, "Test 3" + # Check some edge cases that are easy to work out by hand. +check(closest_integer) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/common.prompt b/benches/HumanEvalPromptsWithPythonSol/common.prompt new file mode 100644 index 0000000..8a228e8 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/common.prompt @@ -0,0 +1,21 @@ +Return sorted unique common elements for two lists. +>>> common([1, 4, 3, 34, 653, 2, 5], [5, 7, 1, 5, 9, 653, 121]) +[1, 5, 653] +>>> common([5, 3, 2, 8], [3, 2]) +[2, 3] + + +Here's a correct unverified solution in python: +```python +def common(l1: list, l2: list): + ret = set() + for e1 in l1: + for e2 in l2: + if e1 == e2: + ret.add(e1) + return sorted(list(ret)) +def check(common): + assert common([1, 4, 3, 34, 653, 2, 5], [5, 7, 1, 5, 9, 653, 121]) == [1, 5, 653] + assert common([5, 3, 2, 8], [3, 2]) == [2, 3] +check(common) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/compare.prompt b/benches/HumanEvalPromptsWithPythonSol/compare.prompt new file mode 100644 index 0000000..912f28f --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/compare.prompt @@ -0,0 +1,24 @@ +I think we all remember that feeling when the result of some long-awaited +event is finally known. The feelings and thoughts you have at that moment are +definitely worth noting down and comparing. +Your task is to determine if a person correctly guessed the results of a number of matches. +You are given two arrays of scores and guesses of equal length, where each index shows a match. +Return an array of the same length denoting how far off each guess was. If they have guessed correctly, +the value is 0, and if not, the value is the absolute difference between the guess and the score. + + +example: + +compare([1,2,3,4,5,1],[1,2,3,4,2,-2]) -> [0,0,0,0,3,3] +compare([0,5,0,0,0,4],[4,1,1,0,0,-2]) -> [4,4,1,0,0,6] + +Here's a correct unverified solution in python: +```python +def compare(game,guess): + return [abs(x-y) for x,y in zip(game,guess)] +def check(compare): + # Check some simple cases + assert compare([1,2,3,4,5,1],[1,2,3,4,2,-2])==[0,0,0,0,3,3], "This prints if this assert fails 1 (good for debugging!)" + assert compare([0,5,0,0,0,4],[4,1,1,0,0,-2])==[4,4,1,0,0,6] +check(compare) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/compare_one.prompt b/benches/HumanEvalPromptsWithPythonSol/compare_one.prompt new file mode 100644 index 0000000..a2663f7 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/compare_one.prompt @@ -0,0 +1,29 @@ + +Create a function that takes integers, floats, or strings representing +real numbers, and returns the larger variable in its given variable type. +Return None if the values are equal. +Note: If a real number is represented as a string, the floating point might be . or , + +compare_one(1, 2.5) ➞ 2.5 +compare_one(1, "2,3") ➞ "2,3" +compare_one("5,1", "6") ➞ "6" +compare_one("1", 1) ➞ None + +Here's a correct unverified solution in python: +```python +def compare_one(a, b): + temp_a, temp_b = a, b + if isinstance(temp_a, str): temp_a = temp_a.replace(',','.') + if isinstance(temp_b, str): temp_b = temp_b.replace(',','.') + if float(temp_a) == float(temp_b): return None + return a if float(temp_a) > float(temp_b) else b +def check(compare_one): + # Check some simple cases + assert compare_one(1, 2.5) == 2.5 + assert compare_one(1, "2,3") == "2,3" + assert compare_one("5,1", "6") == "6" + assert compare_one("1", 1) == None + # Check some edge cases that are easy to work out by hand. + assert True +check(compare_one) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/concatenate.prompt b/benches/HumanEvalPromptsWithPythonSol/concatenate.prompt new file mode 100644 index 0000000..2094286 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/concatenate.prompt @@ -0,0 +1,18 @@ + Concatenate list of strings into a single string +>>> concatenate([]) +'' +>>> concatenate(['a', 'b', 'c']) +'abc' + +Here's a correct unverified solution in python: +```python +from typing import List + + +def concatenate(strings: List[str]) -> str: + return ''.join(strings) +def check(concatenate): + assert concatenate([]) == '' + assert concatenate(['a', 'b', 'c']) == 'abc' +check(concatenate) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/correct_bracketing.prompt b/benches/HumanEvalPromptsWithPythonSol/correct_bracketing.prompt new file mode 100644 index 0000000..1cb405c --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/correct_bracketing.prompt @@ -0,0 +1,47 @@ + brackets is a string of "(" and ")". +return True if every opening bracket has a corresponding closing bracket. + +>>> correct_bracketing("(") +False +>>> correct_bracketing("()") +True +>>> correct_bracketing("(()())") +True +>>> correct_bracketing(")(()") +False + +Here's a correct unverified solution in python: +```python +def correct_bracketing(brackets: str): + depth = 0 + for b in brackets: + if b == "<": + depth += 1 + else: + depth -= 1 + if depth < 0: + return False + return depth == 0 +def check(correct_bracketing): + assert correct_bracketing("<>") + assert correct_bracketing("<<><>>") + assert not correct_bracketing("><<>") + assert not correct_bracketing("<") +check(correct_bracketing) +def correct_bracketing(brackets: str): + depth = 0 + for b in brackets: + if b == "(": + depth += 1 + else: + depth -= 1 + if depth < 0: + return False + return depth == 0 +def check(correct_bracketing): + assert correct_bracketing("()") + assert correct_bracketing("(()())") + assert not correct_bracketing(")(()") + assert not correct_bracketing("(") +check(correct_bracketing) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/count_distinct_characters.prompt b/benches/HumanEvalPromptsWithPythonSol/count_distinct_characters.prompt new file mode 100644 index 0000000..e59f262 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/count_distinct_characters.prompt @@ -0,0 +1,15 @@ + Given a string, find out how many distinct characters (regardless of case) does it consist of +>>> count_distinct_characters('xyzXYZ') +3 +>>> count_distinct_characters('Jerry') +4 + +Here's a correct unverified solution in python: +```python +def count_distinct_characters(string: str) -> int: + return len(set(string.lower())) +def check(count_distinct_characters): + assert count_distinct_characters('xyzXYZ') == 3 + assert count_distinct_characters('Jerry') == 4 +check(count_distinct_characters) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/count_nums.prompt b/benches/HumanEvalPromptsWithPythonSol/count_nums.prompt new file mode 100644 index 0000000..73f9e53 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/count_nums.prompt @@ -0,0 +1,28 @@ + +Write a function count_nums which takes an array of integers and returns +the number of elements which has a sum of digits > 0. +If a number is negative, then its first signed digit will be negative: +e.g. -123 has signed digits -1, 2, and 3. +>>> count_nums([]) == 0 +>>> count_nums([-1, 11, -11]) == 1 +>>> count_nums([1, 1, 2]) == 3 + +Here's a correct unverified solution in python: +```python +def count_nums(arr): + def digits_sum(n): + neg = 1 + if n < 0: n, neg = -1 * n, -1 + n = [int(i) for i in str(n)] + n[0] = n[0] * neg + return sum(n) + return len(list(filter(lambda x: x > 0, [digits_sum(i) for i in arr]))) +def check(count_nums): + # Check some simple cases + assert count_nums([]) == 0 + assert count_nums([-1, 11, -11]) == 1 + assert count_nums([1, 1, 2]) == 3 + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(count_nums) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/count_up_to.prompt b/benches/HumanEvalPromptsWithPythonSol/count_up_to.prompt new file mode 100644 index 0000000..7e55bed --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/count_up_to.prompt @@ -0,0 +1,33 @@ +Implement a function that takes an non-negative integer and returns an array of the first n +integers that are prime numbers and less than n. +for example: +count_up_to(5) => [2,3] +count_up_to(11) => [2,3,5,7] +count_up_to(0) => [] +count_up_to(20) => [2,3,5,7,11,13,17,19] +count_up_to(1) => [] +count_up_to(18) => [2,3,5,7,11,13,17] + +Here's a correct unverified solution in python: +```python +def count_up_to(n): + primes = [] + for i in range(2, n): + is_prime = True + for j in range(2, i): + if i % j == 0: + is_prime = False + break + if is_prime: + primes.append(i) + return primes + +def check(count_up_to): + assert count_up_to(5) == [2,3] + assert count_up_to(11) == [2,3,5,7] + assert count_up_to(0) == [] + assert count_up_to(20) == [2,3,5,7,11,13,17,19] + assert count_up_to(1) == [] + assert count_up_to(18) == [2,3,5,7,11,13,17] +check(count_up_to) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/count_upper.prompt b/benches/HumanEvalPromptsWithPythonSol/count_upper.prompt new file mode 100644 index 0000000..9c0de1e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/count_upper.prompt @@ -0,0 +1,25 @@ + +Given a string s, count the number of uppercase vowels in even indices. + +For example: +count_upper('aBCdEf') returns 1 +count_upper('abcdefg') returns 0 +count_upper('dBBE') returns 0 + +Here's a correct unverified solution in python: +```python +def count_upper(s): + count = 0 + for i in range(0,len(s),2): + if s[i] in "AEIOU": + count += 1 + return count +def check(count_upper): + # Check some simple cases + assert count_upper('aBCdEf') == 1 + assert count_upper('abcdefg') == 0 + assert count_upper('dBBE') == 0 + # Check some edge cases that are easy to work out by hand. + assert True +check(count_upper) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/cycpattern_check.prompt b/benches/HumanEvalPromptsWithPythonSol/cycpattern_check.prompt new file mode 100644 index 0000000..26c4079 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/cycpattern_check.prompt @@ -0,0 +1,32 @@ +You are given 2 words. You need to return True if the second word or any of its rotations is a substring in the first word +cycpattern_check("abcd","abd") => False +cycpattern_check("hello","ell") => True +cycpattern_check("whassup","psus") => False +cycpattern_check("abab","baa") => True +cycpattern_check("efef","eeff") => False +cycpattern_check("himenss","simen") => True + + +Here's a correct unverified solution in python: +```python +def cycpattern_check(a , b): + l = len(b) + pat = b + b + for i in range(len(a) - l + 1): + for j in range(l + 1): + if a[i:i+l] == pat[j:j+l]: + return True + return False +def check(cycpattern_check): + # Check some simple cases + #assert True, "This prints if this assert fails 1 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. + #assert True, "This prints if this assert fails 2 (also good for debugging!)" + assert cycpattern_check("abcd","abd") == False , "test #0" + assert cycpattern_check("hello","ell") == True , "test #1" + assert cycpattern_check("whassup","psus") == False , "test #2" + assert cycpattern_check("abab","baa") == True , "test #3" + assert cycpattern_check("efef","eeff") == False , "test #4" + assert cycpattern_check("himenss","simen") == True , "test #5" +check(cycpattern_check) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/decimal_to_binary.prompt b/benches/HumanEvalPromptsWithPythonSol/decimal_to_binary.prompt new file mode 100644 index 0000000..ff0cf11 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/decimal_to_binary.prompt @@ -0,0 +1,23 @@ +You will be given a number in decimal form and your task is to convert it to +binary format. The function should return a string, with each character representing a binary +number. Each character in the string will be '0' or '1'. + +There will be an extra couple of characters 'db' at the beginning and at the end of the string. +The extra characters are there to help with the format. + +Examples: +decimal_to_binary(15) # returns "db1111db" +decimal_to_binary(32) # returns "db100000db" + +Here's a correct unverified solution in python: +```python +def decimal_to_binary(decimal): + return "db" + bin(decimal)[2:] + "db" +def check(decimal_to_binary): + # Check some simple cases + assert decimal_to_binary(32) == "db100000db" + assert decimal_to_binary(15) == "db1111db", "This prints if this assert fails 1 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(decimal_to_binary) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/derivative.prompt b/benches/HumanEvalPromptsWithPythonSol/derivative.prompt new file mode 100644 index 0000000..39e2764 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/derivative.prompt @@ -0,0 +1,17 @@ + xs represent coefficients of a polynomial. +xs[0] + xs[1] * x + xs[2] * x^2 + .... + Return derivative of this polynomial in the same form. +>>> derivative([3, 1, 2, 4, 5]) +[1, 4, 12, 20] +>>> derivative([1, 2, 3]) +[2, 6] + +Here's a correct unverified solution in python: +```python +def derivative(xs: list): + return [(i * x) for i, x in enumerate(xs)][1:] +def check(derivative): + assert derivative([3, 1, 2, 4, 5]) == [1, 4, 12, 20] + assert derivative([1, 2, 3]) == [2, 6] +check(derivative) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/digitSum.prompt b/benches/HumanEvalPromptsWithPythonSol/digitSum.prompt new file mode 100644 index 0000000..5bd2e40 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/digitSum.prompt @@ -0,0 +1,30 @@ +Task +Write a function that takes a string as input and returns the sum of the upper characters only' +ASCII codes. + +Examples: + digitSum("") => 0 + digitSum("abAB") => 131 + digitSum("abcCd") => 67 + digitSum("helloE") => 69 + digitSum("woArBld") => 131 + digitSum("aAaaaXa") => 153 + +Here's a correct unverified solution in python: +```python +def digitSum(s): + if s == "": return 0 + return sum(ord(char) if char.isupper() else 0 for char in s) +def check(digitSum): + # Check some simple cases + assert True, "This prints if this assert fails 1 (good for debugging!)" + assert digitSum("") == 0, "Error" + assert digitSum("abAB") == 131, "Error" + assert digitSum("abcCd") == 67, "Error" + assert digitSum("helloE") == 69, "Error" + assert digitSum("woArBld") == 131, "Error" + assert digitSum("aAaaaXa") == 153, "Error" + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(digitSum) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/digits.prompt b/benches/HumanEvalPromptsWithPythonSol/digits.prompt new file mode 100644 index 0000000..c72f962 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/digits.prompt @@ -0,0 +1,28 @@ +Given a positive integer n, return the product of the odd digits. +Return 0 if all digits are even. +For example: +digits(1) == 1 +digits(4) == 0 +digits(235) == 15 + +Here's a correct unverified solution in python: +```python +def digits(n): + product = 1 + odd_count = 0 + for digit in str(n): + int_digit = int(digit) + if int_digit%2 == 1: + product= product*int_digit + odd_count+=1 + if odd_count ==0: + return 0 + else: + return product +def check(digits): + # Check some simple cases + assert digits(1) == 1 + assert digits(4) == 0 + assert digits(235) ==15 +check(digits) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/do_algebra.prompt b/benches/HumanEvalPromptsWithPythonSol/do_algebra.prompt new file mode 100644 index 0000000..81f7c64 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/do_algebra.prompt @@ -0,0 +1,32 @@ + +Given two lists operator, and operand. The first list has basic algebra operations, and +the second list is a list of integers. Use the two given lists to build the algebric +expression and return the evaluation of this expression. + +The basic algebra operations: +Addition ( + ) +Subtraction ( - ) +Multiplication ( * ) +Floor division ( // ) +Exponentiation ( ** ) + +Example: +operator['+', '*', '-'] +array = [2, 3, 4, 5] +result = 2 + 3 * 4 - 5 +=> result = 9 + +Note: + The length of operator list is equal to the length of operand list minus one. + Operand is a list of of non-negative integers. + Operator list has at least one operator, and operand list has at least two operands. + + +Here's a correct unverified solution in python: +```python +def do_algebra(operator, operand): + expression = str(operand[0]) + for oprt, oprn in zip(operator, operand[1:]): + expression+= oprt + str(oprn) + return eval(expression) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/double_the_difference.prompt b/benches/HumanEvalPromptsWithPythonSol/double_the_difference.prompt new file mode 100644 index 0000000..bbbf2ea --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/double_the_difference.prompt @@ -0,0 +1,23 @@ + +Given a list of numbers, return the sum of squares of the numbers +in the list that are odd. Ignore numbers that are negative or not integers. + +double_the_difference([1, 3, 2, 0]) == 1 + 9 + 0 + 0 = 10 +double_the_difference([-1, -2, 0]) == 0 +double_the_difference([9, -2]) == 81 +double_the_difference([0]) == 0 + +If the input list is empty, return 0. + +Here's a correct unverified solution in python: +```python +def double_the_difference(lst): + return sum([i**2 for i in lst if i > 0 and i%2!=0 and "." not in str(i)]) +def check(double_the_difference): + # Check some simple cases + assert double_the_difference([1,3,2,0]) == 10 , "This prints if this assert fails 1 (good for debugging!)" + assert double_the_difference([-1,-2,0]) == 0 , "This prints if this assert fails 2 (good for debugging!)" + assert double_the_difference([9,-2]) == 81 , "This prints if this assert fails 3 (good for debugging!)" + assert double_the_difference([0]) == 0 , "This prints if this assert fails 4 (good for debugging!)" +check(double_the_difference) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/eat.prompt b/benches/HumanEvalPromptsWithPythonSol/eat.prompt new file mode 100644 index 0000000..e408135 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/eat.prompt @@ -0,0 +1,46 @@ + +You're a hungry rabbit, and you already have eaten a certain number of carrots, +but now you need to eat more carrots to complete the day's meals. +you should return an array of [ total number of eaten carrots after your meals, + the number of carrots left after your meals ] +if there are not enough remaining carrots, you will eat all remaining carrots, but will still be hungry. + +Example: +* eat(5, 6, 10) -> [11, 4] +* eat(4, 8, 9) -> [12, 1] +* eat(1, 10, 10) -> [11, 0] +* eat(2, 11, 5) -> [7, 0] + +Variables: +@number : integer + the number of carrots that you have eaten. +@need : integer + the number of carrots that you need to eat. +@remaining : integer + the number of remaining carrots thet exist in stock + +Constrain: +* 0 <= number <= 1000 +* 0 <= need <= 1000 +* 0 <= remaining <= 1000 + +Have fun :) + +Here's a correct unverified solution in python: +```python +def eat(number, need, remaining): + if(need <= remaining): + return [ number + need , remaining-need ] + else: + return [ number + remaining , 0] +def check(eat): + # Check some simple cases + assert True, "This prints if this assert fails 1 (good for debugging!)" + assert eat(5, 6, 10) == [11, 4], "Error" + assert eat(4, 8, 9) == [12, 1], "Error" + assert eat(1, 10, 10) == [11, 0], "Error" + assert eat(2, 11, 5) == [7, 0], "Error" + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(eat) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/encode.prompt b/benches/HumanEvalPromptsWithPythonSol/encode.prompt new file mode 100644 index 0000000..3548ebc --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/encode.prompt @@ -0,0 +1,27 @@ + +Write a function that takes a message, and encodes in such a +way that it swaps case of all letters, replaces all vowels in +the message with the letter that appears 2 places ahead of that +vowel in the english alphabet. +Assume only letters. + +Examples: +>>> encode('test') +'TGST' +>>> encode('This is a message') +'tHKS KS C MGSSCGG' + +Here's a correct unverified solution in python: +```python +def encode(message): + vowels = "aeiouAEIOU" + vowels_replace = dict([(i, chr(ord(i) + 2)) for i in vowels]) + message = message.swapcase() + return ''.join([vowels_replace[i] if i in vowels else i for i in message]) +def check(encode): + # Check some simple cases + assert encode('test') == 'TGST', "This prints if this assert fails 1 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert encode('This is a message') == 'tHKS KS C MGSSCGG', "This prints if this assert fails 2 (also good for debugging!)" +check(encode) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/encode_cyclic.prompt b/benches/HumanEvalPromptsWithPythonSol/encode_cyclic.prompt new file mode 100644 index 0000000..62c2169 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/encode_cyclic.prompt @@ -0,0 +1,19 @@ + +returns encoded string by cycling groups of three characters. + +Here's a correct unverified solution in python: +```python +def encode_cyclic(s: str): + """ + returns encoded string by cycling groups of three characters. + """ + # split string to groups. Each of length 3. + groups = [s[(3 * i):min((3 * i + 3), len(s))] for i in range((len(s) + 2) // 3)] + # cycle elements in each group. Unless group has fewer elements than 3. + groups = [(group[1:] + group[0]) if len(group) == 3 else group for group in groups] + return "".join(groups) + + +def decode_cyclic(s: str): + return encode_cyclic(encode_cyclic(s)) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/encode_shift.prompt b/benches/HumanEvalPromptsWithPythonSol/encode_shift.prompt new file mode 100644 index 0000000..6ad7ed9 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/encode_shift.prompt @@ -0,0 +1,15 @@ + +returns encoded string by shifting every character by 5 in the alphabet. + +Here's a correct unverified solution in python: +```python +def encode_shift(s: str): + """ + returns encoded string by shifting every character by 5 in the alphabet. + """ + return "".join([chr(((ord(ch) + 5 - ord("a")) % 26) + ord("a")) for ch in s]) + + +def decode_shift(s: str): + return "".join([chr(((ord(ch) - 5 - ord("a")) % 26) + ord("a")) for ch in s]) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/encrypt.prompt b/benches/HumanEvalPromptsWithPythonSol/encrypt.prompt new file mode 100644 index 0000000..a62906e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/encrypt.prompt @@ -0,0 +1,29 @@ +Create a function encrypt that takes a string as an argument and +returns a string encrypted with the alphabet being rotated. +The alphabet should be rotated in a manner such that the letters +shift down by two multiplied to two places. +For example: +encrypt('hi') returns 'lm' +encrypt('asdfghjkl') returns 'ewhjklnop' +encrypt('gf') returns 'kj' +encrypt('et') returns 'ix' + +Here's a correct unverified solution in python: +```python +def encrypt(s): + d = 'abcdefghijklmnopqrstuvwxyz' + out = '' + for c in s: + if c in d: + out += d[(d.index(c)+2*2) % 26] + else: + out += c + return out +def check(encrypt): + # Check some simple cases + assert encrypt('hi') == 'lm', "This prints if this assert fails 1 (good for debugging!)" + assert encrypt('asdfghjkl') == 'ewhjklnop', "This prints if this assert fails 1 (good for debugging!)" + assert encrypt('gf') == 'kj', "This prints if this assert fails 1 (good for debugging!)" + assert encrypt('et') == 'ix' +check(encrypt) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/even_odd_count.prompt b/benches/HumanEvalPromptsWithPythonSol/even_odd_count.prompt new file mode 100644 index 0000000..1c6d48b --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/even_odd_count.prompt @@ -0,0 +1,25 @@ +Given an integer. return a tuple that has the number of even and odd digits respectively. + + Example: + even_odd_count(-12) ==> (1, 1) + even_odd_count(123) ==> (1, 2) + +Here's a correct unverified solution in python: +```python +def even_odd_count(num): + even_count = 0 + odd_count = 0 + for i in str(abs(num)): + if int(i)%2==0: + even_count +=1 + else: + odd_count +=1 + return (even_count, odd_count) +def check(even_odd_count): + # Check some simple cases + assert even_odd_count(-12) == (1, 1) + assert even_odd_count(123) == (1, 2) + # Check some edge cases that are easy to work out by hand. + assert True +check(even_odd_count) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/even_odd_palindrome.prompt b/benches/HumanEvalPromptsWithPythonSol/even_odd_palindrome.prompt new file mode 100644 index 0000000..bd7486c --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/even_odd_palindrome.prompt @@ -0,0 +1,43 @@ + +Given a positive integer n, return a tuple that has the number of even and odd +integer palindromes that fall within the range(1, n), inclusive. + +Example 1: + + Input: 3 + Output: (1, 2) + Explanation: + Integer palindrome are 1, 2, 3. one of them is even, and two of them are odd. + +Example 2: + + Input: 12 + Output: (4, 6) + Explanation: + Integer palindrome are 1, 2, 3, 4, 5, 6, 7, 8, 9, 11. four of them are even, and 6 of them are odd. + +Note: + 1. 1 <= n <= 10^3 + 2. returned tuple has the number of even and odd integer palindromes respectively. + +Here's a correct unverified solution in python: +```python +def even_odd_palindrome(n): + def is_palindrome(n): + return str(n) == str(n)[::-1] + + even_palindrome_count = 0 + odd_palindrome_count = 0 + + for i in range(1, n+1): + if i%2 == 1 and is_palindrome(i): + odd_palindrome_count += 1 + elif i%2 == 0 and is_palindrome(i): + even_palindrome_count += 1 + return (even_palindrome_count, odd_palindrome_count) +def check(even_odd_palindrome): + # Check some simple cases + assert even_odd_palindrome(12) == (4, 6) + assert even_odd_palindrome(3) == (1, 2) +check(even_odd_palindrome) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/exchange.prompt b/benches/HumanEvalPromptsWithPythonSol/exchange.prompt new file mode 100644 index 0000000..40aa6c6 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/exchange.prompt @@ -0,0 +1,33 @@ +In this problem, you will implement a function that takes two lists of numbers, +and determines whether it is possible to perform an exchange of elements +between them to make lst1 a list of only even numbers. +There is no limit on the number of exchanged elements between lst1 and lst2. +If it is possible to exchange elements between the lst1 and lst2 to make +all the elements of lst1 to be even, return "YES". +Otherwise, return "NO". +For example: +exchange([1, 2, 3, 4], [1, 2, 3, 4]) => "YES" +exchange([1, 2, 3, 4], [1, 5, 3, 4]) => "NO" +It is assumed that the input lists will be non-empty. + +Here's a correct unverified solution in python: +```python +def exchange(lst1, lst2): + odd = 0 + even = 0 + for i in lst1: + if i%2 == 1: + odd += 1 + for i in lst2: + if i%2 == 0: + even += 1 + if even >= odd: + return "YES" + return "NO" + +def check(exchange): + # Check some simple cases + assert exchange([1, 2, 3, 4], [1, 2, 3, 4]) == "YES" + assert exchange([1, 2, 3, 4], [1, 5, 3, 4]) == "NO" +check(exchange) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/f.prompt b/benches/HumanEvalPromptsWithPythonSol/f.prompt new file mode 100644 index 0000000..8b91d96 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/f.prompt @@ -0,0 +1,26 @@ + Implement the function f that takes n as a parameter, +and returns a list of size n, such that the value of the element at index i is the factorial of i if i is even +or the sum of numbers from 1 to i otherwise. +i starts from 1. +the factorial of i is the multiplication of the numbers from 1 to i (1 * 2 * ... * i). +Example: +f(5) == [1, 2, 6, 24, 15] + +Here's a correct unverified solution in python: +```python +def f(n): + ret = [] + for i in range(1,n+1): + if i%2 == 0: + x = 1 + for j in range(1,i+1): x *= j + ret += [x] + else: + x = 0 + for j in range(1,i+1): x += j + ret += [x] + return ret +def check(f): + assert f(5) == [1, 2, 6, 24, 15] +check(f) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/factorize.prompt b/benches/HumanEvalPromptsWithPythonSol/factorize.prompt new file mode 100644 index 0000000..0482037 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/factorize.prompt @@ -0,0 +1,35 @@ + Return list of prime factors of given integer in the order from smallest to largest. +Each of the factors should be listed number of times corresponding to how many times it appeares in factorization. +Input number should be equal to the product of all factors +>>> factorize(8) +[2, 2, 2] +>>> factorize(25) +[5, 5] +>>> factorize(70) +[2, 5, 7] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def factorize(n: int) -> List[int]: + import math + fact = [] + i = 2 + while i <= int(math.sqrt(n) + 1): + if n % i == 0: + fact.append(i) + n //= i + else: + i += 1 + + if n > 1: + fact.append(n) + return fact +def check(factorize): + assert factorize(8) == [2, 2, 2] + assert factorize(25) == [5,5] + assert factorize(70) == [2,5,7] +check(factorize) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/fib.prompt b/benches/HumanEvalPromptsWithPythonSol/fib.prompt new file mode 100644 index 0000000..de0bb78 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/fib.prompt @@ -0,0 +1,22 @@ +Return n-th Fibonacci number. +>>> fib(10) +55 +>>> fib(1) +1 +>>> fib(8) +21 + +Here's a correct unverified solution in python: +```python +def fib(n: int): + if n == 0: + return 0 + if n == 1: + return 1 + return fib(n - 1) + fib(n - 2) +def check(fib): + assert fib(10) == 55 + assert fib(1) == 1 + assert fib(8) == 21 +check(fib) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/fib4.prompt b/benches/HumanEvalPromptsWithPythonSol/fib4.prompt new file mode 100644 index 0000000..00dc125 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/fib4.prompt @@ -0,0 +1,32 @@ +The Fib4 number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: +fib4(0) -> 0 +fib4(1) -> 0 +fib4(2) -> 2 +fib4(3) -> 0 +fib4(n) -> fib4(n-1) + fib4(n-2) + fib4(n-3) + fib4(n-4). +Please write a function to efficiently compute the n-th element of the fib4 number sequence. Do not use recursion. +>>> fib4(5) +4 +>>> fib4(6) +8 +>>> fib4(7) +14 + +Here's a correct unverified solution in python: +```python +def fib4(n: int): + results = [0, 0, 2, 0] + if n < 4: + return results[n] + + for _ in range(4, n + 1): + results.append(results[-1] + results[-2] + results[-3] + results[-4]) + results.pop(0) + + return results[-1] +def check(fib4): + assert fib4(5) == 4 + assert fib4(6) == 8 + assert fib4(7) == 14 +check(fib4) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/fibfib.prompt b/benches/HumanEvalPromptsWithPythonSol/fibfib.prompt new file mode 100644 index 0000000..330690f --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/fibfib.prompt @@ -0,0 +1,29 @@ +The FibFib number sequence is a sequence similar to the Fibbonacci sequnece that's defined as follows: +fibfib(0) == 0 +fibfib(1) == 0 +fibfib(2) == 1 +fibfib(n) == fibfib(n-1) + fibfib(n-2) + fibfib(n-3). +Please write a function to efficiently compute the n-th element of the fibfib number sequence. +>>> fibfib(1) +0 +>>> fibfib(5) +4 +>>> fibfib(8) +24 + +Here's a correct unverified solution in python: +```python +def fibfib(n: int): + if n == 0: + return 0 + if n == 1: + return 0 + if n == 2: + return 1 + return fibfib(n - 1) + fibfib(n - 2) + fibfib(n - 3) +def check(fibfib): + assert fibfib(1) == 0 + assert fibfib(5) == 4 + assert fibfib(8) == 24 +check(fibfib) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/file_name_check.prompt b/benches/HumanEvalPromptsWithPythonSol/file_name_check.prompt new file mode 100644 index 0000000..92b9c1b --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/file_name_check.prompt @@ -0,0 +1,36 @@ +Create a function which takes a string representing a file's name, and returns +'Yes' if the the file's name is valid, and returns 'No' otherwise. +A file's name is considered to be valid if and only if all the following conditions +are met: +- There should not be more than three digits ('0'-'9') in the file's name. +- The file's name contains exactly one dot '.' +- The substring before the dot should not be empty, and it starts with a letter from +the latin alphapet ('a'-'z' and 'A'-'Z'). +- The substring after the dot should be one of these: ['txt', 'exe', 'dll'] +Examples: +file_name_check("example.txt") # => 'Yes' +file_name_check("1example.dll") # => 'No' (the name should start with a latin alphapet letter) + +Here's a correct unverified solution in python: +```python +def file_name_check(file_name): + suf = ['txt', 'exe', 'dll'] + lst = file_name.split(sep='.') + if len(lst) != 2: + return 'No' + if not lst[1] in suf: + return 'No' + if len(lst[0]) == 0: + return 'No' + if not lst[0][0].isalpha(): + return 'No' + t = len([x for x in lst[0] if x.isdigit()]) + if t > 3: + return 'No' + return 'Yes' +def check(file_name_check): + # Check some simple cases + assert file_name_check("example.txt") == 'Yes' + assert file_name_check("1example.dll") == 'No' +check(file_name_check) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/filter_by_prefix.prompt b/benches/HumanEvalPromptsWithPythonSol/filter_by_prefix.prompt new file mode 100644 index 0000000..63780d3 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/filter_by_prefix.prompt @@ -0,0 +1,18 @@ + Filter an input list of strings only for ones that start with a given prefix. +>>> filter_by_prefix([], 'a') +[] +>>> filter_by_prefix(['abc', 'bcd', 'cde', 'array'], 'a') +['abc', 'array'] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def filter_by_prefix(strings: List[str], prefix: str) -> List[str]: + return [x for x in strings if x.startswith(prefix)] +def check(filter_by_prefix): + assert filter_by_prefix([], 'a') == [] + assert filter_by_prefix(['abc', 'bcd', 'cde', 'array'], 'a') == ['abc', 'array'] +check(filter_by_prefix) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/filter_by_substring.prompt b/benches/HumanEvalPromptsWithPythonSol/filter_by_substring.prompt new file mode 100644 index 0000000..32d6cd2 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/filter_by_substring.prompt @@ -0,0 +1,19 @@ + Filter an input list of strings only for ones that contain given substring +>>> filter_by_substring([], 'a') +[] +>>> filter_by_substring(['abc', 'bacd', 'cde', 'array'], 'a') +['abc', 'bacd', 'array'] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def filter_by_substring(strings: List[str], substring: str) -> List[str]: + return [x for x in strings if substring in x] + +def check(filter_by_substring): + assert filter_by_substring([], 'a') == [] + assert filter_by_substring(['abc', 'bacd', 'cde', 'array'], 'a') == ['abc', 'bacd', 'array'] +check(filter_by_substring) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/filter_integers.prompt b/benches/HumanEvalPromptsWithPythonSol/filter_integers.prompt new file mode 100644 index 0000000..1eab5c5 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/filter_integers.prompt @@ -0,0 +1,18 @@ + Filter given list of any python values only for integers +>>> filter_integers(['a', 3.14, 5]) +[5] +>>> filter_integers([1, 2, 3, 'abc', {}, []]) +[1, 2, 3] + +Here's a correct unverified solution in python: +```python +from typing import Any, List + + +def filter_integers(values: List[Any]) -> List[int]: + return [x for x in values if isinstance(x, int)] +def check(filter_integers): + assert filter_integers(['a', 3.14, 5]) == [5] + assert filter_integers([1, 2, 3, 'abc', {}, []]) == [1,2,3] +check(filter_integers) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/find_closest_elements.prompt b/benches/HumanEvalPromptsWithPythonSol/find_closest_elements.prompt new file mode 100644 index 0000000..7c14b51 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/find_closest_elements.prompt @@ -0,0 +1,34 @@ + From a supplied list of numbers (of length at least two) select and return two that are the closest to each +other and return them in order (smaller number, larger number). +>>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) +(2.0, 2.2) +>>> find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.0]) +(2.0, 2.0) + +Here's a correct unverified solution in python: +```python +from typing import List, Tuple + + +def find_closest_elements(numbers: List[float]) -> Tuple[float, float]: + closest_pair = None + distance = None + + for idx, elem in enumerate(numbers): + for idx2, elem2 in enumerate(numbers): + if idx != idx2: + if distance is None: + distance = abs(elem - elem2) + closest_pair = tuple(sorted([elem, elem2])) + else: + new_distance = abs(elem - elem2) + if new_distance < distance: + distance = new_distance + closest_pair = tuple(sorted([elem, elem2])) + + return closest_pair +def check(find_closest_elements): + assert find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.2]) == (2.0, 2.2) + assert find_closest_elements([1.0, 2.0, 3.0, 4.0, 5.0, 2.0]) == (2.0, 2.0) +check(find_closest_elements) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/find_max.prompt b/benches/HumanEvalPromptsWithPythonSol/find_max.prompt new file mode 100644 index 0000000..1c82e40 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/find_max.prompt @@ -0,0 +1,20 @@ +Write a function that accepts a list of strings. +The list contains different words. Return the word with maximum number +of unique characters. If multiple strings have maximum number of unique +characters, return the one which comes first in lexicographical order. + +find_max(["name", "of", "string"]) == "string" +find_max(["name", "enam", "game"]) == "enam" +find_max(["aaaaaaa", "bb" ,"cc"]) == ""aaaaaaa" + +Here's a correct unverified solution in python: +```python +def find_max(words): + return sorted(words, key = lambda x: (-len(set(x)), x))[0] +def check(find_max): + # Check some simple cases + assert (find_max(["name", "of", "string"]) == "string"), "t1" + assert (find_max(["name", "enam", "game"]) == "enam"), 't2' + assert (find_max(["aaaaaaa", "bb", "cc"]) == "aaaaaaa"), 't3' +check(find_max) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/fix_spaces.prompt b/benches/HumanEvalPromptsWithPythonSol/fix_spaces.prompt new file mode 100644 index 0000000..ff1f4f1 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/fix_spaces.prompt @@ -0,0 +1,42 @@ + +Given a string text, replace all spaces in it with underscores, +and if a string has more than 2 consecutive spaces, +then replace all consecutive spaces with - + +fix_spaces("Example") == "Example" +fix_spaces("Example 1") == "Example_1" +fix_spaces(" Example 2") == "_Example_2" +fix_spaces(" Example 3") == "_Example-3" + +Here's a correct unverified solution in python: +```python +def fix_spaces(text): + new_text = "" + i = 0 + start, end = 0, 0 + while i < len(text): + if text[i] == " ": + end += 1 + else: + if end - start > 2: + new_text += "-"+text[i] + elif end - start > 0: + new_text += "_"*(end - start)+text[i] + else: + new_text += text[i] + start, end = i+1, i+1 + i+=1 + if end - start > 2: + new_text += "-" + elif end - start > 0: + new_text += "_" + return new_text +def check(fix_spaces): + # Check some simple cases + assert fix_spaces("Example") == "Example", "This prints if this assert fails 1 (good for debugging!)" + assert fix_spaces("Example 1") == "Example_1" + assert fix_spaces(" Example 2") == "_Example_2" + # Check some edge cases that are easy to work out by hand. + assert fix_spaces(" Example 3") == "_Example-3" +check(fix_spaces) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/fizz_buzz.prompt b/benches/HumanEvalPromptsWithPythonSol/fizz_buzz.prompt new file mode 100644 index 0000000..d2cfdf9 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/fizz_buzz.prompt @@ -0,0 +1,26 @@ +Return the number of times the digit 7 appears in integers less than n which are divisible by 11 or 13. +>>> fizz_buzz(50) +0 +>>> fizz_buzz(78) +2 +>>> fizz_buzz(79) +3 + +Here's a correct unverified solution in python: +```python +def fizz_buzz(n: int): + ns = [] + for i in range(n): + if i % 11 == 0 or i % 13 == 0: + ns.append(i) + s = ''.join(list(map(str, ns))) + ans = 0 + for c in s: + ans += (c == '7') + return ans +def check(fizz_buzz): + assert fizz_buzz(50) == 0 + assert fizz_buzz(78) == 2 + assert fizz_buzz(79) == 3 +check(fizz_buzz) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/flip_case.prompt b/benches/HumanEvalPromptsWithPythonSol/flip_case.prompt new file mode 100644 index 0000000..b9683d6 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/flip_case.prompt @@ -0,0 +1,12 @@ + For a given string, flip lowercase characters to uppercase and uppercase to lowercase. +>>> flip_case('Hello') +'hELLO' + +Here's a correct unverified solution in python: +```python +def flip_case(string: str) -> str: + return string.swapcase() +def check(flip_case): + assert flip_case('Hello') == 'hELLO' +check(flip_case) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/fruit_distribution.prompt b/benches/HumanEvalPromptsWithPythonSol/fruit_distribution.prompt new file mode 100644 index 0000000..3ccf408 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/fruit_distribution.prompt @@ -0,0 +1,28 @@ + +In this task, you will be given a string that represents a number of apples and oranges +that are distributed in a basket of fruit this basket contains +apples, oranges, and mango fruits. Given the string that represents the total number of +the oranges and apples and an integer that represent the total number of the fruits +in the basket return the number of the mango fruits in the basket. +for examble: +fruit_distribution("5 apples and 6 oranges", 19) ->19 - 5 - 6 = 8 +fruit_distribution("0 apples and 1 oranges",3) -> 3 - 0 - 1 = 2 +fruit_distribution("2 apples and 3 oranges", 100) -> 100 - 2 - 3 = 95 +fruit_distribution("100 apples and 1 oranges",120) -> 120 - 100 - 1 = 19 + +Here's a correct unverified solution in python: +```python +def fruit_distribution(s,n): + lis = list() + for i in s.split(' '): + if i.isdigit(): + lis.append(int(i)) + return n - sum(lis) +def check(fruit_distribution): + # Check some simple cases + assert fruit_distribution("5 apples and 6 oranges",19) == 8 + assert fruit_distribution("0 apples and 1 oranges",3) == 2 + assert fruit_distribution("2 apples and 3 oranges",100) == 95 + assert fruit_distribution("1 apples and 100 oranges",120) == 19 +check(fruit_distribution) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/generate_integers.prompt b/benches/HumanEvalPromptsWithPythonSol/generate_integers.prompt new file mode 100644 index 0000000..968dc39 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/generate_integers.prompt @@ -0,0 +1,26 @@ + +Given two positive integers a and b, return the even digits between a +and b, in ascending order. + +For example: +generate_integers(2, 8) => [2, 4, 6, 8] +generate_integers(8, 2) => [2, 4, 6, 8] +generate_integers(10, 14) => [] + +Here's a correct unverified solution in python: +```python +def generate_integers(a, b): + lower = max(2, min(a, b)) + upper = min(8, max(a, b)) + + return [i for i in range(lower, upper+1) if i % 2 == 0] +def check(generate_integers): + # Check some simple cases + assert generate_integers(2, 10) == [2, 4, 6, 8], "Test 1" + assert generate_integers(10, 2) == [2, 4, 6, 8], "Test 2" + assert generate_integers(132, 2) == [2, 4, 6, 8], "Test 3" + assert generate_integers(17,89) == [], "Test 4" + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(generate_integers) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/get_closest_vowel.prompt b/benches/HumanEvalPromptsWithPythonSol/get_closest_vowel.prompt new file mode 100644 index 0000000..5944265 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/get_closest_vowel.prompt @@ -0,0 +1,34 @@ +You are given a word. Your task is to find the closest vowel that stands between +two consonants from the right side of the word (case sensitive). + +Vowels in the beginning and ending doesn't count. Return empty string if you didn't +find any vowel met the above condition. + +You may assume that the given string contains English letter only. + +Example: +get_closest_vowel("yogurt") ==> "u" +get_closest_vowel("FULL") ==> "U" +get_closest_vowel("quick") ==> "" +get_closest_vowel("ab") ==> "" + +Here's a correct unverified solution in python: +```python +def get_closest_vowel(word): + if len(word) < 3: + return "" + + vowels = {"a", "e", "i", "o", "u", "A", "E", 'O', 'U', 'I'} + for i in range(len(word)-2, 0, -1): + if word[i] in vowels: + if (word[i+1] not in vowels) and (word[i-1] not in vowels): + return word[i] + return "" +def check(get_closest_vowel): + # Check some simple cases + assert get_closest_vowel("yogurt") == "u" + assert get_closest_vowel("FULL") == "U" + assert get_closest_vowel("ab") == "" + assert get_closest_vowel("quick") == "" +check(get_closest_vowel) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/get_max_triples.prompt b/benches/HumanEvalPromptsWithPythonSol/get_max_triples.prompt new file mode 100644 index 0000000..2e3b752 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/get_max_triples.prompt @@ -0,0 +1,28 @@ + +You are given a positive integer n. You have to create an integer array a of length n. + For each i (1 ≤ i ≤ n), the value of a[i] = i * i - i + 1. + Return the number of triples (a[i], a[j], a[k]) of a where i < j < k, +and a[i] + a[j] + a[k] is a multiple of 3. + +Example : + Input: n = 5 + Output: 1 + Explanation: + a = [1, 3, 7, 13, 21] + The only valid triple is (1, 7, 13). + +Here's a correct unverified solution in python: +```python +def get_max_triples(n): + A = [i*i - i + 1 for i in range(1,n+1)] + ans = [] + for i in range(n): + for j in range(i+1,n): + for k in range(j+1,n): + if (A[i]+A[j]+A[k])%3 == 0: + ans += [(A[i],A[j],A[k])] + return len(ans) +def check(get_max_triples): + assert get_max_triples(5) == 1 +check(get_max_triples) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/get_odd_collatz.prompt b/benches/HumanEvalPromptsWithPythonSol/get_odd_collatz.prompt new file mode 100644 index 0000000..0a65326 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/get_odd_collatz.prompt @@ -0,0 +1,38 @@ + +Given a positive integer n, return a sorted list that has the odd numbers in collatz sequence. + +The Collatz conjecture is a conjecture in mathematics that concerns a sequence defined +as follows: start with any positive integer n. Then each term is obtained from the +previous term as follows: if the previous term is even, the next term is one half of +the previous term. If the previous term is odd, the next term is 3 times the previous +term plus 1. The conjecture is that no matter what value of n, the sequence will always reach 1. + +Note: + 1. Collatz(1) is [1]. + 2. returned list sorted in increasing order. + +For example: +get_odd_collatz(5) returns [1, 5] # The collatz sequence for 5 is [5, 16, 8, 4, 2, 1], so the odd numbers are only 1, and 5. + +Here's a correct unverified solution in python: +```python +def get_odd_collatz(n): + if n%2==0: + odd_collatz = [] + else: + odd_collatz = [n] + while n > 1: + if n % 2 == 0: + n = n/2 + else: + n = n*3 + 1 + + if n%2 == 1: + odd_collatz.append(int(n)) + + return sorted(odd_collatz) +def check(get_odd_collatz): + # Check some simple cases + assert get_odd_collatz(5) == [1, 5] +check(get_odd_collatz) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/get_positive.prompt b/benches/HumanEvalPromptsWithPythonSol/get_positive.prompt new file mode 100644 index 0000000..36a21a8 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/get_positive.prompt @@ -0,0 +1,15 @@ +Return only positive numbers in the list. +>>> get_positive([-1, 2, -4, 5, 6]) +[2, 5, 6] +>>> get_positive([5, 3, -5, 2, -3, 3, 9, 0, 123, 1, -10]) +[5, 3, 2, 3, 9, 123, 1] + +Here's a correct unverified solution in python: +```python +def get_positive(l: list): + return [e for e in l if e > 0] +def check(get_positive): + assert get_positive([-1, 2, -4, 5, 6]) == [2, 5, 6] + assert get_positive([5, 3, -5, 2, -3, 3, 9, 0, 123, 1, -10]) == [5, 3, 2, 3, 9, 123, 1] +check(get_positive) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/get_row.prompt b/benches/HumanEvalPromptsWithPythonSol/get_row.prompt new file mode 100644 index 0000000..f313564 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/get_row.prompt @@ -0,0 +1,37 @@ + +You are given a 2 dimensional data, as a nested lists, +which is similar to matrix, however, unlike matrices, +each row may contain a different number of columns. +Given lst, and integer x, find integers x in the list, +and return list of tuples, [(x1, y1), (x2, y2) ...] such that +each tuple is a coordinate - (row, columns), starting with 0. +Sort coordinates initially by rows in ascending order. +Also, sort coordinates of the row by columns in descending order. + +Examples: +get_row([ + [1,2,3,4,5,6], + [1,2,3,4,1,6], + [1,2,3,4,5,1] +], 1) == [(0, 0), (1, 4), (1, 0), (2, 5), (2, 0)] +get_row([], 1) == [] +get_row([[], [1], [1, 2, 3]], 3) == [(2, 2)] + +Here's a correct unverified solution in python: +```python +def get_row(lst, x): + coords = [(i, j) for i in range(len(lst)) for j in range(len(lst[i])) if lst[i][j] == x] + return sorted(sorted(coords, key=lambda x: x[1], reverse=True), key=lambda x: x[0]) +def check(get_row): + # Check some simple cases + assert get_row([ + [1,2,3,4,5,6], + [1,2,3,4,1,6], + [1,2,3,4,5,1] + ], 1) == [(0, 0), (1, 4), (1, 0), (2, 5), (2, 0)] + assert get_row([], 1) == [] + assert get_row([[], [1], [1, 2, 3]], 3) == [(2, 2)] + # Check some edge cases that are easy to work out by hand. + assert True +check(get_row) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/greatest_common_divisor.prompt b/benches/HumanEvalPromptsWithPythonSol/greatest_common_divisor.prompt new file mode 100644 index 0000000..1abe252 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/greatest_common_divisor.prompt @@ -0,0 +1,17 @@ + Return a greatest common divisor of two integers a and b +>>> greatest_common_divisor(3, 5) +1 +>>> greatest_common_divisor(25, 15) +5 + +Here's a correct unverified solution in python: +```python +def greatest_common_divisor(a: int, b: int) -> int: + while b: + a, b = b, a % b + return a +def check(greatest_common_divisor): + assert greatest_common_divisor(3, 5) == 1 + assert greatest_common_divisor(25, 15) == 5 +check(greatest_common_divisor) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/has_close_elements.prompt b/benches/HumanEvalPromptsWithPythonSol/has_close_elements.prompt new file mode 100644 index 0000000..d4ee235 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/has_close_elements.prompt @@ -0,0 +1,29 @@ + Check if in given list of numbers, are any two numbers closer to each other than +given threshold. +>>> has_close_elements([1.0, 2.0, 3.0], 0.5) +False +>>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3) +True + +Here's a correct unverified solution in python: +```python +from typing import List + + +def has_close_elements(numbers: List[float], threshold: float) -> bool: + for idx, elem in enumerate(numbers): + for idx2, elem2 in enumerate(numbers): + if idx != idx2: + distance = abs(elem - elem2) + if distance < threshold: + return True + + + return False + +def check(has_close_elements): + assert has_close_elements([1.0, 2.0, 3.0], 0.5) == False + assert has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3) == True + +check(has_close_elements) +print(has_close_elements([1.0, 2.0, 3.0], 0.5))``` diff --git a/benches/HumanEvalPromptsWithPythonSol/hex_key.prompt b/benches/HumanEvalPromptsWithPythonSol/hex_key.prompt new file mode 100644 index 0000000..b977a7a --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/hex_key.prompt @@ -0,0 +1,36 @@ +You have been tasked to write a function that receives +a hexadecimal number as a string and counts the number of hexadecimal +digits that are primes (prime number, or a prime, is a natural number +greater than 1 that is not a product of two smaller natural numbers). +Hexadecimal digits are 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, A, B, C, D, E, F. +Prime numbers are 2, 3, 5, 7, 11, 13, 17,... +So you have to determine a number of the following digits: 2, 3, 5, 7, +B (=decimal 11), D (=decimal 13). +Note: you may assume the input is always correct or empty string, +and symbols A,B,C,D,E,F are always uppercase. +Examples: +For num = "AB" the output should be 1. +For num = "1077E" the output should be 2. +For num = "ABED1A33" the output should be 4. +For num = "123456789ABCDEF0" the output should be 6. +For num = "2020" the output should be 2. + +Here's a correct unverified solution in python: +```python +def hex_key(num): + primes = ('2', '3', '5', '7', 'B', 'D') + total = 0 + for i in range(0, len(num)): + if num[i] in primes: + total += 1 + return total +def check(hex_key): + # Check some simple cases + assert hex_key("AB") == 1, "First test error: " + str(hex_key("AB")) + assert hex_key("1077E") == 2, "Second test error: " + str(hex_key("1077E")) + assert hex_key("ABED1A33") == 4, "Third test error: " + str(hex_key("ABED1A33")) + assert hex_key("2020") == 2, "Fourth test error: " + str(hex_key("2020")) + assert hex_key("123456789ABCDEF0") == 6, "Fifth test error: " + str(hex_key("123456789ABCDEF0")) + # Check some edge cases that are easy to work out by hand. +check(hex_key) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/histogram.prompt b/benches/HumanEvalPromptsWithPythonSol/histogram.prompt new file mode 100644 index 0000000..c6f7b14 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/histogram.prompt @@ -0,0 +1,38 @@ +Given a string representing a space separated lowercase letters, return a dictionary +of the letter with the most repetition and containing the corresponding count. +If several letters have the same occurrence, return all of them. + +Example: +histogram('a b c') == {'a': 1, 'b': 1, 'c': 1} +histogram('a b b a') == {'a': 2, 'b': 2} +histogram('a b c a b') == {'a': 2, 'b': 2} +histogram('b b b b a') == {'b': 4} +histogram('') == {} + + +Here's a correct unverified solution in python: +```python +def histogram(test): + dict1={} + list1=test.split(" ") + t=0 + + for i in list1: + if(list1.count(i)>t) and i!='': + t=list1.count(i) + if t>0: + for i in list1: + if(list1.count(i)==t): + + dict1[i]=t + return dict1 +def check(histogram): + # Check some simple cases + assert histogram('a b b a') == {'a':2,'b': 2}, "This prints if this assert fails 1 (good for debugging!)" + assert histogram('a b c a b') == {'a': 2, 'b': 2}, "This prints if this assert fails 2 (good for debugging!)" + assert histogram('a b c') == {'a': 1,'b': 1,'c': 1}, "This prints if this assert fails 4 (good for debugging!)" + assert histogram('b b b b a') == {'b': 4}, "This prints if this assert fails 5 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert histogram('') == {}, "This prints if this assert fails 7 (also good for debugging!)" +check(histogram) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/how_many_times.prompt b/benches/HumanEvalPromptsWithPythonSol/how_many_times.prompt new file mode 100644 index 0000000..f856593 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/how_many_times.prompt @@ -0,0 +1,24 @@ + Find how many times a given substring can be found in the original string. Count overlaping cases. +>>> how_many_times('', 'a') +0 +>>> how_many_times('aaa', 'a') +3 +>>> how_many_times('aaaa', 'aa') +3 + +Here's a correct unverified solution in python: +```python +def how_many_times(string: str, substring: str) -> int: + times = 0 + + for i in range(len(string) - len(substring) + 1): + if string[i:i+len(substring)] == substring: + times += 1 + + return times +def check(how_many_times): + assert how_many_times('', 'a') == 0 + assert how_many_times('aaa', 'a') == 3 + assert how_many_times('aaaa', 'aa') == 3 +check(how_many_times) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/incr_list.prompt b/benches/HumanEvalPromptsWithPythonSol/incr_list.prompt new file mode 100644 index 0000000..20050cc --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/incr_list.prompt @@ -0,0 +1,15 @@ +Return list with elements incremented by 1. +>>> incr_list([1, 2, 3]) +[2, 3, 4] +>>> incr_list([5, 3, 5, 2, 3, 3, 9, 0, 123]) +[6, 4, 6, 3, 4, 4, 10, 1, 124] + +Here's a correct unverified solution in python: +```python +def incr_list(l: list): + return [(e + 1) for e in l] +def check(incr_list): + assert incr_list([1, 2, 3]) == [2, 3, 4] + assert incr_list([5, 2, 5, 2, 3, 3, 9, 0, 123]) == [6, 3, 6, 3, 4, 4, 10, 1, 124] +check(incr_list) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/int_to_mini_roman.prompt b/benches/HumanEvalPromptsWithPythonSol/int_to_mini_roman.prompt new file mode 100644 index 0000000..676ad2c --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/int_to_mini_roman.prompt @@ -0,0 +1,34 @@ + +Given a positive integer, obtain its roman numeral equivalent as a string, +and return it in lowercase. +Restrictions: 1 <= num <= 1000 + +Examples: +>>> int_to_mini_roman(19) == 'xix' +>>> int_to_mini_roman(152) == 'clii' +>>> int_to_mini_roman(426) == 'cdxxvi' + +Here's a correct unverified solution in python: +```python +def int_to_mini_roman(number): + num = [1, 4, 5, 9, 10, 40, 50, 90, + 100, 400, 500, 900, 1000] + sym = ["I", "IV", "V", "IX", "X", "XL", + "L", "XC", "C", "CD", "D", "CM", "M"] + i = 12 + res = '' + while number: + div = number // num[i] + number %= num[i] + while div: + res += sym[i] + div -= 1 + i -= 1 + return res.lower() +def check(int_to_mini_roman): + # Check some simple cases + assert int_to_mini_roman(19) == 'xix' + assert int_to_mini_roman(152) == 'clii' + assert int_to_mini_roman(426) == 'cdxxvi' +check(int_to_mini_roman) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/intersection.prompt b/benches/HumanEvalPromptsWithPythonSol/intersection.prompt new file mode 100644 index 0000000..190d55e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/intersection.prompt @@ -0,0 +1,45 @@ +You are given two intervals, +where each interval is a pair of integers. For example, interval = (start, end) = (1, 2). +The given intervals are closed which means that the interval (start, end) +includes both start and end. +For each given interval, it is assumed that its start is less or equal its end. +Your task is to determine whether the length of intersection of these two +intervals is a prime number. +Example, the intersection of the intervals (1, 3), (2, 4) is (2, 3) +which its length is 1, which not a prime number. +If the length of the intersection is a prime number, return "YES", +otherwise, return "NO". +If the two intervals don't intersect, return "NO". + + +[input/output] samples: +intersection((1, 2), (2, 3)) ==> "NO" +intersection((-1, 1), (0, 4)) ==> "NO" +intersection((-3, -1), (-5, 5)) ==> "YES" + +Here's a correct unverified solution in python: +```python +def intersection(interval1, interval2): + def is_prime(num): + if num == 1 or num == 0: + return False + if num == 2: + return True + for i in range(2, num): + if num%i == 0: + return False + return True + + l = max(interval1[0], interval2[0]) + r = min(interval1[1], interval2[1]) + length = r - l + if length > 0 and is_prime(length): + return "YES" + return "NO" +def check(intersection): + # Check some simple cases + assert intersection((1, 2), (2, 3)) == "NO" + assert intersection((-1, 1), (0, 4)) == "NO" + assert intersection((-3, -1), (-5, 5)) == "YES" +check(intersection) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/intersperse.prompt b/benches/HumanEvalPromptsWithPythonSol/intersperse.prompt new file mode 100644 index 0000000..7704465 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/intersperse.prompt @@ -0,0 +1,25 @@ + Insert a number 'delimeter' between every two consecutive elements of input list `numbers' +>>> intersperse([], 4) +[] +>>> intersperse([1, 2, 3], 4) +[1, 4, 2, 4, 3] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def intersperse(numbers: List[int], delimeter: int) -> List[int]: + if not numbers: + return [] + + result = [] + + for n in numbers[:-1]: + result.append(n) + result.append(delimeter) + + result.append(numbers[-1]) + + return result +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/is_bored.prompt b/benches/HumanEvalPromptsWithPythonSol/is_bored.prompt new file mode 100644 index 0000000..1f79fb4 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/is_bored.prompt @@ -0,0 +1,23 @@ + +You'll be given a string of words, and your task is to count the number +of boredoms. A boredom is a sentence that starts with the word "I". +Sentences are delimited by '.', '?' or '!'. + +For example: +>>> is_bored("Hello world") +0 +>>> is_bored("The sky is blue. The sun is shining. I love this weather") +1 + +Here's a correct unverified solution in python: +```python +def is_bored(S): + import re + sentences = re.split(r'[.?!]\s*', S) + return sum(sentence[0:2] == 'I ' for sentence in sentences) +def check(is_bored): + # Check some simple cases + assert is_bored("Hello world") == 0, "Test 1" + assert is_bored("The sky is blue. The sun is shining. I love this weather") == 1, "Test 3" +check(is_bored) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/is_equal_to_sum_even.prompt b/benches/HumanEvalPromptsWithPythonSol/is_equal_to_sum_even.prompt new file mode 100644 index 0000000..dad9405 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/is_equal_to_sum_even.prompt @@ -0,0 +1,16 @@ +Evaluate whether the given number n can be written as the sum of exactly 4 positive even numbers +Example +is_equal_to_sum_even(4) == False +is_equal_to_sum_even(6) == False +is_equal_to_sum_even(8) == True + +Here's a correct unverified solution in python: +```python +def is_equal_to_sum_even(n): + return n%2 == 0 and n >= 8 +def check(is_equal_to_sum_even): + assert is_equal_to_sum_even(4) == False + assert is_equal_to_sum_even(6) == False + assert is_equal_to_sum_even(8) == True +check(is_equal_to_sum_even) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/is_happy.prompt b/benches/HumanEvalPromptsWithPythonSol/is_happy.prompt new file mode 100644 index 0000000..cd9de35 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/is_happy.prompt @@ -0,0 +1,32 @@ +You are given a string s. +Your task is to check if the string is happy or not. +A string is happy if its length is at least 3 and every 3 consecutive letters are distinct +For example: +is_happy(a) => False +is_happy(aa) => False +is_happy(abcd) => True +is_happy(aabb) => False +is_happy(adb) => True +is_happy(xyy) => False + +Here's a correct unverified solution in python: +```python +def is_happy(s): + if len(s) < 3: + return False + + for i in range(len(s) - 2): + + if s[i] == s[i+1] or s[i+1] == s[i+2] or s[i] == s[i+2]: + return False + return True +def check(is_happy): + # Check some simple cases + assert is_happy("a") == False , "a" + assert is_happy("aa") == False , "aa" + assert is_happy("abcd") == True , "abcd" + assert is_happy("aabb") == False , "aabb" + assert is_happy("adb") == True , "adb" + assert is_happy("xyy") == False , "xyy" +check(is_happy) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/is_multiply_prime.prompt b/benches/HumanEvalPromptsWithPythonSol/is_multiply_prime.prompt new file mode 100644 index 0000000..9b7aed8 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/is_multiply_prime.prompt @@ -0,0 +1,28 @@ +Write a function that returns true if the given number is the multiplication of 3 prime numbers +and false otherwise. +Knowing that (a) is less then 100. +Example: +is_multiply_prime(30) == True +30 = 2 * 3 * 5 + +Here's a correct unverified solution in python: +```python +def is_multiply_prime(a): + def is_prime(n): + for j in range(2,n): + if n%j == 0: + return False + return True + + for i in range(2,101): + if not is_prime(i): continue + for j in range(2,101): + if not is_prime(j): continue + for k in range(2,101): + if not is_prime(k): continue + if i*j*k == a: return True + return False +def check(is_multiply_prime): + assert is_multiply_prime(30) == True +check(is_multiply_prime) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/is_nested.prompt b/benches/HumanEvalPromptsWithPythonSol/is_nested.prompt new file mode 100644 index 0000000..fe79260 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/is_nested.prompt @@ -0,0 +1,44 @@ + +Create a function that takes a string as input which contains only square brackets. +The function should return True if and only if there is a valid subsequence of brackets +where at least one bracket in the subsequence is nested. + +is_nested('[[]]') ➞ True +is_nested('[]]]]]]][[[[[]') ➞ False +is_nested('[][]') ➞ False +is_nested('[]') ➞ False +is_nested('[[][]]') ➞ True +is_nested('[[]][[') ➞ True + +Here's a correct unverified solution in python: +```python +def is_nested(string): + opening_bracket_index = [] + closing_bracket_index = [] + for i in range(len(string)): + if string[i] == '[': + opening_bracket_index.append(i) + else: + closing_bracket_index.append(i) + closing_bracket_index.reverse() + cnt = 0 + i = 0 + l = len(closing_bracket_index) + for idx in opening_bracket_index: + if i < l and idx < closing_bracket_index[i]: + cnt += 1 + i += 1 + return cnt >= 2 + + +def check(is_nested): + # Check some simple cases + assert is_nested('[[]]') == True, "This prints if this assert fails 1 (good for debugging!)" + assert is_nested('[]]]]]]][[[[[]') == False + assert is_nested('[][]') == False + assert is_nested('[]') == False + assert is_nested('[[]][[') == True + assert is_nested('[[][]]') == True + # Check some edge cases that are easy to work out by hand. +check(is_nested) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/is_palindrome.prompt b/benches/HumanEvalPromptsWithPythonSol/is_palindrome.prompt new file mode 100644 index 0000000..79097aa --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/is_palindrome.prompt @@ -0,0 +1,45 @@ + +Checks if given string is a palindrome +>>> is_palindrome('') +True +>>> is_palindrome('aba') +True +>>> is_palindrome('aaaaa') +True +>>> is_palindrome('zbcd') +False + +Here's a correct unverified solution in python: +```python +def is_palindrome(string: str) -> bool: + """ Test if given string is a palindrome """ + return string == string[::-1] + + +def make_palindrome(string: str) -> str: + if not string: + return '' + + beginning_of_suffix = 0 + + while not is_palindrome(string[beginning_of_suffix:]): + beginning_of_suffix += 1 + + return string + string[:beginning_of_suffix][::-1] +def check(make_palindrome): + assert make_palindrome('') == '' + assert make_palindrome('cat') == 'catac' + assert make_palindrome('cata') == 'catac' +check(make_palindrome) +def is_palindrome(text: str): + for i in range(len(text)): + if text[i] != text[len(text) - 1 - i]: + return False + return True +def check(is_palindrome): + assert is_palindrome('') == True + assert is_palindrome('aba') == True + assert is_palindrome('aaaaa') == True + assert is_palindrome('zbcd') == False +check(is_palindrome) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/is_prime.prompt b/benches/HumanEvalPromptsWithPythonSol/is_prime.prompt new file mode 100644 index 0000000..568322e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/is_prime.prompt @@ -0,0 +1,35 @@ +Return true if a given number is prime, and false otherwise. +>>> is_prime(6) +False +>>> is_prime(101) +True +>>> is_prime(11) +True +>>> is_prime(13441) +True +>>> is_prime(61) +True +>>> is_prime(4) +False +>>> is_prime(1) +False + +Here's a correct unverified solution in python: +```python +def is_prime(n): + if n < 2: + return False + for k in range(2, n - 1): + if n % k == 0: + return False + return True +def check(is_prime): + assert is_prime(6) == False + assert is_prime(101) == True + assert is_prime(11) == True + assert is_prime(13441) == True + assert is_prime(61) == True + assert is_prime(4) == False + assert is_prime(1) == False +check(is_prime) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/is_simple_power.prompt b/benches/HumanEvalPromptsWithPythonSol/is_simple_power.prompt new file mode 100644 index 0000000..0b551a2 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/is_simple_power.prompt @@ -0,0 +1,31 @@ +Your task is to write a function that returns true if a number x is a simple +power of n and false in other cases. +x is a simple power of n if n**int=x +For example: +is_simple_power(1, 4) => true +is_simple_power(2, 2) => true +is_simple_power(8, 2) => true +is_simple_power(3, 2) => false +is_simple_power(3, 1) => false +is_simple_power(5, 3) => false + +Here's a correct unverified solution in python: +```python +def is_simple_power(x, n): + if (n == 1): + return (x == 1) + power = 1 + while (power < x): + power = power * n + return (power == x) +def check(is_simple_power): + # Check some simple cases + assert is_simple_power(1, 4)== True, "This prints if this assert fails 1 (good for debugging!)" + assert is_simple_power(2, 2)==True, "This prints if this assert fails 1 (good for debugging!)" + assert is_simple_power(8, 2)==True, "This prints if this assert fails 1 (good for debugging!)" + assert is_simple_power(3, 2)==False, "This prints if this assert fails 1 (good for debugging!)" + assert is_simple_power(3, 1)==False, "This prints if this assert fails 1 (good for debugging!)" + assert is_simple_power(5, 3)==False, "This prints if this assert fails 1 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. +check(is_simple_power) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/is_sorted.prompt b/benches/HumanEvalPromptsWithPythonSol/is_sorted.prompt new file mode 100644 index 0000000..c430068 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/is_sorted.prompt @@ -0,0 +1,42 @@ + +Given a list of numbers, return whether or not they are sorted +in ascending order. If list has more than 1 duplicate of the same +number, return False. Assume no negative numbers and only integers. + +Examples +is_sorted([5]) ➞ True +is_sorted([1, 2, 3, 4, 5]) ➞ True +is_sorted([1, 3, 2, 4, 5]) ➞ False +is_sorted([1, 2, 3, 4, 5, 6]) ➞ True +is_sorted([1, 2, 3, 4, 5, 6, 7]) ➞ True +is_sorted([1, 3, 2, 4, 5, 6, 7]) ➞ False +is_sorted([1, 2, 2, 3, 3, 4]) ➞ True +is_sorted([1, 2, 2, 2, 3, 4]) ➞ False + +Here's a correct unverified solution in python: +```python +def is_sorted(lst): + count_digit = dict([(i, 0) for i in lst]) + for i in lst: + count_digit[i]+=1 + if any(count_digit[i] > 2 for i in lst): + return False + if all(lst[i-1] <= lst[i] for i in range(1, len(lst))): + return True + else: + return False + + +def check(is_sorted): + # Check some simple cases + assert is_sorted([5]) == True + assert is_sorted([1, 2, 3, 4, 5]) == True + assert is_sorted([1, 3, 2, 4, 5]) == False + assert is_sorted([1, 2, 3, 4, 5, 6]) == True + assert is_sorted([1, 2, 3, 4, 5, 6, 7]) == True + assert is_sorted([1, 3, 2, 4, 5, 6, 7]) == False, "This prints if this assert fails 1 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert is_sorted([1, 2, 2, 2, 3, 4]) == False, "This prints if this assert fails 5 (good for debugging!)" + assert is_sorted([1, 2, 2, 3, 3, 4]) == True, "This prints if this assert fails 7 (good for debugging!)" +check(is_sorted) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/iscube.prompt b/benches/HumanEvalPromptsWithPythonSol/iscube.prompt new file mode 100644 index 0000000..fd20d70 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/iscube.prompt @@ -0,0 +1,28 @@ + +Write a function that takes an integer a and returns True +if this ingeger is a cube of some integer number. +Note: you may assume the input is always valid. +Examples: +iscube(1) ==> True +iscube(2) ==> False +iscube(-1) ==> True +iscube(64) ==> True +iscube(0) ==> True +iscube(180) ==> False + +Here's a correct unverified solution in python: +```python +def iscube(a): + a = abs(a) + return int(round(a ** (1. / 3))) ** 3 == a +def check(iscube): + # Check some simple cases + assert iscube(1) == True, "First test error: " + str(iscube(1)) + assert iscube(2) == False, "Second test error: " + str(iscube(2)) + assert iscube(-1) == True, "Third test error: " + str(iscube(-1)) + assert iscube(64) == True, "Fourth test error: " + str(iscube(64)) + assert iscube(180) == False, "Fifth test error: " + str(iscube(180)) + # Check some edge cases that are easy to work out by hand. + assert iscube(0) == True, "1st edge test error: " + str(iscube(0)) +check(iscube) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/largest_divisor.prompt b/benches/HumanEvalPromptsWithPythonSol/largest_divisor.prompt new file mode 100644 index 0000000..7c27d0d --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/largest_divisor.prompt @@ -0,0 +1,14 @@ + For a given number n, find the largest number that divides n evenly, smaller than n +>>> largest_divisor(15) +5 + +Here's a correct unverified solution in python: +```python +def largest_divisor(n: int) -> int: + for i in reversed(range(n)): + if n % i == 0: + return i +def check(largest_divisor): + assert largest_divisor(15) == 5 +check(largest_divisor) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/largest_prime_factor.prompt b/benches/HumanEvalPromptsWithPythonSol/largest_prime_factor.prompt new file mode 100644 index 0000000..0a06d8a --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/largest_prime_factor.prompt @@ -0,0 +1,26 @@ +Return the largest prime factor of n. Assume n > 1 and is not a prime. +>>> largest_prime_factor(13195) +29 +>>> largest_prime_factor(2048) +2 + +Here's a correct unverified solution in python: +```python +def largest_prime_factor(n: int): + def is_prime(k): + if k < 2: + return False + for i in range(2, k - 1): + if k % i == 0: + return False + return True + largest = 1 + for j in range(2, n + 1): + if n % j == 0 and is_prime(j): + largest = max(largest, j) + return largest +def check(largest_prime_factor): + assert largest_prime_factor(2048) == 2 + assert largest_prime_factor(13195) == 29 +check(largest_prime_factor) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/largest_smallest_integers.prompt b/benches/HumanEvalPromptsWithPythonSol/largest_smallest_integers.prompt new file mode 100644 index 0000000..22acd4f --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/largest_smallest_integers.prompt @@ -0,0 +1,26 @@ + +Create a function that returns a tuple (a, b), where 'a' is +the largest of negative integers, and 'b' is the smallest +of positive integers in a list. +If there is no negative or positive integers, return them as None. + +Examples: +largest_smallest_integers([2, 4, 1, 3, 5, 7]) == (None, 1) +largest_smallest_integers([]) == (None, None) +largest_smallest_integers([0]) == (None, None) + +Here's a correct unverified solution in python: +```python +def largest_smallest_integers(lst): + smallest = list(filter(lambda x: x < 0, lst)) + largest = list(filter(lambda x: x > 0, lst)) + return (max(smallest) if smallest else None, min(largest) if largest else None) +def check(largest_smallest_integers): + # Check some simple cases + assert largest_smallest_integers([2, 4, 1, 3, 5, 7]) == (None, 1) + assert largest_smallest_integers([]) == (None, None) + assert largest_smallest_integers([0]) == (None, None) + # Check some edge cases that are easy to work out by hand. + assert True +check(largest_smallest_integers) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/longest.prompt b/benches/HumanEvalPromptsWithPythonSol/longest.prompt new file mode 100644 index 0000000..0336c5d --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/longest.prompt @@ -0,0 +1,28 @@ + Out of list of strings, return the longest one. Return the first one in case of multiple +strings of the same length. Return None in case the input list is empty. +>>> longest([]) + +>>> longest(['a', 'b', 'c']) +'a' +>>> longest(['a', 'bb', 'ccc']) +'ccc' + +Here's a correct unverified solution in python: +```python +from typing import List, Optional + + +def longest(strings: List[str]) -> Optional[str]: + if not strings: + return None + + maxlen = max(len(x) for x in strings) + for s in strings: + if len(s) == maxlen: + return s +def check(longest): + assert longest([]) == None + assert longest(['a', 'b', 'c']) == 'a' + assert longest(['a', 'bb', 'ccc']) == 'ccc' +check(longest) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/make_a_pile.prompt b/benches/HumanEvalPromptsWithPythonSol/make_a_pile.prompt new file mode 100644 index 0000000..540f574 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/make_a_pile.prompt @@ -0,0 +1,24 @@ + +Given a positive integer n, you have to make a pile of n levels of stones. +The first level has n stones. +The number of stones in the next level is: + - the next odd number if n is odd. + - the next even number if n is even. +Return the number of stones in each level in a list, where element at index +i represents the number of stones in the level (i+1). + +Examples: +>>> make_a_pile(3) +[3, 5, 7] + +Here's a correct unverified solution in python: +```python +def make_a_pile(n): + return [n + 2*i for i in range(n)] +def check(make_a_pile): + # Check some simple cases + assert make_a_pile(3) == [3, 5, 7], "Test 3" + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(make_a_pile) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/match_parens.prompt b/benches/HumanEvalPromptsWithPythonSol/match_parens.prompt new file mode 100644 index 0000000..db7e881 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/match_parens.prompt @@ -0,0 +1,63 @@ + +You are given a list of two strings, both strings consist of open +parentheses '(' or close parentheses ')' only. +Your job is to check if it is possible to concatenate the two strings in +some order, that the resulting string will be good. +A string S is considered to be good if and only if all parentheses in S +are balanced. For example: the string '(())()' is good, while the string +'())' is not. +Return 'Yes' if there's a way to make a good string, and return 'No' otherwise. + +Examples: +match_parens(['()(', ')']) == 'Yes' +match_parens([')', ')']) == 'No' + +Here's a correct unverified solution in python: +```python +def match_parens(lst): + def check(s): + val = 0 + for i in s: + if i == '(': + val = val + 1 + else: + val = val - 1 + if val < 0: + return False + return True if val == 0 else False + + S1 = lst[0] + lst[1] + S2 = lst[1] + lst[0] + return 'Yes' if check(S1) or check(S2) else 'No' + def check(s): + val = 0 + for i in s: + if i == '(': + val = val + 1 + else: + val = val - 1 + if val < 0: + return False + return True if val == 0 else False + S1 = lst[0] + lst[1] + S2 = lst[1] + lst[0] + return 'Yes' if check(S1) or check(S2) else 'No' + def check(s): + val = 0 + for i in s: + if i == '(': + val = val + 1 + else: + val = val - 1 + if val < 0: + return False + return True if val == 0 else False + S1 = lst[0] + lst[1] + S2 = lst[1] + lst[0] + return 'Yes' if check(S1) or check(S2) else 'No' +def check(match_parens): + # Check some simple cases + assert match_parens(['()(', ')']) == 'Yes' + assert match_parens([')', ')']) == 'No' +check(match_parens) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/max_element.prompt b/benches/HumanEvalPromptsWithPythonSol/max_element.prompt new file mode 100644 index 0000000..ed67f6b --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/max_element.prompt @@ -0,0 +1,19 @@ +Return maximum element in the list. +>>> max_element([1, 2, 3]) +3 +>>> max_element([5, 3, -5, 2, -3, 3, 9, 0, 123, 1, -10]) +123 + +Here's a correct unverified solution in python: +```python +def max_element(l: list): + m = l[0] + for e in l: + if e > m: + m = e + return m +def check(max_element): + assert max_element([1, 2, 3]) == 3 + assert max_element([5, 3, -5, 2, -3, 3, 9, 0, 123, 1, -10]) == 123 +check(max_element) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/max_fill.prompt b/benches/HumanEvalPromptsWithPythonSol/max_fill.prompt new file mode 100644 index 0000000..549ec46 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/max_fill.prompt @@ -0,0 +1,47 @@ + +You are given a rectangular grid of wells. Each row represents a single well, +and each 1 in a row represents a single unit of water. +Each well has a corresponding bucket that can be used to extract water from it, +and all buckets have the same capacity. +Your task is to use the buckets to empty the wells. +Output the number of times you need to lower the buckets. + +Example 1: + Input: + grid : [[0,0,1,0], [0,1,0,0], [1,1,1,1]] + bucket_capacity : 1 + Output: 6 + +Example 2: + Input: + grid : [[0,0,1,1], [0,0,0,0], [1,1,1,1], [0,1,1,1]] + bucket_capacity : 2 + Output: 5 + +Example 3: + Input: + grid : [[0,0,0], [0,0,0]] + bucket_capacity : 5 + Output: 0 + +Constraints: + * all wells have the same length + * 1 <= grid.length <= 10^2 + * 1 <= grid[:,1].length <= 10^2 + * grid[i][j] -> 0 | 1 + * 1 <= capacity <= 10 + +Here's a correct unverified solution in python: +```python +def max_fill(grid, capacity): + import math + return sum([math.ceil(sum(arr)/capacity) for arr in grid]) +def check(max_fill): + # Check some simple cases + assert True, "This prints if this assert fails 1 (good for debugging!)" + assert max_fill([[0,0,1,0], [0,1,0,0], [1,1,1,1]], 1) == 6, "Error" + assert max_fill([[0,0,1,1], [0,0,0,0], [1,1,1,1], [0,1,1,1]], 2) == 5, "Error" + assert max_fill([[0,0,0], [0,0,0]], 5) == 0, "Error" + # Check some edge cases that are easy to work out by hand. +check(max_fill) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/maximum.prompt b/benches/HumanEvalPromptsWithPythonSol/maximum.prompt new file mode 100644 index 0000000..c9d637a --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/maximum.prompt @@ -0,0 +1,39 @@ + +Given an array arr of integers and a positive integer k, return a sorted list +of length k with the maximum k numbers in arr. + +Example 1: + + Input: arr = [-3, -4, 5], k = 3 + Output: [-4, -3, 5] + +Example 2: + + Input: arr = [4, -4, 4], k = 2 + Output: [4, 4] + +Example 3: + + Input: arr = [-3, 2, 1, 2, -1, -2, 1], k = 1 + Output: [2] + +Note: + 1. The length of the array will be in the range of [1, 1000]. + 2. The elements in the array will be in the range of [-1000, 1000]. + 3. 0 <= k <= len(arr) + +Here's a correct unverified solution in python: +```python +def maximum(arr, k): + if k == 0: + return [] + arr.sort() + ans = arr[-k:] + return ans +def check(maximum): + # Check some simple cases + assert maximum([-3, -4, 5], 3) == [-4, -3, 5] + assert maximum([4, -4, 4], 2) == [4, 4] + assert maximum([-3, 2, 1, 2, -1, -2, 1], 1) == [2] +check(maximum) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/mean_absolute_derivation.prompt b/benches/HumanEvalPromptsWithPythonSol/mean_absolute_derivation.prompt new file mode 100644 index 0000000..264487f --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/mean_absolute_derivation.prompt @@ -0,0 +1,17 @@ + For a given list of input numbers, calculate Mean Absolute Deviation +around the mean of this dataset. +Mean Absolute Deviation is the average absolute difference between each +element and a centerpoint (mean in this case): +MAD = average | x - x_mean | +>>> mean_absolute_deviation([1.0, 2.0, 3.0, 4.0]) +1.0 + +Here's a correct unverified solution in python: +```python +from typing import List + + +def mean_absolute_deviation(numbers: List[float]) -> float: + mean = sum(numbers) / len(numbers) + return sum(abs(x - mean) for x in numbers) / len(numbers) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/median.prompt b/benches/HumanEvalPromptsWithPythonSol/median.prompt new file mode 100644 index 0000000..a8e81d3 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/median.prompt @@ -0,0 +1,19 @@ +Return median of elements in the list l. +>>> median([3, 1, 2, 4, 5]) +3 +>>> median([-10, 4, 6, 1000, 10, 20]) +15.0 + +Here's a correct unverified solution in python: +```python +def median(l: list): + l = sorted(l) + if len(l) % 2 == 1: + return l[len(l) // 2] + else: + return (l[len(l) // 2 - 1] + l[len(l) // 2]) / 2.0 +def check(median): + assert median([3, 1, 2, 4, 5]) == 3 + assert median([-10, 4, 6, 1000, 10, 20]) == 8.0 +check(median) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/minPath.prompt b/benches/HumanEvalPromptsWithPythonSol/minPath.prompt new file mode 100644 index 0000000..c14b75e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/minPath.prompt @@ -0,0 +1,66 @@ + +Given a grid with N rows and N columns (N >= 2) and a positive integer k, +each cell of the grid contains a value. Every integer in the range [1, N * N] +inclusive appears exactly once on the cells of the grid. + +You have to find the minimum path of length k in the grid. You can start +from any cell, and in each step you can move to any of the neighbor cells, +in other words, you can go to cells which share an edge with you current +cell. +Please note that a path of length k means visiting exactly k cells (not +necessarily distinct). +You CANNOT go off the grid. +A path A (of length k) is considered less than a path B (of length k) if +after making the ordered lists of the values on the cells that A and B go +through (let's call them lst_A and lst_B), lst_A is lexicographically less +than lst_B, in other words, there exist an integer index i (1 <= i <= k) +such that lst_A[i] < lst_B[i] and for any j (1 <= j < i) we have +lst_A[j] = lst_B[j]. +It is guaranteed that the answer is unique. +Return an ordered list of the values on the cells that the minimum path go through. + +Examples: + + Input: grid = [ [1,2,3], [4,5,6], [7,8,9]], k = 3 + Output: [1, 2, 1] + + Input: grid = [ [5,9,3], [4,1,6], [7,8,2]], k = 1 + Output: [1] + +Here's a correct unverified solution in python: +```python +def minPath(grid, k): + n = len(grid) + val = n * n + 1 + for i in range(n): + for j in range(n): + if grid[i][j] == 1: + temp = [] + if i != 0: + temp.append(grid[i - 1][j]) + + if j != 0: + temp.append(grid[i][j - 1]) + + if i != n - 1: + temp.append(grid[i + 1][j]) + + if j != n - 1: + temp.append(grid[i][j + 1]) + + val = min(temp) + + ans = [] + for i in range(k): + if i % 2 == 0: + ans.append(1) + else: + ans.append(val) + return ans +def check(minPath): + # Check some simple cases + print + assert minPath([[1, 2, 3], [4, 5, 6], [7, 8, 9]], 3) == [1, 2, 1] + assert minPath([[5, 9, 3], [4, 1, 6], [7, 8, 2]], 1) == [1] +check(minPath) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/minSubArraySum.prompt b/benches/HumanEvalPromptsWithPythonSol/minSubArraySum.prompt new file mode 100644 index 0000000..e61e43e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/minSubArraySum.prompt @@ -0,0 +1,27 @@ + +Given an array of integers nums, find the minimum sum of any non-empty sub-array +of nums. +Example +minSubArraySum([2, 3, 4, 1, 2, 4]) == 1 +minSubArraySum([-1, -2, -3]) == -6 + +Here's a correct unverified solution in python: +```python +def minSubArraySum(nums): + max_sum = 0 + s = 0 + for num in nums: + s += -num + if (s < 0): + s = 0 + max_sum = max(s, max_sum) + if max_sum == 0: + max_sum = max(-i for i in nums) + min_sum = -max_sum + return min_sum +def check(minSubArraySum): + # Check some simple cases + assert minSubArraySum([2, 3, 4, 1, 2, 4]) == 1, "This prints if this assert fails 1 (good for debugging!)" + assert minSubArraySum([-1, -2, -3]) == -6 +check(minSubArraySum) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/modp.prompt b/benches/HumanEvalPromptsWithPythonSol/modp.prompt new file mode 100644 index 0000000..59b537a --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/modp.prompt @@ -0,0 +1,27 @@ +Return 2^n modulo p (be aware of numerics). +>>> modp(3, 5) +3 +>>> modp(1101, 101) +2 +>>> modp(0, 101) +1 +>>> modp(3, 11) +8 +>>> modp(100, 101) +1 + +Here's a correct unverified solution in python: +```python +def modp(n: int, p: int): + ret = 1 + for i in range(n): + ret = (2 * ret) % p + return ret +def check(modp): + assert modp(3, 5) == 3 + assert modp(1101, 101) == 2 + assert modp(0, 101) == 1 + assert modp(3, 11) == 8 + assert modp(100, 101) == 1 +check(modp) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/monotonic.prompt b/benches/HumanEvalPromptsWithPythonSol/monotonic.prompt new file mode 100644 index 0000000..e72ea2a --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/monotonic.prompt @@ -0,0 +1,20 @@ +Return True is list elements are monotonically increasing or decreasing. +>>> monotonic([1, 2, 4, 20]) +True +>>> monotonic([1, 20, 4, 10]) +False +>>> monotonic([4, 1, 0, -10]) +True + +Here's a correct unverified solution in python: +```python +def monotonic(l: list): + if l == sorted(l) or l == sorted(l, reverse=True): + return True + return False +def check(monotonic): + assert monotonic([1, 2, 4, 10]) == True + assert monotonic([1, 20, 4, 10]) == False + assert monotonic([4, 1, 0, -10]) == True +check(monotonic) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/move_one_ball.prompt b/benches/HumanEvalPromptsWithPythonSol/move_one_ball.prompt new file mode 100644 index 0000000..3024c69 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/move_one_ball.prompt @@ -0,0 +1,48 @@ +We have an array 'arr' of N integers arr[1], arr[2], ..., arr[N].The +numbers in the array will be randomly ordered. Your task is to determine if +it is possible to get an array sorted in non-decreasing order by performing +the following operation on the given array: + You are allowed to perform right shift operation any number of times. + +One right shift operation means shifting all elements of the array by one +position in the right direction. The last element of the array will be moved to +the starting position in the array i.e. 0th index. + +If it is possible to obtain the sorted array by performing the above operation +then return True else return False. +If the given array is empty then return True. + +Note: The given list is guaranteed to have unique elements. + +For Example: + +move_one_ball([3, 4, 5, 1, 2])==>True +Explanation: By performin 2 right shift operations, non-decreasing order can + be achieved for the given array. +move_one_ball([3, 5, 4, 1, 2])==>False +Explanation:It is not possible to get non-decreasing order for the given + array by performing any number of right shift operations. + + +Here's a correct unverified solution in python: +```python +def move_one_ball(arr): + if len(arr)==0: + return True + sorted_array=sorted(arr) + my_arr=[] + + min_value=min(arr) + min_index=arr.index(min_value) + my_arr=arr[min_index:]+arr[0:min_index] + for i in range(len(arr)): + if my_arr[i]!=sorted_array[i]: + return False + return True +def check(move_one_ball): + # Check some simple cases + assert move_one_ball([3, 4, 5, 1, 2])==True, "This prints if this assert fails 1 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert move_one_ball([3, 5, 4, 1, 2])==False, "This prints if this assert fails 2 (also good for debugging!)" +check(move_one_ball) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/multiply.prompt b/benches/HumanEvalPromptsWithPythonSol/multiply.prompt new file mode 100644 index 0000000..db80778 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/multiply.prompt @@ -0,0 +1,21 @@ +Complete the function that takes two integers and returns +the product of their unit digits. +Assume the input is always valid. +Examples: +multiply(148, 412) should return 16. +multiply(19, 28) should return 72. +multiply(2020, 1851) should return 0. +multiply(14,-15) should return 20. + +Here's a correct unverified solution in python: +```python +def multiply(a, b): + return abs(a % 10) * abs(b % 10) +def check(multiply): + # Check some simple cases + assert multiply(148, 412) == 16, "First test error: " + str(multiply(148, 412)) + assert multiply(19, 28) == 72, "Second test error: " + str(multiply(19, 28)) + assert multiply(2020, 1851) == 0, "Third test error: " + str(multiply(2020, 1851)) + assert multiply(14,-15) == 20, "Fourth test error: " + str(multiply(14,-15)) +check(multiply) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/next_smallest.prompt b/benches/HumanEvalPromptsWithPythonSol/next_smallest.prompt new file mode 100644 index 0000000..ca10de6 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/next_smallest.prompt @@ -0,0 +1,25 @@ + +You are given a list of integers. +Write a function next_smallest() that returns the 2nd smallest element of the list. +Return None if there is no such element. + +next_smallest([1, 2, 3, 4, 5]) == 2 +next_smallest([5, 1, 4, 3, 2]) == 2 +next_smallest([]) == None +next_smallest([1, 1]) == None + +Here's a correct unverified solution in python: +```python +def next_smallest(lst): + lst = sorted(set(lst)) + return None if len(lst) < 2 else lst[1] +def check(next_smallest): + # Check some simple cases + assert next_smallest([1, 2, 3, 4, 5]) == 2 + assert next_smallest([5, 1, 4, 3, 2]) == 2 + assert next_smallest([]) == None + assert next_smallest([1, 1]) == None + # Check some edge cases that are easy to work out by hand. + assert True +check(next_smallest) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/numerical_letter_grade.prompt b/benches/HumanEvalPromptsWithPythonSol/numerical_letter_grade.prompt new file mode 100644 index 0000000..da59fc7 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/numerical_letter_grade.prompt @@ -0,0 +1,65 @@ +It is the last week of the semester and the teacher has to give the grades +to students. The teacher has been making her own algorithm for grading. +The only problem is, she has lost the code she used for grading. +She has given you a list of GPAs for some students and you have to write +a function that can output a list of letter grades using the following table: + GPA | Letter grade + 4.0 A+ + > 3.7 A + > 3.3 A- + > 3.0 B+ + > 2.7 B + > 2.3 B- + > 2.0 C+ + > 1.7 C + > 1.3 C- + > 1.0 D+ + > 0.7 D + > 0.0 D- + 0.0 E + + +Example: +grade_equation([4.0, 3, 1.7, 2, 3.5]) ==> ['A+', 'B', 'C-', 'C', 'A-'] + +Here's a correct unverified solution in python: +```python +def numerical_letter_grade(grades): + + + letter_grade = [] + for gpa in grades: + if gpa == 4.0: + letter_grade.append("A+") + elif gpa > 3.7: + letter_grade.append("A") + elif gpa > 3.3: + letter_grade.append("A-") + elif gpa > 3.0: + letter_grade.append("B+") + elif gpa > 2.7: + letter_grade.append("B") + elif gpa > 2.3: + letter_grade.append("B-") + elif gpa > 2.0: + letter_grade.append("C+") + elif gpa > 1.7: + letter_grade.append("C") + elif gpa > 1.3: + letter_grade.append("C-") + elif gpa > 1.0: + letter_grade.append("D+") + elif gpa > 0.7: + letter_grade.append("D") + elif gpa > 0.0: + letter_grade.append("D-") + else: + letter_grade.append("E") + return letter_grade +def check(numerical_letter_grade): + # Check some simple cases + assert numerical_letter_grade([4.0, 3, 1.7, 2, 3.5]) == ['A+', 'B', 'C-', 'C', 'A-'] + # Check some edge cases that are easy to work out by hand. + assert True +check(numerical_letter_grade) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/odd_count.prompt b/benches/HumanEvalPromptsWithPythonSol/odd_count.prompt new file mode 100644 index 0000000..3ff6aad --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/odd_count.prompt @@ -0,0 +1,27 @@ +Given a list of strings, where each string consists of only digits, return a list. +Each element i of the output should be "the number of odd elements in the +string i of the input." where all the i's should be replaced by the number +of odd digits in the i'th string of the input. + +>>> odd_count(['1234567']) +["the number of odd elements 4n the str4ng 4 of the 4nput."] +>>> odd_count(['3',"11111111"]) +["the number of odd elements 1n the str1ng 1 of the 1nput.", + "the number of odd elements 8n the str8ng 8 of the 8nput."] + +Here's a correct unverified solution in python: +```python +def odd_count(lst): + res = [] + for arr in lst: + n = sum(int(d)%2==1 for d in arr) + res.append("the number of odd elements " + str(n) + "n the str"+ str(n) +"ng "+ str(n) +" of the "+ str(n) +"nput.") + return res +def check(odd_count): + # Check some simple cases + assert odd_count(['1234567']) == ["the number of odd elements 4n the str4ng 4 of the 4nput."], "Test 1" + assert odd_count(['3',"11111111"]) == ["the number of odd elements 1n the str1ng 1 of the 1nput.", "the number of odd elements 8n the str8ng 8 of the 8nput."], "Test 2" + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(odd_count) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/order_by_points.prompt b/benches/HumanEvalPromptsWithPythonSol/order_by_points.prompt new file mode 100644 index 0000000..ef6e3e5 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/order_by_points.prompt @@ -0,0 +1,26 @@ + +Write a function which sorts the given list of integers +in ascending order according to the sum of their digits. +Note: if there are several items with similar sum of their digits, +order them based on their index in original list. + +For example: +>>> order_by_points([1, 11, -1, -11, -12]) == [-1, -11, 1, -12, 11] +>>> order_by_points([]) == [] + +Here's a correct unverified solution in python: +```python +def order_by_points(nums): + def digits_sum(n): + neg = 1 + if n < 0: n, neg = -1 * n, -1 + n = [int(i) for i in str(n)] + n[0] = n[0] * neg + return sum(n) + return sorted(nums, key=digits_sum) +def check(order_by_points): + # Check some simple cases + assert order_by_points([1, 11, -1, -11, -12]) == [-1, -11, 1, -12, 11] + assert order_by_points([]) == [] +check(order_by_points) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/pairs_sum_to_zero.prompt b/benches/HumanEvalPromptsWithPythonSol/pairs_sum_to_zero.prompt new file mode 100644 index 0000000..6c4150e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/pairs_sum_to_zero.prompt @@ -0,0 +1,30 @@ + +pairs_sum_to_zero takes a list of integers as an input. +it returns True if there are two distinct elements in the list that +sum to zero, and False otherwise. +>>> pairs_sum_to_zero([1, 3, 5, 0]) +False +>>> pairs_sum_to_zero([1, 3, -2, 1]) +False +>>> pairs_sum_to_zero([1, 2, 3, 7]) +False +>>> pairs_sum_to_zero([2, 4, -5, 3, 5, 7]) +True +>>> pairs_sum_to_zero([1]) +False + +Here's a correct unverified solution in python: +```python +def pairs_sum_to_zero(l): + for i, l1 in enumerate(l): + for j in range(i + 1, len(l)): + if l1 + l[j] == 0: + return True + return False +def check(pairs_sum_to_zero): + assert pairs_sum_to_zero([1, 3, 5, 0]) == False + assert pairs_sum_to_zero([1, 3, -2, 1]) == False + assert pairs_sum_to_zero([1, 2, 3, 7]) == False + assert pairs_sum_to_zero([2, 4, -5, 3, 5, 7]) == True +check(pairs_sum_to_zero) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/parse_music.prompt b/benches/HumanEvalPromptsWithPythonSol/parse_music.prompt new file mode 100644 index 0000000..a616a55 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/parse_music.prompt @@ -0,0 +1,24 @@ + Input to this function is a string representing musical notes in a special ASCII format. +Your task is to parse this string and return list of integers corresponding to how many beats does each +not last. + +Here is a legend: +'o' - whole note, lasts four beats +'o|' - half note, lasts two beats +'.|' - quater note, lasts one beat + +>>> parse_music('o o| .| o| o| .| .| .| .| o o') +[4, 2, 1, 2, 2, 1, 1, 1, 1, 4, 4] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def parse_music(music_string: str) -> List[int]: + note_map = {'o': 4, 'o|': 2, '.|': 1} + return [note_map[x] for x in music_string.split(' ') if x] +def check(parse_music): + assert parse_music('o o| .| o| o| .| .| .| .| o o') == [4, 2, 1, 2, 2, 1, 1, 1, 1, 4, 4] +check(parse_music) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/parse_nested_parens.prompt b/benches/HumanEvalPromptsWithPythonSol/parse_nested_parens.prompt new file mode 100644 index 0000000..9b00579 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/parse_nested_parens.prompt @@ -0,0 +1,31 @@ + Input to this function is a string represented multiple groups for nested parentheses separated by spaces. +For each of the group, output the deepest level of nesting of parentheses. +E.g. (()()) has maximum two levels of nesting while ((())) has three. + +>>> parse_nested_parens('(()()) ((())) () ((())()())') +[2, 3, 1, 3] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def parse_nested_parens(paren_string: str) -> List[int]: + def parse_paren_group(s): + depth = 0 + max_depth = 0 + for c in s: + if c == '(': + depth += 1 + max_depth = max(depth, max_depth) + else: + depth -= 1 + + return max_depth + + return [parse_paren_group(x) for x in paren_string.split(' ') if x] + +def check(parse_nested_parens): + assert parse_nested_parens('(()()) ((())) () ((())()())') == [2, 3, 1, 3] +check(parse_nested_parens) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/pluck.prompt b/benches/HumanEvalPromptsWithPythonSol/pluck.prompt new file mode 100644 index 0000000..a0f528a --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/pluck.prompt @@ -0,0 +1,51 @@ + +"Given an array representing a branch of a tree that has non-negative integer nodes +your task is to pluck one of the nodes and return it. +The plucked node should be the node with the smallest even value. +If multiple nodes with the same smallest even value are found return the node that has smallest index. + +The plucked node should be returned in a list, [ smalest_value, its index ], +If there are no even values or the given array is empty, return []. + +Example 1: + Input: [4,2,3] + Output: [2, 1] + Explanation: 2 has the smallest even value, and 2 has the smallest index. + +Example 2: + Input: [1,2,3] + Output: [2, 1] + Explanation: 2 has the smallest even value, and 2 has the smallest index. + +Example 3: + Input: [] + Output: [] + +Example 4: + Input: [5, 0, 3, 0, 4, 2] + Output: [0, 1] + Explanation: 0 is the smallest value, but there are two zeros, + so we will choose the first zero, which has the smallest index. + +Constraints: + * 1 <= nodes.length <= 10000 + * 0 <= node.value + +Here's a correct unverified solution in python: +```python +def pluck(arr): + if(len(arr) == 0): return [] + evens = list(filter(lambda x: x%2 == 0, arr)) + if(evens == []): return [] + return [min(evens), arr.index(min(evens))] +def check(pluck): + # Check some simple cases + assert True, "This prints if this assert fails 1 (good for debugging!)" + assert pluck([4,2,3]) == [2, 1], "Error" + assert pluck([1,2,3]) == [2, 1], "Error" + assert pluck([]) == [], "Error" + assert pluck([5, 0, 3, 0, 4, 2]) == [0, 1], "Error" + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(pluck) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/poly.prompt b/benches/HumanEvalPromptsWithPythonSol/poly.prompt new file mode 100644 index 0000000..8dd0dfa --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/poly.prompt @@ -0,0 +1,34 @@ + +Evaluates polynomial with coefficients xs at point x. +return xs[0] + xs[1] * x + xs[1] * x^2 + .... xs[n] * x^n + +Here's a correct unverified solution in python: +```python +import math + + +def poly(xs: list, x: float): + """ + Evaluates polynomial with coefficients xs at point x. + return xs[0] + xs[1] * x + xs[1] * x^2 + .... xs[n] * x^n + """ + return sum([coeff * math.pow(x, i) for i, coeff in enumerate(xs)]) + + +def find_zero(xs: list): + begin, end = -1., 1. + while poly(xs, begin) * poly(xs, end) > 0: + begin *= 2.0 + end *= 2.0 + while end - begin > 1e-10: + center = (begin + end) / 2.0 + if poly(xs, center) * poly(xs, begin) > 0: + begin = center + else: + end = center + return begin +def check(find_zero): + assert abs(find_zero([1,2])+0.5<1e-4) + assert abs(find_zero([-6,11,-6,1])-1<1e-4) +check(find_zero) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/prime_fib.prompt b/benches/HumanEvalPromptsWithPythonSol/prime_fib.prompt new file mode 100644 index 0000000..3a61275 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/prime_fib.prompt @@ -0,0 +1,40 @@ + +prime_fib returns n-th number that is a Fibonacci number and it's also prime. +>>> prime_fib(1) +2 +>>> prime_fib(2) +3 +>>> prime_fib(3) +5 +>>> prime_fib(4) +13 +>>> prime_fib(5) +89 + +Here's a correct unverified solution in python: +```python +def prime_fib(n: int): + import math + + def is_prime(p): + if p < 2: + return False + for k in range(2, min(int(math.sqrt(p)) + 1, p - 1)): + if p % k == 0: + return False + return True + f = [0, 1] + while True: + f.append(f[-1] + f[-2]) + if is_prime(f[-1]): + n -= 1 + if n == 0: + return f[-1] +def check(prime_fib): + assert prime_fib(1) == 2 + assert prime_fib(2) == 3 + assert prime_fib(3) == 5 + assert prime_fib(4) == 13 + assert prime_fib(5) == 89 +check(prime_fib) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/prime_length.prompt b/benches/HumanEvalPromptsWithPythonSol/prime_length.prompt new file mode 100644 index 0000000..8b9feb0 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/prime_length.prompt @@ -0,0 +1,26 @@ +Write a function that takes a string and returns True if the string +length is a prime number or False otherwise +Examples +prime_length('Hello') == True +prime_length('abcdcba') == True +prime_length('kittens') == True +prime_length('orange') == False + +Here's a correct unverified solution in python: +```python +def prime_length(string): + l = len(string) + if l == 0 or l == 1: + return False + for i in range(2, l): + if l % i == 0: + return False + return True +def check(prime_length): + # Check some simple cases + assert prime_length('Hello') == True + assert prime_length('abcdcba') == True + assert prime_length('kittens') == True + assert prime_length('orange') == False +check(prime_length) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/prod_signs.prompt b/benches/HumanEvalPromptsWithPythonSol/prod_signs.prompt new file mode 100644 index 0000000..36ee555 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/prod_signs.prompt @@ -0,0 +1,25 @@ + +You are given an array arr of integers and you need to return +sum of magnitudes of integers multiplied by product of all signs +of each number in the array, represented by 1, -1 or 0. +Note: return None for empty arr. + +Example: +>>> prod_signs([1, 2, 2, -4]) == -9 +>>> prod_signs([0, 1]) == 0 +>>> prod_signs([]) == None + +Here's a correct unverified solution in python: +```python +def prod_signs(arr): + if not arr: return None + prod = 0 if 0 in arr else (-1) ** len(list(filter(lambda x: x < 0, arr))) + return prod * sum([abs(i) for i in arr]) +def check(prod_signs): + # Check some simple cases + assert True, "This prints if this assert fails 1 (good for debugging!)" + assert prod_signs([1, 2, 2, -4]) == -9 + assert prod_signs([0, 1]) == 0 + assert prod_signs([]) == None +check(prod_signs) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/remove_duplicates.prompt b/benches/HumanEvalPromptsWithPythonSol/remove_duplicates.prompt new file mode 100644 index 0000000..73a1826 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/remove_duplicates.prompt @@ -0,0 +1,18 @@ + From a list of integers, remove all elements that occur more than once. +Keep order of elements left the same as in the input. +>>> remove_duplicates([1, 2, 3, 2, 4]) +[1, 3, 4] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def remove_duplicates(numbers: List[int]) -> List[int]: + import collections + c = collections.Counter(numbers) + return [n for n in numbers if c[n] <= 1] +def check(remove_duplicates): + assert remove_duplicates([1, 2, 3,2, 4]) == [1, 3, 4] +check(remove_duplicates) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/remove_vowels.prompt b/benches/HumanEvalPromptsWithPythonSol/remove_vowels.prompt new file mode 100644 index 0000000..369a05c --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/remove_vowels.prompt @@ -0,0 +1,28 @@ + +remove_vowels is a function that takes string and returns string without vowels. +>>> remove_vowels('') +'' +>>> remove_vowels("abcdef\nghijklm") +'bcdf\nghjklm' +>>> remove_vowels('abcdef') +'bcdf' +>>> remove_vowels('aaaaa') +'' +>>> remove_vowels('aaBAA') +'B' +>>> remove_vowels('zbcd') +'zbcd' + +Here's a correct unverified solution in python: +```python +def remove_vowels(text): + return "".join([s for s in text if s.lower() not in ["a", "e", "i", "o", "u"]]) +def check(remove_vowels): + assert remove_vowels('') == '' + assert remove_vowels("abcdef\nghijklm") == 'bcdf\nghjklm' + assert remove_vowels('abcdef') == 'bcdf' + assert remove_vowels('aaaaa') == '' + assert remove_vowels('aaBAA') == 'B' + assert remove_vowels('zbcd') == 'zbcd' +check(remove_vowels) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/rescale_to_unit.prompt b/benches/HumanEvalPromptsWithPythonSol/rescale_to_unit.prompt new file mode 100644 index 0000000..8f7d2a0 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/rescale_to_unit.prompt @@ -0,0 +1,18 @@ + Given list of numbers (of at least two elements), apply a linear transform to that list, +such that the smallest number will become 0 and the largest will become 1 +>>> rescale_to_unit([1.0, 2.0, 3.0, 4.0, 5.0]) +[0.0, 0.25, 0.5, 0.75, 1.0] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def rescale_to_unit(numbers: List[float]) -> List[float]: + min_number = min(numbers) + max_number = max(numbers) + return [(x - min_number) / (max_number - min_number) for x in numbers] +def check(rescale_to_unit): + assert rescale_to_unit([1.0, 2.0, 3.0, 4.0, 5.0]) == [0.0, 0.25, 0.5, 0.75, 1.0] +check(rescale_to_unit) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/reverse_delete.prompt b/benches/HumanEvalPromptsWithPythonSol/reverse_delete.prompt new file mode 100644 index 0000000..738631b --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/reverse_delete.prompt @@ -0,0 +1,21 @@ +Task +We are given two strings s and c, you have to deleted all the characters in s that are equal to any character in c +then check if the result string is palindrome. +A string is called palindrome if it reads the same backward as forward. +You should return a tuple containing the result string and True/False for the check. +Example +For s = "abcde", c = "ae", the result should be ('bcd',False) +For s = "abcdef", c = "b" the result should be ('acdef',False) +For s = "abcdedcba", c = "ab", the result should be ('cdedc',True) + +Here's a correct unverified solution in python: +```python +def reverse_delete(s,c): + s = ''.join([char for char in s if char not in c]) + return (s,s[::-1] == s) +def check(reverse_delete): + assert reverse_delete("abcde","ae") == ('bcd',False) + assert reverse_delete("abcdef", "b") == ('acdef',False) + assert reverse_delete("abcdedcba","ab") == ('cdedc',True) +check(reverse_delete) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/right_angle_triangle.prompt b/benches/HumanEvalPromptsWithPythonSol/right_angle_triangle.prompt new file mode 100644 index 0000000..89ad514 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/right_angle_triangle.prompt @@ -0,0 +1,19 @@ + +Given the lengths of the three sides of a triangle. Return True if the three +sides form a right-angled triangle, False otherwise. +A right-angled triangle is a triangle in which one angle is right angle or +90 degree. +Example: +right_angle_triangle(3, 4, 5) == True +right_angle_triangle(1, 2, 3) == False + +Here's a correct unverified solution in python: +```python +def right_angle_triangle(a, b, c): + return a*a == b*b + c*c or b*b == a*a + c*c or c*c == a*a + b*b +def check(right_angle_triangle): + # Check some simple cases + assert right_angle_triangle(3, 4, 5) == True, "This prints if this assert fails 1 (good for debugging!)" + assert right_angle_triangle(1, 2, 3) == False +check(right_angle_triangle) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/rolling_max.prompt b/benches/HumanEvalPromptsWithPythonSol/rolling_max.prompt new file mode 100644 index 0000000..cb15b82 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/rolling_max.prompt @@ -0,0 +1,27 @@ + From a given list of integers, generate a list of rolling maximum element found until given moment +in the sequence. +>>> rolling_max([1, 2, 3, 2, 3, 4, 2]) +[1, 2, 3, 3, 3, 4, 4] + +Here's a correct unverified solution in python: +```python +from typing import List, Tuple + + +def rolling_max(numbers: List[int]) -> List[int]: + running_max = None + result = [] + + for n in numbers: + if running_max is None: + running_max = n + else: + running_max = max(running_max, n) + + result.append(running_max) + + return result +def check(rolling_max): + assert rolling_max([1, 2, 3, 2, 3, 4, 2]) == [1, 2, 3, 3, 3, 4, 4] +check(rolling_max) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/rounded_avg.prompt b/benches/HumanEvalPromptsWithPythonSol/rounded_avg.prompt new file mode 100644 index 0000000..da053a4 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/rounded_avg.prompt @@ -0,0 +1,28 @@ +You are given two positive integers n and m, and your task is to compute the +average of the integers from n through m (including n and m). +Round the answer to the nearest integer and convert that to binary. +If n is greater than m, return -1. +Example: +rounded_avg(1, 5) => "0b11" +rounded_avg(7, 5) => -1 +rounded_avg(10, 20) => "0b1111" +rounded_avg(20, 33) => "0b11010" + +Here's a correct unverified solution in python: +```python +def rounded_avg(n, m): + if m < n: + return -1 + summation = 0 + for i in range(n, m+1): + summation += i + return bin(round(summation/(m - n + 1))) +def check(rounded_avg): + # Check some simple cases + assert rounded_avg(1, 5) == "0b11" + # Check some edge cases that are easy to work out by hand. + assert rounded_avg(7, 5) == -1 + assert rounded_avg(10,20) == "0b1111" + assert rounded_avg(20, 33) == "0b11010" +check(rounded_avg) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/same_chars.prompt b/benches/HumanEvalPromptsWithPythonSol/same_chars.prompt new file mode 100644 index 0000000..0f9b81a --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/same_chars.prompt @@ -0,0 +1,28 @@ + +Check if two words have the same characters. +>>> same_chars('eabcdzzzz', 'dddzzzzzzzddeddabc') +True +>>> same_chars('abcd', 'dddddddabc') +True +>>> same_chars('dddddddabc', 'abcd') +True +>>> same_chars('eabcd', 'dddddddabc') +False +>>> same_chars('abcd', 'dddddddabce') +False +>>> same_chars('eabcdzzzz', 'dddzzzzzzzddddabc') +False + +Here's a correct unverified solution in python: +```python +def same_chars(s0: str, s1: str): + return set(s0) == set(s1) +def check(same_chars): + assert same_chars('eabcdzzzz', 'dddzzzzzzzddeddabc') == True + assert same_chars('abcd', 'dddddddabc') == True + assert same_chars('dddddddabc', 'abcd') == True + assert same_chars('eabcd', 'dddddddabc') == False + assert same_chars('abcd', 'dddddddabcf') == False + assert same_chars('eabcdzzzz', 'dddzzzzzzzddddabc') == False +check(same_chars) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/search.prompt b/benches/HumanEvalPromptsWithPythonSol/search.prompt new file mode 100644 index 0000000..0b2b65f --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/search.prompt @@ -0,0 +1,30 @@ + +You are given a non-empty list of positive integers. Return the greatest integer that is greater than +zero, and has a frequency greater than or equal to the value of the integer itself. +The frequency of an integer is the number of times it appears in the list. +If no such a value exist, return -1. +Examples: + search([4, 1, 2, 2, 3, 1]) == 2 + search([1, 2, 2, 3, 3, 3, 4, 4, 4]) == 3 + search([5, 5, 4, 4, 4]) == -1 + +Here's a correct unverified solution in python: +```python +def search(lst): + frq = [0] * (max(lst) + 1) + for i in lst: + frq[i] += 1; + + ans = -1 + for i in range(1, len(frq)): + if frq[i] >= i: + ans = i + + return ans +def check(search): + # manually generated tests + assert search([4, 1, 2, 2, 3, 1]) == 2 + assert search([1, 2, 2, 3, 3, 3, 4, 4, 4]) == 3 + assert search([5, 5, 4, 4, 4]) == -1 +check(search) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/select_words.prompt b/benches/HumanEvalPromptsWithPythonSol/select_words.prompt new file mode 100644 index 0000000..d490542 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/select_words.prompt @@ -0,0 +1,35 @@ +Given a string s and a natural number n, you have been tasked to implement +a function that returns a list of all words from string s that contain exactly +n consonants, in order these words appear in the string s. +If the string s is empty then the function should return an empty list. +Note: you may assume the input string contains only letters and spaces. +Examples: +select_words("Mary had a little lamb", 4) ==> ["little"] +select_words("Mary had a little lamb", 3) ==> ["Mary", "lamb"] +select_words("simple white space", 2) ==> [] +select_words("Hello world", 4) ==> ["world"] +select_words("Uncle sam", 3) ==> ["Uncle"] + +Here's a correct unverified solution in python: +```python +def select_words(s, n): + result = [] + for word in s.split(): + n_consonants = 0 + for i in range(0, len(word)): + if word[i].lower() not in ["a","e","i","o","u"]: + n_consonants += 1 + if n_consonants == n: + result.append(word) + return result + +def check(select_words): + # Check some simple cases + assert select_words("Mary had a little lamb", 4) == ["little"], "First test error: " + str(select_words("Mary had a little lamb", 4)) + assert select_words("Mary had a little lamb", 3) == ["Mary", "lamb"], "Second test error: " + str(select_words("Mary had a little lamb", 3)) + assert select_words("simple white space", 2) == [], "Third test error: " + str(select_words("simple white space", 2)) + assert select_words("Hello world", 4) == ["world"], "Fourth test error: " + str(select_words("Hello world", 4)) + assert select_words("Uncle sam", 3) == ["Uncle"], "Fifth test error: " + str(select_words("Uncle sam", 3)) + # Check some edge cases that are easy to work out by hand. +check(select_words) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/separate_paren_groups.prompt b/benches/HumanEvalPromptsWithPythonSol/separate_paren_groups.prompt new file mode 100644 index 0000000..7b69a1e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/separate_paren_groups.prompt @@ -0,0 +1,35 @@ + Input to this function is a string containing multiple groups of nested parentheses. Your goal is to +separate those group into separate strings and return the list of those. +Separate groups are balanced (each open brace is properly closed) and not nested within each other +Ignore any spaces in the input string. +>>> separate_paren_groups('( ) (( )) (( )( ))') +['()', '(())', '(()())'] + +Here's a correct unverified solution in python: +```python +from typing import List + + +def separate_paren_groups(paren_string: str) -> List[str]: + result = [] + current_string = [] + current_depth = 0 + + for c in paren_string: + if c == '(': + current_depth += 1 + current_string.append(c) + elif c == ')': + current_depth -= 1 + current_string.append(c) + + if current_depth == 0: + result.append(''.join(current_string)) + current_string.clear() + + return result + +def check(separate_paren_groups): + assert separate_paren_groups('( ) (( )) (( )( ))') == ['()', '(())', '(()())'] +check(separate_paren_groups) +print(separate_paren_groups('( ) (( )) (( )( ))'))``` diff --git a/benches/HumanEvalPromptsWithPythonSol/simplify.prompt b/benches/HumanEvalPromptsWithPythonSol/simplify.prompt new file mode 100644 index 0000000..675eda3 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/simplify.prompt @@ -0,0 +1,28 @@ +Your task is to implement a function that will simplify the expression +x * n. The function returns True if x * n evaluates to a whole number and False +otherwise. Both x and n, are string representation of a fraction, and have the following format, +/ where both numerator and denominator are positive whole numbers. + +You can assume that x, and n are valid fractions, and do not have zero as denominator. + +simplify("1/5", "5/1") = True +simplify("1/6", "2/1") = False +simplify("7/10", "10/2") = False + +Here's a correct unverified solution in python: +```python +def simplify(x, n): + a, b = x.split("/") + c, d = n.split("/") + numerator = int(a) * int(c) + denom = int(b) * int(d) + if (numerator/denom == int(numerator/denom)): + return True + return False +def check(simplify): + # Check some simple cases + assert simplify("1/5", "5/1") == True, 'test1' + assert simplify("1/6", "2/1") == False, 'test2' + assert simplify("7/10", "10/2") == False, 'test4' +check(simplify) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/skjkasdkd.prompt b/benches/HumanEvalPromptsWithPythonSol/skjkasdkd.prompt new file mode 100644 index 0000000..d693702 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/skjkasdkd.prompt @@ -0,0 +1,44 @@ +You are given a list of integers. +You need to find the largest prime value and return the sum of its digits. + +Examples: +For lst = [0,3,2,1,3,5,7,4,5,5,5,2,181,32,4,32,3,2,32,324,4,3] the output should be 10 +For lst = [1,0,1,8,2,4597,2,1,3,40,1,2,1,2,4,2,5,1] the output should be 25 +For lst = [1,3,1,32,5107,34,83278,109,163,23,2323,32,30,1,9,3] the output should be 13 +For lst = [0,724,32,71,99,32,6,0,5,91,83,0,5,6] the output should be 11 +For lst = [0,81,12,3,1,21] the output should be 3 +For lst = [0,8,1,2,1,7] the output should be 7 + +Here's a correct unverified solution in python: +```python +def skjkasdkd(lst): + def isPrime(n): + for i in range(2,int(n**0.5)+1): + if n%i==0: + return False + + return True + maxx = 0 + i = 0 + while i < len(lst): + if(lst[i] > maxx and isPrime(lst[i])): + maxx = lst[i] + i+=1 + result = sum(int(digit) for digit in str(maxx)) + return result + +def check(skjkasdkd): + # Check some simple cases + assert skjkasdkd([0,3,2,1,3,5,7,4,5,5,5,2,181,32,4,32,3,2,32,324,4,3]) == 10, "This prints if this assert fails 1 (good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert skjkasdkd([1,0,1,8,2,4597,2,1,3,40,1,2,1,2,4,2,5,1]) == 25, "This prints if this assert fails 2 (also good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert skjkasdkd([1,3,1,32,5107,34,83278,109,163,23,2323,32,30,1,9,3]) == 13, "This prints if this assert fails 3 (also good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert skjkasdkd([0,724,32,71,99,32,6,0,5,91,83,0,5,6]) == 11, "This prints if this assert fails 4 (also good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert skjkasdkd([0,81,12,3,1,21]) == 3, "This prints if this assert fails 5 (also good for debugging!)" + # Check some edge cases that are easy to work out by hand. + assert skjkasdkd([0,8,1,2,1,7]) == 7, "This prints if this assert fails 6 (also good for debugging!)" +check(skjkasdkd) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/smallest_change.prompt b/benches/HumanEvalPromptsWithPythonSol/smallest_change.prompt new file mode 100644 index 0000000..38a5e99 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/smallest_change.prompt @@ -0,0 +1,27 @@ + +Given an array arr of integers, find the minimum number of elements that +need to be changed to make the array palindromic. A palindromic array is an array that +is read the same backwards and forwards. In one change, you can change one element to any other element. + +For example: +smallest_change([1,2,3,5,4,7,9,6]) == 4 +smallest_change([1, 2, 3, 4, 3, 2, 2]) == 1 +smallest_change([1, 2, 3, 2, 1]) == 0 + +Here's a correct unverified solution in python: +```python +def smallest_change(arr): + ans = 0 + for i in range(len(arr) // 2): + if arr[i] != arr[len(arr) - i - 1]: + ans += 1 + return ans +def check(smallest_change): + # Check some simple cases + assert smallest_change([1,2,3,5,4,7,9,6]) == 4 + assert smallest_change([1, 2, 3, 4, 3, 2, 2]) == 1 + # Check some edge cases that are easy to work out by hand. + assert smallest_change([1, 2, 3, 2, 1]) == 0 + assert smallest_change([3, 1, 1, 3]) == 0 +check(smallest_change) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/solution.prompt b/benches/HumanEvalPromptsWithPythonSol/solution.prompt new file mode 100644 index 0000000..19f51a0 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/solution.prompt @@ -0,0 +1,20 @@ +Given a non-empty list of integers, return the sum of all of the odd elements that are in even positions. + + +Examples +solution([5, 8, 7, 1]) ==> 12 +solution([3, 3, 3, 3, 3]) ==> 9 +solution([30, 13, 24, 321]) ==>0 + +Here's a correct unverified solution in python: +```python +def solution(lst): + return sum([x for idx, x in enumerate(lst) if idx%2==0 and x%2==1]) +def check(solution): + # Check some simple cases + assert solution([5, 8, 7, 1]) == 12 + assert solution([3, 3, 3, 3, 3]) == 9 + assert solution([30, 13, 24, 321]) == 0 + # Check some edge cases that are easy to work out by hand. +check(solution) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/solve.prompt b/benches/HumanEvalPromptsWithPythonSol/solve.prompt new file mode 100644 index 0000000..a020e36 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/solve.prompt @@ -0,0 +1,37 @@ +You are given a string s. +if s[i] is a letter, reverse its case from lower to upper or vise versa, +otherwise keep it as it is. +If the string contains no letters, reverse the string. +The function should return the resulted string. +Examples +solve("1234") = "4321" +solve("ab") = "AB" +solve("#a@C") = "#A@c" + +Here's a correct unverified solution in python: +```python +def solve(s): + flg = 0 + idx = 0 + new_str = list(s) + for i in s: + if i.isalpha(): + new_str[idx] = i.swapcase() + flg = 1 + idx += 1 + s = "" + for i in new_str: + s += i + if flg == 0: + return s[len(s)::-1] + return s +def check(solve): + # Check some simple cases + assert solve("1234") == "4321" + assert solve("ab") == "AB" + assert solve("#a@C") == "#A@c" + # Don't remove this line: +check(solve) +def solve(N): + return bin(sum(int(i) for i in str(N)))[2:] +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/sort_array.prompt b/benches/HumanEvalPromptsWithPythonSol/sort_array.prompt new file mode 100644 index 0000000..0797b68 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/sort_array.prompt @@ -0,0 +1,36 @@ + +In this Kata, you have to sort an array of non-negative integers according to +number of ones in their binary representation in ascending order. +For similar number of ones, sort based on decimal value. + +It must be implemented like this: +>>> sort_array([1, 5, 2, 3, 4]) == [1, 2, 3, 4, 5] +>>> sort_array([-2, -3, -4, -5, -6]) == [-6, -5, -4, -3, -2] +>>> sort_array([1, 0, 2, 3, 4]) [0, 1, 2, 3, 4] + +Here's a correct unverified solution in python: +```python +def sort_array(arr): + return sorted(sorted(arr), key=lambda x: bin(x)[2:].count('1')) +def check(sort_array): + # Check some simple cases + assert True, "This prints if this assert fails 1 (good for debugging!)" + assert sort_array([1,5,2,3,4]) == [1, 2, 4, 3, 5] + assert sort_array([-2,-3,-4,-5,-6]) == [-4, -2, -6, -5, -3] + assert sort_array([1,0,2,3,4]) == [0, 1, 2, 4, 3] + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(sort_array) +def sort_array(array): + return [] if len(array) == 0 else sorted(array, reverse= (array[0]+array[-1]) % 2 == 0) +def check(sort_array): + # Check some simple cases + assert True, "This prints if this assert fails 1 (good for debugging!)" + assert sort_array([]) == [], "Error" + assert sort_array([5]) == [5], "Error" + assert sort_array([2, 4, 3, 0, 1, 5]) == [0, 1, 2, 3, 4, 5], "Error" + assert sort_array([2, 4, 3, 0, 1, 5, 6]) == [6, 5, 4, 3, 2, 1, 0], "Error" + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(sort_array) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/sort_even.prompt b/benches/HumanEvalPromptsWithPythonSol/sort_even.prompt new file mode 100644 index 0000000..3180d4f --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/sort_even.prompt @@ -0,0 +1,25 @@ +This function takes a list l and returns a list l' such that +l' is identical to l in the odd indicies, while its values at the even indicies are equal +to the values of the even indicies of l, but sorted. +>>> sort_even([1, 2, 3]) +[1, 2, 3] +>>> sort_even([5, 6, 3, 4]) +[3, 6, 5, 4] + +Here's a correct unverified solution in python: +```python +def sort_even(l: list): + evens = l[::2] + odds = l[1::2] + evens.sort() + ans = [] + for e, o in zip(evens, odds): + ans.extend([e, o]) + if len(evens) > len(odds): + ans.append(evens[-1]) + return ans +def check(sort_even): + assert tuple(sort_even([1, 2, 3])) == tuple([1, 2, 3]) + assert tuple(sort_even([5, 6,3,4])) == tuple([3,6,5,4]) +check(sort_even) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/sort_numbers.prompt b/benches/HumanEvalPromptsWithPythonSol/sort_numbers.prompt new file mode 100644 index 0000000..5dfe39a --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/sort_numbers.prompt @@ -0,0 +1,29 @@ + Input is a space-delimited string of numberals from 'zero' to 'nine'. +Valid choices are 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight' and 'nine'. +Return the string with numbers sorted from smallest to largest +>>> sort_numbers('three one five') +'one three five' + +Here's a correct unverified solution in python: +```python +from typing import List + + +def sort_numbers(numbers: str) -> str: + value_map = { + 'zero': 0, + 'one': 1, + 'two': 2, + 'three': 3, + 'four': 4, + 'five': 5, + 'six': 6, + 'seven': 7, + 'eight': 8, + 'nine': 9 + } + return ' '.join(sorted([x for x in numbers.split(' ') if x], key=lambda x: value_map[x])) +def check(sort_numbers): + assert sort_numbers('three one five') == 'one three five' +check(sort_numbers) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/sort_third.prompt b/benches/HumanEvalPromptsWithPythonSol/sort_third.prompt new file mode 100644 index 0000000..971d86d --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/sort_third.prompt @@ -0,0 +1,19 @@ +This function takes a list l and returns a list l' such that +l' is identical to l in the indicies that are not divisible by three, while its values at the indicies that are divisible by three are equal +to the values of the corresponding indicies of l, but sorted. +>>> sort_third([1, 2, 3]) +[1, 2, 3] +>>> sort_third([5, 6, 3, 4, 8, 9, 2]) +[2, 6, 3, 4, 8, 9, 5] + +Here's a correct unverified solution in python: +```python +def sort_third(l: list): + l = list(l) + l[::3] = sorted(l[::3]) + return l +def check(sort_third): + assert sort_third([1, 2, 3]) == [1, 2, 3] + assert sort_third([5, 6, 3, 4, 8, 9, 2]) == [2, 6, 3, 4, 8, 9, 5] +check(sort_third) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/sorted_list_sum.prompt b/benches/HumanEvalPromptsWithPythonSol/sorted_list_sum.prompt new file mode 100644 index 0000000..e6fcebc --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/sorted_list_sum.prompt @@ -0,0 +1,29 @@ +Write a function that accepts a list of strings as a parameter, +deletes the strings that have odd lengths from it, +and returns the resulted list with a sorted order, +The list is always a list of strings and never an array of numbers, +and it may contain duplicates. +The order of the list should be ascending by length of each word, and you +should return the list sorted by that rule. +If two words have the same length, sort the list alphabetically. +The function should return a list of strings in sorted order. +You may assume that all words will have the same length. +For example: +assert list_sort(["aa", "a", "aaa"]) => ["aa"] +assert list_sort(["ab", "a", "aaa", "cd"]) => ["ab", "cd"] + +Here's a correct unverified solution in python: +```python +def sorted_list_sum(lst): + lst.sort() + new_lst = [] + for i in lst: + if len(i)%2 == 0: + new_lst.append(i) + return sorted(new_lst, key=len) +def check(sorted_list_sum): + # Check some simple cases + assert sorted_list_sum(["aa", "a", "aaa"]) == ["aa"] + assert sorted_list_sum(["ab", "a", "aaa", "cd"]) == ["ab", "cd"] +check(sorted_list_sum) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/specialFilter.prompt b/benches/HumanEvalPromptsWithPythonSol/specialFilter.prompt new file mode 100644 index 0000000..b691efc --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/specialFilter.prompt @@ -0,0 +1,26 @@ +Write a function that takes an array of numbers as input and returns +the number of elements in the array that are greater than 10 and both +first and last digits of a number are odd (1, 3, 5, 7, 9). +For example: +specialFilter([15, -73, 14, -15]) => 1 +specialFilter([33, -2, -3, 45, 21, 109]) => 2 + +Here's a correct unverified solution in python: +```python +def specialFilter(nums): + + count = 0 + for num in nums: + if num > 10: + odd_digits = (1, 3, 5, 7, 9) + number_as_string = str(num) + if int(number_as_string[0]) in odd_digits and int(number_as_string[-1]) in odd_digits: + count += 1 + + return count +def check(specialFilter): + # Check some simple cases + assert specialFilter([15, -73, 14, -15]) == 1 + assert specialFilter([33, -2, -3, 45, 21, 109]) == 2 +check(specialFilter) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/special_factorial.prompt b/benches/HumanEvalPromptsWithPythonSol/special_factorial.prompt new file mode 100644 index 0000000..550d635 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/special_factorial.prompt @@ -0,0 +1,25 @@ +The Brazilian factorial is defined as: +brazilian_factorial(n) = n! * (n-1)! * (n-2)! * ... * 1! +where n > 0 + +For example: +>>> special_factorial(4) +288 + +The function will receive an integer as input and should return the special +factorial of this integer. + +Here's a correct unverified solution in python: +```python +def special_factorial(n): + fact_i = 1 + special_fact = 1 + for i in range(1, n+1): + fact_i *= i + special_fact *= fact_i + return special_fact +def check(special_factorial): + # Check some simple cases + assert special_factorial(4) == 288, "Test 4" +check(special_factorial) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/split_words.prompt b/benches/HumanEvalPromptsWithPythonSol/split_words.prompt new file mode 100644 index 0000000..5e18c42 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/split_words.prompt @@ -0,0 +1,24 @@ + +Given a string of words, return a list of words split on whitespace, if no whitespaces exists in the text you +should split on commas ',' if no commas exists you should return the number of lower-case letters with odd order in the +alphabet, ord('a') = 0, ord('b') = 1, ... ord('z') = 25 +Examples +split_words("Hello world!") ➞ ["Hello", "world!"] +split_words("Hello,world!") ➞ ["Hello", "world!"] +split_words("abcdef") == 3 + +Here's a correct unverified solution in python: +```python +def split_words(txt): + if " " in txt: + return txt.split() + elif "," in txt: + return txt.replace(',',' ').split() + else: + return len([i for i in txt if i.islower() and ord(i)%2 == 0]) +def check(split_words): + assert split_words("Hello world!") == ["Hello","world!"] + assert split_words("Hello,world!") == ["Hello","world!"] + assert split_words("abcdef") == 3 +check(split_words) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/starts_one_ends.prompt b/benches/HumanEvalPromptsWithPythonSol/starts_one_ends.prompt new file mode 100644 index 0000000..b9c714b --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/starts_one_ends.prompt @@ -0,0 +1,10 @@ + +Given a positive integer n, return the count of the numbers of n-digit +positive integers that start or end with 1. + +Here's a correct unverified solution in python: +```python +def starts_one_ends(n): + if n == 1: return 1 + return 18 * (10 ** (n - 2)) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/strange_sort_list.prompt b/benches/HumanEvalPromptsWithPythonSol/strange_sort_list.prompt new file mode 100644 index 0000000..a20c2c0 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/strange_sort_list.prompt @@ -0,0 +1,28 @@ + +Given list of integers, return list in strange order. +Strange sorting, is when you start with the minimum value, +then maximum of the remaining integers, then minimum and so on. + +Examples: +strange_sort_list([1, 2, 3, 4]) == [1, 4, 2, 3] +strange_sort_list([5, 5, 5, 5]) == [5, 5, 5, 5] +strange_sort_list([]) == [] + +Here's a correct unverified solution in python: +```python +def strange_sort_list(lst): + res, switch = [], True + while lst: + res.append(min(lst) if switch else max(lst)) + lst.remove(res[-1]) + switch = not switch + return res +def check(strange_sort_list): + # Check some simple cases + assert strange_sort_list([1, 2, 3, 4]) == [1, 4, 2, 3] + assert strange_sort_list([5, 5, 5, 5]) == [5, 5, 5, 5] + assert strange_sort_list([]) == [] + # Check some edge cases that are easy to work out by hand. + assert True +check(strange_sort_list) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/string_sequence.prompt b/benches/HumanEvalPromptsWithPythonSol/string_sequence.prompt new file mode 100644 index 0000000..b33b139 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/string_sequence.prompt @@ -0,0 +1,15 @@ + Return a string containing space-delimited numbers starting from 0 upto n inclusive. +>>> string_sequence(0) +'0' +>>> string_sequence(5) +'0 1 2 3 4 5' + +Here's a correct unverified solution in python: +```python +def string_sequence(n: int) -> str: + return ' '.join([str(x) for x in range(n + 1)]) +def check(string_sequence): + assert string_sequence(0) == '0' + assert string_sequence(5) == '0 1 2 3 4 5' +check(string_sequence) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/string_to_md5.prompt b/benches/HumanEvalPromptsWithPythonSol/string_to_md5.prompt new file mode 100644 index 0000000..8dd29b0 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/string_to_md5.prompt @@ -0,0 +1,18 @@ + +Given a string 'text', return its md5 hash equivalent string. +If 'text' is an empty string, return None. + +>>> string_to_md5('Hello world') == '3e25960a79dbc69b674cd4ec67a72c62' + +Here's a correct unverified solution in python: +```python +def string_to_md5(text): + import hashlib + return hashlib.md5(text.encode('ascii')).hexdigest() if text else None +def check(string_to_md5): + # Check some simple cases + assert string_to_md5('Hello world') == '3e25960a79dbc69b674cd4ec67a72c62' + # Check some edge cases that are easy to work out by hand. + assert True +check(string_to_md5) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/string_xor.prompt b/benches/HumanEvalPromptsWithPythonSol/string_xor.prompt new file mode 100644 index 0000000..cbe4800 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/string_xor.prompt @@ -0,0 +1,22 @@ + Input are two strings a and b consisting only of 1s and 0s. +Perform binary XOR on these inputs and return result also as a string. +>>> string_xor('010', '110') +'100' + +Here's a correct unverified solution in python: +```python +from typing import List + + +def string_xor(a: str, b: str) -> str: + def xor(i, j): + if i == j: + return '0' + else: + return '1' + + return ''.join(xor(x, y) for x, y in zip(a, b)) +def check(string_xor): + assert string_xor('010', '110') == '100' +check(string_xor) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/strlen.prompt b/benches/HumanEvalPromptsWithPythonSol/strlen.prompt new file mode 100644 index 0000000..2dffc12 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/strlen.prompt @@ -0,0 +1,15 @@ + Return length of given string +>>> strlen('') +0 +>>> strlen('abc') +3 + +Here's a correct unverified solution in python: +```python +def strlen(string: str) -> int: + return len(string) +def check(strlen): + assert strlen('') == 0 + assert strlen('abc') == 3 +check(strlen) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/sum_product.prompt b/benches/HumanEvalPromptsWithPythonSol/sum_product.prompt new file mode 100644 index 0000000..c91208e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/sum_product.prompt @@ -0,0 +1,26 @@ + For a given list of integers, return a tuple consisting of a sum and a product of all the integers in a list. +Empty sum should be equal to 0 and empty product should be equal to 1. +>>> sum_product([]) +(0, 1) +>>> sum_product([1, 2, 3, 4]) +(10, 24) + +Here's a correct unverified solution in python: +```python +from typing import List, Tuple + + +def sum_product(numbers: List[int]) -> Tuple[int, int]: + sum_value = 0 + prod_value = 1 + + for n in numbers: + sum_value += n + prod_value *= n + return sum_value, prod_value + +def check(sum_product): + assert sum_product([]) == (0, 1) + assert sum_product([1, 2,3,4]) == (10, 24) +check(sum_product) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/sum_squares.prompt b/benches/HumanEvalPromptsWithPythonSol/sum_squares.prompt new file mode 100644 index 0000000..6a9fb34 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/sum_squares.prompt @@ -0,0 +1,45 @@ +" +This function will take a list of integers. For all entries in the list, the function shall square the integer entry if its index is a +multiple of 3 and will cube the integer entry if its index is a multiple of 4 and not a multiple of 3. The function will not +change the entries in the list whose indexes are not a multiple of 3 or 4. The function shall then return the sum of all entries. + +Examples: +For lst = [1,2,3] the output should be 6 +For lst = [] the output should be 0 +For lst = [-1,-5,2,-1,-5] the output should be -126 + +Here's a correct unverified solution in python: +```python +def sum_squares(lst): + import math + squared = 0 + for i in lst: + squared += math.ceil(i)**2 + return squared +def check(sum_squares): + # Check some simple cases + assert sum_squares([1,2,3])==14, "This prints if this assert fails 1 (good for debugging!)" + assert sum_squares([1,4,9])==98, "This prints if this assert fails 1 (good for debugging!)" + assert sum_squares([1,3,5,7])==84, "This prints if this assert fails 1 (good for debugging!)" + assert sum_squares([1.4,4.2,0])==29, "This prints if this assert fails 1 (good for debugging!)" + assert sum_squares([-2.4,1,1])==6, "This prints if this assert fails 1 (good for debugging!)" +check(sum_squares) +def sum_squares(lst): + " + result =[] + for i in range(len(lst)): + if i %3 == 0: + result.append(lst[i]**2) + elif i % 4 == 0 and i%3 != 0: + result.append(lst[i]**3) + else: + result.append(lst[i]) + return sum(result) +def check(sum_squares): + # Check some simple cases + assert sum_squares([1,2,3]) == 6 + assert sum_squares([]) == 0 + assert sum_squares([-1,-5,2,-1,-5]) == -126 + # Don't remove this line: +check(sum_squares) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/sum_to_n.prompt b/benches/HumanEvalPromptsWithPythonSol/sum_to_n.prompt new file mode 100644 index 0000000..07ac4f4 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/sum_to_n.prompt @@ -0,0 +1,24 @@ +sum_to_n is a function that sums numbers from 1 to n. +>>> sum_to_n(30) +465 +>>> sum_to_n(100) +5050 +>>> sum_to_n(5) +15 +>>> sum_to_n(10) +55 +>>> sum_to_n(1) +1 + +Here's a correct unverified solution in python: +```python +def sum_to_n(n: int): + return sum(range(n + 1)) +def check(sum_to_n): + assert sum_to_n(1) == 1 + assert sum_to_n(5) == 15 + assert sum_to_n(10) == 55 + assert sum_to_n(30) == 465 + assert sum_to_n(100) == 5050 +check(sum_to_n) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/total_match.prompt b/benches/HumanEvalPromptsWithPythonSol/total_match.prompt new file mode 100644 index 0000000..8477221 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/total_match.prompt @@ -0,0 +1,40 @@ + +Write a function that accepts two lists of strings and returns the list that has +total number of chars in the all strings of the list less than the other list. + +if the two lists have the same number of chars, return the first list. + +Examples +total_match([], []) ➞ [] +total_match(['hi', 'admin'], ['hI', 'Hi']) ➞ ['hI', 'Hi'] +total_match(['hi', 'admin'], ['hi', 'hi', 'admin', 'project']) ➞ ['hi', 'admin'] +total_match(['hi', 'admin'], ['hI', 'hi', 'hi']) ➞ ['hI', 'hi', 'hi'] +total_match(['4'], ['1', '2', '3', '4', '5']) ➞ ['4'] + +Here's a correct unverified solution in python: +```python +def total_match(lst1, lst2): + l1 = 0 + for st in lst1: + l1 += len(st) + + l2 = 0 + for st in lst2: + l2 += len(st) + + if l1 <= l2: + return lst1 + else: + return lst2 +def check(total_match): + # Check some simple cases + assert True, "This prints if this assert fails 1 (good for debugging!)" + assert total_match([], []) == [] + assert total_match(['hi', 'admin'], ['hi', 'hi', 'admin', 'project']) == ['hi', 'admin'] + assert total_match(['4'], ['1', '2', '3', '4', '5']) == ['4'] + assert total_match(['hi', 'admin'], ['hI', 'Hi']) == ['hI', 'Hi'] + assert total_match(['hi', 'admin'], ['hI', 'hi', 'hi']) == ['hI', 'hi', 'hi'] + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(total_match) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/tri.prompt b/benches/HumanEvalPromptsWithPythonSol/tri.prompt new file mode 100644 index 0000000..7d95d60 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/tri.prompt @@ -0,0 +1,33 @@ +Everyone knows Fibonacci sequence, it was studied deeply by mathematicians in +the last couple centuries. However, what people don't know is Tribonacci sequence. +Tribonacci sequence is defined by the recurrence: +tri(1) = 3 +tri(n) = 1 + n / 2, if n is even. +tri(n) = tri(n - 1) + tri(n - 2) + tri(n + 1), if n is odd. +For example: +tri(2) = 1 + (2 / 2) = 2 +tri(4) = 3 +tri(3) = tri(2) + tri(1) + tri(4) + = 2 + 3 + 3 = 8 +You are given a non-negative integer number n, you have to a return a list of the +first n + 1 numbers of the Tribonacci sequence. +Examples: +tri(3) = [1, 3, 2, 8] + +Here's a correct unverified solution in python: +```python +def tri(n): + if n == 0: + return [1] + my_tri = [1, 3] + for i in range(2, n + 1): + if i % 2 == 0: + my_tri.append(i / 2 + 1) + else: + my_tri.append(my_tri[i - 1] + my_tri[i - 2] + (i + 3) / 2) + return my_tri +def check(tri): + # Check some simple cases + assert tri(3) == [1, 3, 2.0, 8.0] +check(tri) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/triangle_area.prompt b/benches/HumanEvalPromptsWithPythonSol/triangle_area.prompt new file mode 100644 index 0000000..433e0ce --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/triangle_area.prompt @@ -0,0 +1,30 @@ + +Given the lengths of the three sides of a triangle. Return the area of +the triangle rounded to 2 decimal points if the three sides form a valid triangle. +Otherwise return -1 +Three sides make a valid triangle when the sum of any two sides is greater +than the third side. +Example: +triangle_area(3, 4, 5) == 6.00 +triangle_area(1, 2, 10) == -1 + +Here's a correct unverified solution in python: +```python +def triangle_area(a, h): + return a * h / 2.0 +def check(triangle_area): + assert triangle_area(5, 3) == 7.5 +check(triangle_area) +def triangle_area(a, b, c): + if a + b <= c or a + c <= b or b + c <= a: + return -1 + s = (a + b + c)/2 + area = (s * (s - a) * (s - b) * (s - c)) ** 0.5 + area = round(area, 2) + return area +def check(triangle_area): + # Check some simple cases + assert triangle_area(3, 4, 5) == 6.00, "This prints if this assert fails 1 (good for debugging!)" + assert triangle_area(1, 2, 10) == -1 +check(triangle_area) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/triples_sum_to_zero.prompt b/benches/HumanEvalPromptsWithPythonSol/triples_sum_to_zero.prompt new file mode 100644 index 0000000..dacbbf1 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/triples_sum_to_zero.prompt @@ -0,0 +1,32 @@ + +triples_sum_to_zero takes a list of integers as an input. +it returns True if there are three distinct elements in the list that +sum to zero, and False otherwise. + +>>> triples_sum_to_zero([1, 3, 5, 0]) +False +>>> triples_sum_to_zero([1, 3, -2, 1]) +True +>>> triples_sum_to_zero([1, 2, 3, 7]) +False +>>> triples_sum_to_zero([2, 4, -5, 3, 9, 7]) +True +>>> triples_sum_to_zero([1]) +False + +Here's a correct unverified solution in python: +```python +def triples_sum_to_zero(l: list): + for i in range(len(l)): + for j in range(i + 1, len(l)): + for k in range(j + 1, len(l)): + if l[i] + l[j] + l[k] == 0: + return True + return False +def check(triples_sum_to_zero): + assert triples_sum_to_zero([1, 3, 5, 0]) == False + assert triples_sum_to_zero([1, 3, -2, 1]) == True + assert triples_sum_to_zero([1, 2, 3, 7]) == False + assert triples_sum_to_zero([2, 4, -5, 3, 9, 7]) == True +check(triples_sum_to_zero) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/unique.prompt b/benches/HumanEvalPromptsWithPythonSol/unique.prompt new file mode 100644 index 0000000..398f86c --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/unique.prompt @@ -0,0 +1,12 @@ +Return sorted unique elements in a list +>>> unique([5, 3, 5, 2, 3, 3, 9, 0, 123]) +[0, 2, 3, 5, 9, 123] + +Here's a correct unverified solution in python: +```python +def unique(l: list): + return sorted(list(set(l))) +def check(unique): + assert unique([5, 3, 5, 2, 3, 3, 9, 0, 123]) == [0, 2, 3, 5, 9, 123] +check(unique) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/unique_digits.prompt b/benches/HumanEvalPromptsWithPythonSol/unique_digits.prompt new file mode 100644 index 0000000..37e0b9d --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/unique_digits.prompt @@ -0,0 +1,29 @@ +Given a list of positive integers x. return a sorted list of all +elements that hasn't any even digit. + +Note: Returned list should be sorted in increasing order. + +For example: +>>> unique_digits([15, 33, 1422, 1]) +[1, 15, 33] +>>> unique_digits([152, 323, 1422, 10]) +[] + +Here's a correct unverified solution in python: +```python +def unique_digits(x): + odd_digit_elements = [] + for i in x: + if all (int(c) % 2 == 1 for c in str(i)): + odd_digit_elements.append(i) + return sorted(odd_digit_elements) +def check(unique_digits): + # Check some simple cases + assert unique_digits([15, 33, 1422, 1]) == [1, 15, 33] + assert unique_digits([152, 323, 1422, 10]) == [] + assert unique_digits([12345, 2033, 111, 151]) == [111, 151] + assert unique_digits([135, 103, 31]) == [31, 135] + # Check some edge cases that are easy to work out by hand. + assert True +check(unique_digits) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/valid_date.prompt b/benches/HumanEvalPromptsWithPythonSol/valid_date.prompt new file mode 100644 index 0000000..2d2c326 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/valid_date.prompt @@ -0,0 +1,47 @@ +You have to write a function which validates a given date string and +returns True if the date is valid otherwise False. +The date is valid if all of the following rules are satisfied: +1. The date string is not empty. +2. The number of days is not less than 1 or higher than 31 days for months 1,3,5,7,8,10,12. And the number of days is not less than 1 or higher than 30 days for months 4,6,9,11. And, the number of days is not less than 1 or higher than 29 for the month 2. +3. The months should not be less than 1 or higher than 12. +4. The date should be in the format: mm-dd-yyyy + +for example: +valid_date('03-11-2000') => True + +valid_date('15-01-2012') => False + +valid_date('04-0-2040') => False + +valid_date('06-04-2020') => True + +valid_date('06/04/2020') => False + +Here's a correct unverified solution in python: +```python +def valid_date(date): + try: + date = date.strip() + month, day, year = date.split('-') + month, day, year = int(month), int(day), int(year) + if month < 1 or month > 12: + return False + if month in [1,3,5,7,8,10,12] and day < 1 or day > 31: + return False + if month in [4,6,9,11] and day < 1 or day > 30: + return False + if month == 2 and day < 1 or day > 29: + return False + except: + return False + + return True +def check(valid_date): + # Check some simple cases + assert valid_date('03-11-2000') == True + assert valid_date('15-01-2012') == False + assert valid_date('04-0-2040') == False + assert valid_date('06-04-2020') == True + assert valid_date('06/04/2020') == False +check(valid_date) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/vowels_count.prompt b/benches/HumanEvalPromptsWithPythonSol/vowels_count.prompt new file mode 100644 index 0000000..8c6c78e --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/vowels_count.prompt @@ -0,0 +1,23 @@ + +Add more test cases. + +Here's a correct unverified solution in python: +```python +FIX = """ +Add more test cases. +""" + +def vowels_count(s): + vowels = "aeiouAEIOU" + n_vowels = sum(c in vowels for c in s) + if s[-1] == 'y' or s[-1] == 'Y': + n_vowels += 1 + return n_vowels +def check(vowels_count): + # Check some simple cases + assert vowels_count("abcde") == 2, "Test 6" + assert vowels_count("ACEDY") == 3, "Test 7" + # Check some edge cases that are easy to work out by hand. + assert True, "This prints if this assert fails 2 (also good for debugging!)" +check(vowels_count) +``` diff --git a/benches/HumanEvalPromptsWithPythonSol/will_it_fly.prompt b/benches/HumanEvalPromptsWithPythonSol/will_it_fly.prompt new file mode 100644 index 0000000..76fefd8 --- /dev/null +++ b/benches/HumanEvalPromptsWithPythonSol/will_it_fly.prompt @@ -0,0 +1,38 @@ + +Write a function that returns True if the object q will fly, and False otherwise. +The object q will fly if it's balanced (it is a palindromic list) and the sum of its elements is less than or equal the maximum possible weight w. + +Example: +will_it_fly([1, 2], 5) ➞ False +# 1+2 is less than the maximum possible weight, but it's unbalanced. + +will_it_fly([3, 2, 3], 1) ➞ False +# it's balanced, but 3+2+3 is more than the maximum possible weight. + +will_it_fly([3, 2, 3], 9) ➞ True +# 3+2+3 is less than the maximum possible weight, and it's balanced. + +will_it_fly([3], 5) ➞ True +# 3 is less than the maximum possible weight, and it's balanced. + +Here's a correct unverified solution in python: +```python +def will_it_fly(q,w): + if sum(q) > w: + return False + + i, j = 0, len(q)-1 + while i Date: Thu, 22 Aug 2024 02:11:37 +0200 Subject: [PATCH 08/15] Update dafnybench prompts for retrying and claude-3.5 --- .gitignore | 2 +- prompts/dafnybench/ask_for_fixed.txt | 3 +++ prompts/dafnybench/sys.txt | 1 + verified_cogen/experiments/incremental_run.py | 0 4 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 prompts/dafnybench/ask_for_fixed.txt create mode 100644 verified_cogen/experiments/incremental_run.py diff --git a/.gitignore b/.gitignore index a6bc561..4671105 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ llm-generated .envrc **/.DS_Store pyrightconfig.json -**/DafnyBench/ +benches/DafnyBench/ break_assert.rs .venv **/target diff --git a/prompts/dafnybench/ask_for_fixed.txt b/prompts/dafnybench/ask_for_fixed.txt new file mode 100644 index 0000000..d1c1e23 --- /dev/null +++ b/prompts/dafnybench/ask_for_fixed.txt @@ -0,0 +1,3 @@ +This answer got Dafny verification error: +{error} +Please try again by taking the Dafny feedback. diff --git a/prompts/dafnybench/sys.txt b/prompts/dafnybench/sys.txt index d48cdcc..43043a1 100644 --- a/prompts/dafnybench/sys.txt +++ b/prompts/dafnybench/sys.txt @@ -1,2 +1,3 @@ You are an expert in Dafny. You will be given tasks dealing with Dafny programs including precise annotations. +You respond only with code blocks. \ No newline at end of file diff --git a/verified_cogen/experiments/incremental_run.py b/verified_cogen/experiments/incremental_run.py new file mode 100644 index 0000000..e69de29 From d4c7f9a2c63eb05c4764cf6e4afa11df16c7d699 Mon Sep 17 00:00:00 2001 From: gt22 Date: Thu, 22 Aug 2024 02:15:22 +0200 Subject: [PATCH 09/15] Allow no-loop programs for verification --- verified_cogen/runners/invariants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/verified_cogen/runners/invariants.py b/verified_cogen/runners/invariants.py index 784072e..ef7332d 100644 --- a/verified_cogen/runners/invariants.py +++ b/verified_cogen/runners/invariants.py @@ -57,10 +57,10 @@ def insert(cls, llm: LLM, prg: str, checks: str, mode: Mode) -> str: @classmethod def precheck(cls, prg: str, mode: Mode): - while_count = prg.count("while") - if while_count == 0: - raise ValueError("No loops in program") if mode == Mode.REGEX: + while_count = prg.count("while") + if while_count == 0: + raise ValueError("No loops in program") if while_count > 1: raise ValueError( "Multiple loops in program, not supported in regex mode" From 90e2b27dde578bd6f3f02ee02adfe1181db6d6ea Mon Sep 17 00:00:00 2001 From: gt22 Date: Thu, 22 Aug 2024 02:15:32 +0200 Subject: [PATCH 10/15] Incremental run for benchmarks --- verified_cogen/experiments/incremental_run.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/verified_cogen/experiments/incremental_run.py b/verified_cogen/experiments/incremental_run.py index e69de29..d20621e 100644 --- a/verified_cogen/experiments/incremental_run.py +++ b/verified_cogen/experiments/incremental_run.py @@ -0,0 +1,52 @@ +import pathlib +import logging +from verified_cogen.main import get_args, rename_file +from verified_cogen.tools.modes import Mode +from verified_cogen.tools.verifier import Verifier +from verified_cogen.llm.llm import LLM +from verified_cogen.runners.invariants import InvariantRunner + +logger = logging.getLogger(__name__) + + +def main(): + args = get_args() + mode = Mode(args.insert_conditions_mode) + assert mode != Mode.REGEX + assert args.dir is not None + assert args.bench_type == "invariants" + assert args.runs == 1 + assert args.retries == 0 + + directory = pathlib.Path(args.dir) + marker_directory = directory / f"../tries_{directory.name}" + marker_directory.mkdir(exist_ok=True, parents=True) + files = list(directory.glob("[!.]*")) + + verifier = Verifier(args.shell, args.verifier_command) + + for file in files: + llm = LLM( + args.grazie_token, + args.llm_profile, + args.prompts_directory, + args.temperature, + ) + display_name = rename_file(file) + marker_file = marker_directory.joinpath(file.relative_to(directory)) + if marker_file.exists(): + print("Skipping:", display_name) + continue + print("Processing:", display_name) + try: + tries = InvariantRunner.run_on_file( + logger, verifier, mode, llm, args.tries, str(file) + ) + except: + tries = None + with marker_file.open("w") as f: + f.write(str(tries)) + + +if __name__ == "__main__": + main() From 797a0b8bc13d46c5a013611814d72a3ff5a5fc6f Mon Sep 17 00:00:00 2001 From: gt22 Date: Thu, 22 Aug 2024 03:05:02 +0200 Subject: [PATCH 11/15] Add validating runner --- gui/src/main.rs | 9 +++- verified_cogen/args.py | 2 +- verified_cogen/main.py | 2 + verified_cogen/runners/validating.py | 70 ++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 verified_cogen/runners/validating.py diff --git a/gui/src/main.rs b/gui/src/main.rs index 317fe9b..70a480f 100644 --- a/gui/src/main.rs +++ b/gui/src/main.rs @@ -73,12 +73,13 @@ enum BenchMode { Invariants, Generic, Generate, + Validating, } impl BenchMode { fn llm_generated_path(&self, path: &str) -> PathBuf { let name = match self { - BenchMode::Invariants | BenchMode::Generic => basename(path).to_string(), + BenchMode::Invariants | BenchMode::Generic | BenchMode::Validating => basename(path).to_string(), BenchMode::Generate => { let base = basename(path); base.chars() @@ -97,6 +98,7 @@ impl Display for BenchMode { BenchMode::Invariants => write!(f, "invariants"), BenchMode::Generic => write!(f, "generic"), BenchMode::Generate => write!(f, "generate"), + BenchMode::Validating => write!(f, "validating") } } } @@ -486,6 +488,11 @@ impl AppState { BenchMode::Generate, "Generate", ); + ui.radio_value( + &mut self.settings.bench_type, + BenchMode::Validating, + "Validating" + ); }); ui.horizontal(|ui| { let max_rect = ui.max_rect(); diff --git a/verified_cogen/args.py b/verified_cogen/args.py index 77cf220..2278440 100644 --- a/verified_cogen/args.py +++ b/verified_cogen/args.py @@ -17,7 +17,7 @@ def get_args(): ) parser.add_argument( "--bench-type", - help="benchmark type, available: {invariants, generic, generate}", + help="benchmark type, available: {invariants, generic, generate, validating}", default="invariants", ) parser.add_argument("--temperature", help="model temperature", default=0, type=int) diff --git a/verified_cogen/main.py b/verified_cogen/main.py index abfc367..d7ffc00 100644 --- a/verified_cogen/main.py +++ b/verified_cogen/main.py @@ -5,6 +5,7 @@ from verified_cogen.runners.generate import GenerateRunner from verified_cogen.runners.generic import GenericRunner from verified_cogen.runners.invariants import InvariantRunner +from verified_cogen.runners.validating import ValidatingRunner from verified_cogen.tools import pprint_stat, rename_file, tabulate_list from verified_cogen.tools.modes import Mode from verified_cogen.tools.verifier import Verifier @@ -81,6 +82,7 @@ def main(): "invariants": InvariantRunner, "generic": GenericRunner, "generate": GenerateRunner, + "validating": ValidatingRunner, }[args.bench_type] verifier = Verifier(args.shell, args.verifier_command, args.verifier_timeout) diff --git a/verified_cogen/runners/validating.py b/verified_cogen/runners/validating.py new file mode 100644 index 0000000..0bf408e --- /dev/null +++ b/verified_cogen/runners/validating.py @@ -0,0 +1,70 @@ +import re + +from verified_cogen.tools.modes import Mode + +from verified_cogen.llm import LLM +from verified_cogen.runners import Runner +from verified_cogen.runners.invariants import InvariantRunner + +# Regular expression to match Dafny method definitions +method_pattern = re.compile(r'method\s+(\w+)\s*\((.*?)\)\s*returns\s*\((.*?)\)(.*?)\{', re.DOTALL) + +def generate_validators(dafny_code): + """ + Create validator-methods for every method in dafny program + + Example: + Input: + method xor(a : char, b : char) returns (result : char) + ensures result == (if a == b then '0' else '1') + { if (a == b) { result := '0'; } else { result := '1'; } } + Output: + method xor_valid(a : char, b : char) returns (result : char) + ensures result == (if a == b then '0' else '1') + { var ret := xor(a, b); return ret; } + """ + methods = method_pattern.finditer(dafny_code) + + validators = [] + + for match in methods: + method_name = match.group(1) + parameters = match.group(2) + returns = match.group(3) + specs = match.group(4) + + validator = f"method {method_name}_valid({parameters}) returns ({returns}){specs}" + validator += "{ var ret := " + + validator += f"{method_name}({', '.join(param.split(':')[0].strip() for param in parameters.split(',') if param.strip())});" + + validator += " return ret; }\n" + + validators.append(validator) + + return '\n'.join(validators) + + +class ValidatingRunner(Runner): + + @classmethod + def _add_validators(cls, prg: str, inv_prg: str): + validators = generate_validators(prg) + val_prg = inv_prg + "\n// ==== verifiers ==== //\n" + validators + return val_prg + + @classmethod + def rewrite(cls, llm: LLM, prg: str) -> str: + return ValidatingRunner._add_validators(prg, InvariantRunner.rewrite(llm, prg)) + + @classmethod + def produce(cls, llm: LLM, prg: str) -> str: + return ValidatingRunner._add_validators(prg, InvariantRunner.produce(llm, prg)) + + @classmethod + def insert(cls, llm: LLM, prg: str, checks: str, mode: Mode) -> str: + return ValidatingRunner._add_validators(prg, InvariantRunner.insert(llm, prg, mode)) + + @classmethod + def precheck(cls, prg: str, mode: Mode): + return InvariantRunner.precheck(prg, mode) \ No newline at end of file From 39766b0e3932c68f5a3e0975ba670a5cf6a6b0d3 Mon Sep 17 00:00:00 2001 From: gt22 Date: Thu, 22 Aug 2024 12:13:01 +0200 Subject: [PATCH 12/15] Fix validating.insert --- verified_cogen/runners/validating.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verified_cogen/runners/validating.py b/verified_cogen/runners/validating.py index 0bf408e..b8601f0 100644 --- a/verified_cogen/runners/validating.py +++ b/verified_cogen/runners/validating.py @@ -63,7 +63,7 @@ def produce(cls, llm: LLM, prg: str) -> str: @classmethod def insert(cls, llm: LLM, prg: str, checks: str, mode: Mode) -> str: - return ValidatingRunner._add_validators(prg, InvariantRunner.insert(llm, prg, mode)) + return ValidatingRunner._add_validators(prg, InvariantRunner.insert(llm, prg, checks, mode)) @classmethod def precheck(cls, prg: str, mode: Mode): From a88d9e48bf7eed1930fadef78fd90aac8bfcbb2f Mon Sep 17 00:00:00 2001 From: gt22 Date: Tue, 3 Sep 2024 01:53:19 +0200 Subject: [PATCH 13/15] Integrate invariant stripping from marks --- verified_cogen/experiments/incremental_run.py | 14 +++++---- verified_cogen/runners/__init__.py | 29 ++++++++++++------- verified_cogen/runners/validating.py | 21 ++++++++++++-- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/verified_cogen/experiments/incremental_run.py b/verified_cogen/experiments/incremental_run.py index d20621e..35d7654 100644 --- a/verified_cogen/experiments/incremental_run.py +++ b/verified_cogen/experiments/incremental_run.py @@ -4,7 +4,7 @@ from verified_cogen.tools.modes import Mode from verified_cogen.tools.verifier import Verifier from verified_cogen.llm.llm import LLM -from verified_cogen.runners.invariants import InvariantRunner +from verified_cogen.runners.validating import ValidatingRunner logger = logging.getLogger(__name__) @@ -19,10 +19,9 @@ def main(): assert args.retries == 0 directory = pathlib.Path(args.dir) - marker_directory = directory / f"../tries_{directory.name}" + marker_directory = pathlib.Path(f"results/tries_{directory.name}") marker_directory.mkdir(exist_ok=True, parents=True) - files = list(directory.glob("[!.]*")) - + files = list(directory.glob("[!.]*.dfy")) verifier = Verifier(args.shell, args.verifier_command) for file in files: @@ -39,10 +38,13 @@ def main(): continue print("Processing:", display_name) try: - tries = InvariantRunner.run_on_file( + tries = ValidatingRunner.run_on_file( logger, verifier, mode, llm, args.tries, str(file) ) - except: + except KeyboardInterrupt: + return + except Exception as e: + print(e) tries = None with marker_file.open("w") as f: f.write(str(tries)) diff --git a/verified_cogen/runners/__init__.py b/verified_cogen/runners/__init__.py index 75cbe67..3e7aef7 100644 --- a/verified_cogen/runners/__init__.py +++ b/verified_cogen/runners/__init__.py @@ -31,6 +31,10 @@ def insert(cls, llm: LLM, prg: str, checks: str, mode: Mode) -> str: def precheck(cls, prg: str, mode: Mode): pass + @classmethod + def preprocess(cls, prg: str, mode: Mode) -> str: + return prg + @classmethod def invoke(cls, logger: Logger, llm: LLM, prg: str, mode: Mode) -> str: logger.info("Invoking LLM") @@ -41,6 +45,14 @@ def invoke(cls, logger: Logger, llm: LLM, prg: str, mode: Mode) -> str: logger.info("Invocation done") return inv_prg + @classmethod + def verify_program(cls, verifier: Verifier, name: str, prg: str): + LLM_GENERATED_DIR.mkdir(parents=True, exist_ok=True) + output = LLM_GENERATED_DIR / name + with open(output, "w") as f: + f.write(prg) + return verifier.verify(output) + @classmethod def try_fixing( cls, @@ -53,11 +65,7 @@ def try_fixing( ) -> Optional[int]: tries = total_tries while tries > 0: - LLM_GENERATED_DIR.mkdir(parents=True, exist_ok=True) - output = LLM_GENERATED_DIR / name - with open(output, "w") as f: - f.write(inv_prg) - verification_result = verifier.verify(output) + verification_result = cls.verify_program(verifier, name, inv_prg) if verification_result is None: logger.info("Verification timed out") tries -= 1 @@ -87,18 +95,19 @@ def run_on_file( total_tries: int, file: str, ) -> Optional[int]: + name = basename(file) logger.info(f"Running on {file}") - verification_result = verifier.verify(pathlib.Path(file)) + with open(file, "r") as f: + prg = cls.preprocess(f.read(), mode) + + verification_result = cls.verify_program(verifier, name, prg) if verification_result is not None and verification_result[0]: return 0 elif verification_result is None: logger.info("Verification timed out") - - with open(file, "r") as f: - prg = f.read() cls.precheck(prg, mode) inv_prg = cls.invoke(logger, llm, prg, mode) return cls.try_fixing( - logger, verifier, llm, total_tries, inv_prg, basename(file) + logger, verifier, llm, total_tries, inv_prg, name ) diff --git a/verified_cogen/runners/validating.py b/verified_cogen/runners/validating.py index b8601f0..79c49cc 100644 --- a/verified_cogen/runners/validating.py +++ b/verified_cogen/runners/validating.py @@ -9,7 +9,8 @@ # Regular expression to match Dafny method definitions method_pattern = re.compile(r'method\s+(\w+)\s*\((.*?)\)\s*returns\s*\((.*?)\)(.*?)\{', re.DOTALL) -def generate_validators(dafny_code): + +def generate_validators(dafny_code: str) -> str: """ Create validator-methods for every method in dafny program @@ -45,6 +46,18 @@ def generate_validators(dafny_code): return '\n'.join(validators) +def remove_asserts_and_invariants(dafny_code: str) -> str: + patterns = [ + r'// assert-line.*\n', + r'// assert-start.*?// assert-end\n', + r'// invariants-start.*?// invariants-end\n' + ] + combined_pattern = '|'.join(patterns) + cleaned_code = re.sub(combined_pattern, '', dafny_code, flags=re.DOTALL) + cleaned_code = re.sub(r'\n\s*\n', '\n', cleaned_code) + return cleaned_code.strip() + + class ValidatingRunner(Runner): @classmethod @@ -53,6 +66,10 @@ def _add_validators(cls, prg: str, inv_prg: str): val_prg = inv_prg + "\n// ==== verifiers ==== //\n" + validators return val_prg + @classmethod + def preprocess(cls, prg: str, mode: Mode) -> str: + return remove_asserts_and_invariants(prg) + @classmethod def rewrite(cls, llm: LLM, prg: str) -> str: return ValidatingRunner._add_validators(prg, InvariantRunner.rewrite(llm, prg)) @@ -67,4 +84,4 @@ def insert(cls, llm: LLM, prg: str, checks: str, mode: Mode) -> str: @classmethod def precheck(cls, prg: str, mode: Mode): - return InvariantRunner.precheck(prg, mode) \ No newline at end of file + return InvariantRunner.precheck(prg, mode) From 189c4d71e9b91dc30b03c1ed537ff327ff7cf6d6 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Wed, 4 Sep 2024 11:42:36 +0300 Subject: [PATCH 14/15] Add tests that run on github actions --- .github/workflows/test-pytest.yml | 38 ++++++++++ tests/test_remove.py | 107 +++++++++++++++++++++++++++ verified_cogen/runners/validating.py | 30 +++++--- 3 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/test-pytest.yml create mode 100644 tests/test_remove.py diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml new file mode 100644 index 0000000..fa349e2 --- /dev/null +++ b/.github/workflows/test-pytest.yml @@ -0,0 +1,38 @@ +name: Run pytest + +on: [push] + +env: + PYTHON_VERSION: "3.11" + POETRY_VERSION: "1.4.2" + POETRY_URL: https://install.python-poetry.org + +jobs: + run-pytest: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + id: setup_python + - name: Cache Poetry cache + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry + key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }} + - name: Cache Packages + uses: actions/cache@v3 + with: + path: ~/.local + key: poetry-local-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }} + - name: Install Poetry ${{ env.POETRY_VERSION }} + run: | + curl -sSL ${{ env.POETRY_URL }} | python - --version ${{ env.POETRY_VERSION }} + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Install Dependencies + run: poetry install + - name: Run pytest + run: poetry run pytest diff --git a/tests/test_remove.py b/tests/test_remove.py new file mode 100644 index 0000000..975ee75 --- /dev/null +++ b/tests/test_remove.py @@ -0,0 +1,107 @@ +from textwrap import dedent +from verified_cogen.runners.validating import remove_asserts_and_invariants + + +def test_remove_line(): + code = dedent( + """\ + method main() { + assert a == 1; // assert-line + }""" + ) + assert remove_asserts_and_invariants(code) == dedent( + """\ + method main() { + }""" + ) + + +def test_remove_multiline_assert(): + code = dedent( + """\ + method main() { + // assert-start + assert a == 1 by { + + } + // assert-end + }""" + ) + assert remove_asserts_and_invariants(code) == dedent( + """\ + method main() { + }""" + ) + + +def test_remove_invariants(): + code = dedent( + """\ + method main() { + while true + // invariants-start + invariant false + invariant true + // invariants-end + { + } + }""" + ) + assert remove_asserts_and_invariants(code) == dedent( + """\ + method main() { + while true + { + } + }""" + ) + + +def test_remove_all(): + code = dedent( + """\ + method is_prime(k: int) returns (result: bool) + requires k >= 2 + ensures result ==> forall i :: 2 <= i < k ==> k % i != 0 + ensures !result ==> exists j :: 2 <= j < k && k % j == 0 + { + var i := 2; + result := true; + while i < k + // invariants-start + invariant 2 <= i <= k + invariant !result ==> exists j :: 2 <= j < i && k % j == 0 + invariant result ==> forall j :: 2 <= j < i ==> k % j != 0 + // invariants-end + { + if k % i == 0 { + result := false; + } + assert result ==> forall j :: 2 <= j < i ==> k % j != 0; // assert-line + // assert-start + assert !result ==> exists j :: 2 <= j < i && k % j == 0 by { + assert true; + } + // assert-end + i := i + 1; + } + }""" + ) + assert remove_asserts_and_invariants(code) == dedent( + """\ + method is_prime(k: int) returns (result: bool) + requires k >= 2 + ensures result ==> forall i :: 2 <= i < k ==> k % i != 0 + ensures !result ==> exists j :: 2 <= j < k && k % j == 0 + { + var i := 2; + result := true; + while i < k + { + if k % i == 0 { + result := false; + } + i := i + 1; + } + }""" + ) diff --git a/verified_cogen/runners/validating.py b/verified_cogen/runners/validating.py index 79c49cc..63fa5b9 100644 --- a/verified_cogen/runners/validating.py +++ b/verified_cogen/runners/validating.py @@ -7,7 +7,9 @@ from verified_cogen.runners.invariants import InvariantRunner # Regular expression to match Dafny method definitions -method_pattern = re.compile(r'method\s+(\w+)\s*\((.*?)\)\s*returns\s*\((.*?)\)(.*?)\{', re.DOTALL) +method_pattern = re.compile( + r"method\s+(\w+)\s*\((.*?)\)\s*returns\s*\((.*?)\)(.*?)\{", re.DOTALL +) def generate_validators(dafny_code: str) -> str: @@ -34,7 +36,9 @@ def generate_validators(dafny_code: str) -> str: returns = match.group(3) specs = match.group(4) - validator = f"method {method_name}_valid({parameters}) returns ({returns}){specs}" + validator = ( + f"method {method_name}_valid({parameters}) returns ({returns}){specs}" + ) validator += "{ var ret := " validator += f"{method_name}({', '.join(param.split(':')[0].strip() for param in parameters.split(',') if param.strip())});" @@ -43,23 +47,23 @@ def generate_validators(dafny_code: str) -> str: validators.append(validator) - return '\n'.join(validators) + return "\n".join(validators) def remove_asserts_and_invariants(dafny_code: str) -> str: patterns = [ - r'// assert-line.*\n', - r'// assert-start.*?// assert-end\n', - r'// invariants-start.*?// invariants-end\n' + r" *// assert-start.*?// assert-end\n", + r" *// invariants-start.*?// invariants-end\n", ] - combined_pattern = '|'.join(patterns) - cleaned_code = re.sub(combined_pattern, '', dafny_code, flags=re.DOTALL) - cleaned_code = re.sub(r'\n\s*\n', '\n', cleaned_code) - return cleaned_code.strip() + combined_pattern = "|".join(patterns) + cleaned_code = re.sub(combined_pattern, "", dafny_code, flags=re.DOTALL) + cleaned_code = re.sub(r"\n\s*\n", "\n", cleaned_code) + lines = cleaned_code.split("\n") + lines = [line for line in lines if "// assert-line" not in line] + return "\n".join(lines).strip() class ValidatingRunner(Runner): - @classmethod def _add_validators(cls, prg: str, inv_prg: str): validators = generate_validators(prg) @@ -80,7 +84,9 @@ def produce(cls, llm: LLM, prg: str) -> str: @classmethod def insert(cls, llm: LLM, prg: str, checks: str, mode: Mode) -> str: - return ValidatingRunner._add_validators(prg, InvariantRunner.insert(llm, prg, checks, mode)) + return ValidatingRunner._add_validators( + prg, InvariantRunner.insert(llm, prg, checks, mode) + ) @classmethod def precheck(cls, prg: str, mode: Mode): From a6fb45c7766835a8605a3f0b8037e644731ddd26 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Wed, 4 Sep 2024 11:44:16 +0300 Subject: [PATCH 15/15] Update poetry and actions --- .github/workflows/test-pytest.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-pytest.yml index fa349e2..82a3786 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-pytest.yml @@ -4,7 +4,7 @@ on: [push] env: PYTHON_VERSION: "3.11" - POETRY_VERSION: "1.4.2" + POETRY_VERSION: "1.8.3" POETRY_URL: https://install.python-poetry.org jobs: @@ -12,19 +12,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} id: setup_python - name: Cache Poetry cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/pypoetry key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }} - name: Cache Packages - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.local key: poetry-local-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/*.yml') }}