From b6aacaad69f9216353ef9c0eb0e5cbe9bf48b7d5 Mon Sep 17 00:00:00 2001 From: Issa Hanou Date: Fri, 4 Oct 2024 15:16:44 +0200 Subject: [PATCH 1/6] new attempt at clean PSB2 with basement, coinsums, fizzbuzz, and gcd --- src/data/PSB2_2021/PSB2_2021.jl | 35 +++++ src/data/PSB2_2021/README.md | 37 +++++ src/data/PSB2_2021/base_grammars.jl | 148 ++++++++++++++++++ src/data/PSB2_2021/citation.bib | 16 ++ src/data/PSB2_2021/data.jl | 25 +++ .../PSB2_2021/example_using_the_benchmark.jl | 18 +++ src/data/PSB2_2021/grammars.jl | 130 +++++++++++++++ src/data/PSB2_2021/program_examples.jl | 68 ++++++++ src/data/PSB2_2021/psb2_primitives.jl | 20 +++ src/data/PSB2_2021/retrieve_all_tasks.jl | 28 ++++ 10 files changed, 525 insertions(+) create mode 100644 src/data/PSB2_2021/PSB2_2021.jl create mode 100644 src/data/PSB2_2021/README.md create mode 100644 src/data/PSB2_2021/base_grammars.jl create mode 100644 src/data/PSB2_2021/citation.bib create mode 100644 src/data/PSB2_2021/data.jl create mode 100644 src/data/PSB2_2021/example_using_the_benchmark.jl create mode 100644 src/data/PSB2_2021/grammars.jl create mode 100644 src/data/PSB2_2021/program_examples.jl create mode 100644 src/data/PSB2_2021/psb2_primitives.jl create mode 100644 src/data/PSB2_2021/retrieve_all_tasks.jl diff --git a/src/data/PSB2_2021/PSB2_2021.jl b/src/data/PSB2_2021/PSB2_2021.jl new file mode 100644 index 00000000..d23ac113 --- /dev/null +++ b/src/data/PSB2_2021/PSB2_2021.jl @@ -0,0 +1,35 @@ +module PSB2_2021 + +using JSON +using HerbSpecification +# using HerbInterpret + +include("data.jl") + +export + parse_line_json + +# interpret = HerbInterpret.interpret + +""" + parse_line_json(line::AbstractString)::IOExample + +Parses a line from a file in the `strings` dataset +""" +function parse_line_json(line::AbstractString)::IOExample + js = JSON.parse(line) + inputs = Dict{Symbol, Any}() + outputs = Dict{Symbol, Any}() + for (k, v) in js + if occursin("output", k) + outputs[Symbol(k)] = v + elseif occursin("input", k) + inputs[Symbol(k)] = v + else + throw(KeyError("Unknown type of JSON key: no input or output")) + end + end + return IOExample(inputs, outputs) +end + +end # module PSB2_2021 \ No newline at end of file diff --git a/src/data/PSB2_2021/README.md b/src/data/PSB2_2021/README.md new file mode 100644 index 00000000..b6628b18 --- /dev/null +++ b/src/data/PSB2_2021/README.md @@ -0,0 +1,37 @@ +# PSB2: The Second Program Synthesis Benchmark Suite + +This dataset is comprised of 25 different tasks defined in PSB2. The following table (from [the paper](https://dl.acm.org/doi/abs/10.1145/3449639.3459285?casa_token=biEgaE8LwGkAAAAA%3AyObtJCr1MPh3ObTIh6RQUFP7Sx2E4isZAOpTHNWLkJuCcmOPRGnR94xTCddGkTJLwEbx_LpKfFv8)) describes the tasks. + +afbeelding + +## Instruction Set +The instruction set used in the benchmark paper is PushGP, "a stack-based programming language built specifically for use in genetic programming". More information can be found in a paper on [Push3](https://dl.acm.org/doi/10.1145/1068009.1068292) and [PushGP](https://link.springer.com/article/10.1023/A:1014538503543). This directory includes a (work in progress) Julia reimplementation of the required instruction set. The translation from Clojure (the language that Push is implemented in - original implementation can be found [here](https://github.com/thelmuth/Clojush/tree/psb2-v1.0/src/clojush/instructions)) was completed partially by hand, but with the help of ChatGPT and Copilot. + +The table below shows the different input sets used for each problem in the benchmark, the sets of grammars that are available, as well as the constants. + +![image](https://github.com/Herb-AI/HerbBenchmarks.jl/assets/23522361/2f7aac44-833f-4acd-b052-30bbb93bf561) + + +For more information, see: +> T. Helmuth and P. Kelly, “PSB2: The Second Program Synthesis Benchmark Suite”. Zenodo, Apr. 10, 2021. doi: 10.5281/zenodo.5084812. + + +## Structure of benchmark folder +The 25 tasks of the benchmark are added iteratively. The `data.jl` does already contain examples of all tasks, but the rest only for the already implemented ones, see below. + +We do not keep all data in this repository. The `retrieve_all_tasks.jl` functionality can be used to retrieve larger problem examples. The `data.jl` file keeps a small list of IOExamples per task. + +- `grammars.jl` This file holds the grammars for each of the benchmark problems, it constructs the specific grammars defining the constants, inputs, and outputs, and it merges all necessary grammars into one for each problem. +- `base_grammars.jl` This file includes the base grammars with general functions for integers, strings, characters, booleans, lists, and execution statements. When an ephemeral random constant (ERC) is used, we interpret this as adding one character noise to the grammar. +- `psb2_primitives.jl` This file includes extra grammar functions, like functions used in the base grammars and `merge_grammar`. +- `data.jl` This file holds some example problems with a small set of IOExamples for each problem in the benchmark. +- `program_examples.jl` This file shows some example programs for each problem: possible outputs of the synthesis task. +- `retrieve_all_tasks.jl` This file shows the functionality for retrieving larger problems from the benchmark, which can be downloaded and written to a JSON file. +- `example_using_the_benchmark.jl` This file shows an example how to use the benchmarks: where to find the grammar and the data. + +### Adding a PSB2 benchmark task + +To add another task you have to define the following: +- The data format is already specified in `data.jl` which you can use to structure the rest. +- The program example should be added in the `program_examples.jl`, where you can check that the program is possible to make using the grammar you defined for the task. Use the naming convention `program_{problem_name}`. +- The grammar needs to be defined in the `grammars.jl`, which is a merge between functions of the `base_grammars.jl` and an `input_{problem_name}` input grammar defining the constants as defined in the Instruction Set table. If it specifies an ephemeral random constant (ERC) is used, we interpret this as adding one character of noise to the grammar. Use the naming convention `grammar_{problem_name}` for the output of the `merge_grammar` of the subgrammars from `grammars.jl` with the new input grammar. Also define a `minimal_grammar_{problem_name}` containing all the functions used by the program from `program_{problem_name}` as a small test case. \ No newline at end of file diff --git a/src/data/PSB2_2021/base_grammars.jl b/src/data/PSB2_2021/base_grammars.jl new file mode 100644 index 00000000..d6da6106 --- /dev/null +++ b/src/data/PSB2_2021/base_grammars.jl @@ -0,0 +1,148 @@ +include("psb2_primitives.jl") + +# TODO parsing of integers, rename Int +grammar_integer = @csgrammar begin + Int = Int + Int + Int = Int - Int + Int = Int * Int + Int = Int / Int + Int = Int % Int + Int = Int ^ Int + Int = Int ^ 2 + Int = Int + 1 + Int = Int - 1 + Int = Int * -1 + Int = max(Int, Int) + Int = min(Int, Int) + Int = abs(Int) + Int = Integer(Boolean) + Int = Integer(String) + Int = Integer(Float) + Int = Integer(Character) + Int = ceil(Int) + Int = floor(Int) + Boolean = Int > Int + Boolean = Int >= Int + Boolean = Int < Int + Boolean = Int <= Int + Boolean = Int == Int + Boolean = Int != Int + Int = Boolean ? Int : Int + Expression = Int + Expression = begin Expression; Expression end + Int = Int -> while Bool; Expression end +end + +grammar_state_integer = @csgrammar begin + # TODO need string/float/int/bool? + State = Dict(Sym => Int) + State = Dict(Sym => Int, Sym => Int) + State = Dict(Sym => Int, Sym => Int, Sym => Int) + State = merge!(state, State) + State = push(State, Sym => Int) + Int = get(state, Sym, "Key not found") + Expression = State | Int + Expression = let state = State; Expression end +end + +grammar_list_integer = @csgrammar begin + List = map(Func, List) + Func = (x -> Int) + Int = x + Int = length(List) + Int = sum(List) + Int = indexin(Int, List) + List = getindex(List, Int:Int) +end + +grammar_float = @csgrammar begin + Number = Float | Int + Number = Number + Number + Number = Number - Number + Number = Number * Number + Number = Number / Number + Number = Number % Number + Number = Number ^ Number + Number = Number ^ 2 + Float = sqrt(Number) + Number = Number + 1 + Number = Number - 1 + Number = Number * -1 + Number = min(Number, Number) + Number = max(Number, Number) + Number = abs(Number) + Number = ceil(Number) + Number = floor(Number) + Float = cos(Number) + Float = sin(Number) + Float = tan(Number) + Float = argcos(Number) + Float = argsin(Number) + Float = argtan(Number) + Float = log(Number, Int) + Float = log(Number, 2) + Float = log(Number, 10) + Float = float(Int) + Float = float(String) + Float = float(Boolean) + Expression = Float + Expression = begin Expression; Expression end + Float = Boolean ? Expression : Expression + Float = Float -> while Bool; Expression end +end + +grammar_boolean = @csgrammar begin + Boolean = Boolean && Boolean + Boolean = Boolean || Boolean + Boolean = !Boolean + Boolean = Boolean ⊻ Boolean + Boolean = Bool(Int) + Boolean = Bool(Float) + Boolean = Boolean ? Boolean : Boolean +end + +grammar_character = @csgrammar begin + Character = Char(Int) + Character = Char(Float) + Character = Char(Boolean) + Character = Char(String) + Boolean = islowercase(Character) + Boolean = isuppercase(Character) + Boolean = isletter(Character) + Boolean = isdigit(Character) + Boolean = iswhitespace(Character) + Character = Boolean ? Character : Character +end + +grammar_string = @csgrammar begin + String = string(Character) + String = string(Int) + String = string(Boolean) + String = string(Float) + String = String * String + String = String * string(Character) + String = String[Int] + String = chop(String, head=Int, last=Int) + String = chop(String, head=0, last=1) + String = first(String) + String = last(String) + String = reverse(String) + String = replace(String, String=>String) + String = replace(String, Character=>Character) + String = replace(String, String=>String, count=1) + String = replace(String, Character=>Character, count=1) + String = replace(String, Character=>"") + String = uppercase(String) + String = lowercase(String) + String = replace_in_string(String, Int, Character) + String = "" + Int = length(String) + Int = findfirst(Character, String) + Int = count(String, String) + Boolean = contains(Character, String) + Boolean = contains(String, String) + String = Boolean ? String : String + Expression = String + Expression = begin Expression; Expression end + String = String -> while Boolean; Expression end +end \ No newline at end of file diff --git a/src/data/PSB2_2021/citation.bib b/src/data/PSB2_2021/citation.bib new file mode 100644 index 00000000..3b2b0151 --- /dev/null +++ b/src/data/PSB2_2021/citation.bib @@ -0,0 +1,16 @@ +@InProceedings{Helmuth:2021:GECCO, + author = "Thomas Helmuth and Peter Kelly", + title = "{PSB2}: The Second Program Synthesis Benchmark Suite", + booktitle = "2021 Genetic and Evolutionary Computation Conference", + series = {GECCO '21}, + year = "2021", + isbn13 = {978-1-4503-8350-9}, + address = {Lille, France}, + size = {10 pages}, + doi = {10.1145/3449639.3459285}, + publisher = {ACM}, + publisher_address = {New York, NY, USA}, + month = {10-14} # jul, + doi-url = {https://doi.org/10.1145/3449639.3459285}, + URL = {https://dl.acm.org/doi/10.1145/3449639.3459285}, +} \ No newline at end of file diff --git a/src/data/PSB2_2021/data.jl b/src/data/PSB2_2021/data.jl new file mode 100644 index 00000000..645384cc --- /dev/null +++ b/src/data/PSB2_2021/data.jl @@ -0,0 +1,25 @@ +problem_basement = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[-1]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[-100]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[-1, 100]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[-95, 100]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[-30, 5]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[-50, 50]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, -1]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, -2]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[3, -97]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[99, -100]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, -1, -1]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, -2, -1]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, -1, -1]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, -1]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[2, -2, -1]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[5, -6, -5]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[5, -5, -5]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[100, -100, -100]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[-100, -100, -100]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[-1, 100, 99]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[-10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10, -10]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[-1, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[5, -10, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, -99]), Dict{Symbol, Any}(:output1 => 19)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, 10]), Dict{Symbol, Any}(:output1 => 18)), IOExample(Dict{Symbol, Any}(:input1 => Any[93, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5]), Dict{Symbol, Any}(:output1 => 19)), IOExample(Dict{Symbol, Any}(:input1 => Any[99, -10, -9, -12, -10, -5, -12, -3, -10, -9, -9, -10, -12, -11, -12, -9, -10, -12, -11, -10]), Dict{Symbol, Any}(:output1 => 12)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 1, 1, 1, 2, 2, 2, 2, -1, -1, -1, -2, -3, -4, -5, -1, 1, 2, 3, 4]), Dict{Symbol, Any}(:output1 => 14)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, -1, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, -1]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, -1]), Dict{Symbol, Any}(:output1 => 10)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1]), Dict{Symbol, Any}(:output1 => 19)), IOExample(Dict{Symbol, Any}(:input1 => Any[50, -30, 0, 0, 0, 0, 0, 0, 0, 0, -30, 0, 0, 0, 0, 0, 0, 0, 0, -30]), Dict{Symbol, Any}(:output1 => 10)), IOExample(Dict{Symbol, Any}(:input1 => Any[50, 0, 0, 0, 0, 0, 0, 0, 0, 0, -30, 0, 0, 0, 0, 0, 0, 0, 0, -30]), Dict{Symbol, Any}(:output1 => 19)), IOExample(Dict{Symbol, Any}(:input1 => Any[50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -60]), Dict{Symbol, Any}(:output1 => 19)), IOExample(Dict{Symbol, Any}(:input1 => Any[20, -30, 0, 0, 0, 0, 0, 0, 0, 0, -30, 0, 0, 0, 0, 0, 0, 0, 0, -30]), Dict{Symbol, Any}(:output1 => 1))]) +problem_bouncing_balls = Problem([IOExample(Dict{Symbol, Any}(:input1 => 1.001, :input2 => 1.0, :input3 => 1), Dict{Symbol, Any}(:output1 => 2.001)), IOExample(Dict{Symbol, Any}(:input1 => 100.0, :input2 => 99.999, :input3 => 20), Dict{Symbol, Any}(:output1 => 3999.599534511501)), IOExample(Dict{Symbol, Any}(:input1 => 100.0, :input2 => 1.0, :input3 => 20), Dict{Symbol, Any}(:output1 => 102.02020201974588)), IOExample(Dict{Symbol, Any}(:input1 => 15.319, :input2 => 5.635, :input3 => 1), Dict{Symbol, Any}(:output1 => 20.954)), IOExample(Dict{Symbol, Any}(:input1 => 2.176, :input2 => 1.787, :input3 => 1), Dict{Symbol, Any}(:output1 => 3.963)), IOExample(Dict{Symbol, Any}(:input1 => 17.165, :input2 => 5.627, :input3 => 1), Dict{Symbol, Any}(:output1 => 22.791999999999998)), IOExample(Dict{Symbol, Any}(:input1 => 60.567, :input2 => 37.053, :input3 => 1), Dict{Symbol, Any}(:output1 => 97.62)), IOExample(Dict{Symbol, Any}(:input1 => 62.145, :input2 => 62.058, :input3 => 1), Dict{Symbol, Any}(:output1 => 124.203)), IOExample(Dict{Symbol, Any}(:input1 => 36.311, :input2 => 33.399, :input3 => 1), Dict{Symbol, Any}(:output1 => 69.71000000000001)), IOExample(Dict{Symbol, Any}(:input1 => 46.821, :input2 => 8.151, :input3 => 1), Dict{Symbol, Any}(:output1 => 54.971999999999994))]) +problem_bowling = Problem([IOExample(Dict{Symbol, Any}(:input1 => "--------------------"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "XXXXXXXXXXXX"), Dict{Symbol, Any}(:output1 => 300)), IOExample(Dict{Symbol, Any}(:input1 => "5/5/5/5/5/5/5/5/5/5/5"), Dict{Symbol, Any}(:output1 => 150)), IOExample(Dict{Symbol, Any}(:input1 => "7115XXX548/279-X53"), Dict{Symbol, Any}(:output1 => 145)), IOExample(Dict{Symbol, Any}(:input1 => "532/4362X179-41447/5"), Dict{Symbol, Any}(:output1 => 100)), IOExample(Dict{Symbol, Any}(:input1 => "24815361356212813581"), Dict{Symbol, Any}(:output1 => 75)), IOExample(Dict{Symbol, Any}(:input1 => "------X------------"), Dict{Symbol, Any}(:output1 => 10)), IOExample(Dict{Symbol, Any}(:input1 => "----------3/--------"), Dict{Symbol, Any}(:output1 => 10)), IOExample(Dict{Symbol, Any}(:input1 => "--------------1-----"), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => "11111111111111111111"), Dict{Symbol, Any}(:output1 => 20)), IOExample(Dict{Symbol, Any}(:input1 => "111111X111111111111"), Dict{Symbol, Any}(:output1 => 30)), IOExample(Dict{Symbol, Any}(:input1 => "-4-/-2-/-7-6-/-3-/-4"), Dict{Symbol, Any}(:output1 => 66)), IOExample(Dict{Symbol, Any}(:input1 => "-/-/-/-/-/-/-/-/-/-/-"), Dict{Symbol, Any}(:output1 => 100)), IOExample(Dict{Symbol, Any}(:input1 => "X52X52X52X52X52"), Dict{Symbol, Any}(:output1 => 120)), IOExample(Dict{Symbol, Any}(:input1 => "XXXXX----------"), Dict{Symbol, Any}(:output1 => 120)), IOExample(Dict{Symbol, Any}(:input1 => "XXXXX81XXX-1"), Dict{Symbol, Any}(:output1 => 208)), IOExample(Dict{Symbol, Any}(:input1 => "XXXX9/XXX2/XXX"), Dict{Symbol, Any}(:output1 => 251)), IOExample(Dict{Symbol, Any}(:input1 => "XXXXXXXXXXX9"), Dict{Symbol, Any}(:output1 => 299)), IOExample(Dict{Symbol, Any}(:input1 => "--X34--------------"), Dict{Symbol, Any}(:output1 => 24)), IOExample(Dict{Symbol, Any}(:input1 => "------3/61----------"), Dict{Symbol, Any}(:output1 => 23)), IOExample(Dict{Symbol, Any}(:input1 => "----------XX7-----"), Dict{Symbol, Any}(:output1 => 51))]) +problem_camel_case = Problem([IOExample(Dict{Symbol, Any}(:input1 => ""), Dict{Symbol, Any}(:output1 => "")), IOExample(Dict{Symbol, Any}(:input1 => "nospaceordash"), Dict{Symbol, Any}(:output1 => "nospaceordash")), IOExample(Dict{Symbol, Any}(:input1 => "two-words"), Dict{Symbol, Any}(:output1 => "twoWords")), IOExample(Dict{Symbol, Any}(:input1 => "two words"), Dict{Symbol, Any}(:output1 => "two words")), IOExample(Dict{Symbol, Any}(:input1 => "all separate words"), Dict{Symbol, Any}(:output1 => "all separate words")), IOExample(Dict{Symbol, Any}(:input1 => "all-one-word-dashed"), Dict{Symbol, Any}(:output1 => "allOneWordDashed")), IOExample(Dict{Symbol, Any}(:input1 => "loooooong-wooooords"), Dict{Symbol, Any}(:output1 => "loooooongWooooords")), IOExample(Dict{Symbol, Any}(:input1 => "loooooong wooooords"), Dict{Symbol, Any}(:output1 => "loooooong wooooords")), IOExample(Dict{Symbol, Any}(:input1 => "a-b-c-d-e-f-g-h-i-j"), Dict{Symbol, Any}(:output1 => "aBCDEFGHIJ")), IOExample(Dict{Symbol, Any}(:input1 => "a b c d e f g h i j"), Dict{Symbol, Any}(:output1 => "a b c d e f g h i j")), IOExample(Dict{Symbol, Any}(:input1 => "saaaaaaaaaaaaaaaaame"), Dict{Symbol, Any}(:output1 => "saaaaaaaaaaaaaaaaame"))]) +problem_coin_sums = Problem([IOExample(Dict{Symbol, Any}(:input1 => 1), Dict{Symbol, Any}(:output1 => 1, :output4 => 0, :output3 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 2), Dict{Symbol, Any}(:output1 => 2, :output4 => 0, :output3 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 3), Dict{Symbol, Any}(:output1 => 3, :output4 => 0, :output3 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 4), Dict{Symbol, Any}(:output1 => 4, :output4 => 0, :output3 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 5), Dict{Symbol, Any}(:output1 => 0, :output4 => 0, :output3 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 6), Dict{Symbol, Any}(:output1 => 1, :output4 => 0, :output3 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 7), Dict{Symbol, Any}(:output1 => 2, :output4 => 0, :output3 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 8), Dict{Symbol, Any}(:output1 => 3, :output4 => 0, :output3 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 9), Dict{Symbol, Any}(:output1 => 4, :output4 => 0, :output3 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 10), Dict{Symbol, Any}(:output1 => 0, :output4 => 0, :output3 => 1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 11), Dict{Symbol, Any}(:output1 => 1, :output4 => 0, :output3 => 1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 12), Dict{Symbol, Any}(:output1 => 2, :output4 => 0, :output3 => 1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 13), Dict{Symbol, Any}(:output1 => 3, :output4 => 0, :output3 => 1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 14), Dict{Symbol, Any}(:output1 => 4, :output4 => 0, :output3 => 1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 15), Dict{Symbol, Any}(:output1 => 0, :output4 => 0, :output3 => 1, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 16), Dict{Symbol, Any}(:output1 => 1, :output4 => 0, :output3 => 1, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 17), Dict{Symbol, Any}(:output1 => 2, :output4 => 0, :output3 => 1, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 18), Dict{Symbol, Any}(:output1 => 3, :output4 => 0, :output3 => 1, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 19), Dict{Symbol, Any}(:output1 => 4, :output4 => 0, :output3 => 1, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 20), Dict{Symbol, Any}(:output1 => 0, :output4 => 0, :output3 => 2, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 21), Dict{Symbol, Any}(:output1 => 1, :output4 => 0, :output3 => 2, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 22), Dict{Symbol, Any}(:output1 => 2, :output4 => 0, :output3 => 2, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 23), Dict{Symbol, Any}(:output1 => 3, :output4 => 0, :output3 => 2, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 24), Dict{Symbol, Any}(:output1 => 4, :output4 => 0, :output3 => 2, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 25), Dict{Symbol, Any}(:output1 => 0, :output4 => 1, :output3 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 26), Dict{Symbol, Any}(:output1 => 1, :output4 => 1, :output3 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 27), Dict{Symbol, Any}(:output1 => 2, :output4 => 1, :output3 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 28), Dict{Symbol, Any}(:output1 => 3, :output4 => 1, :output3 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 29), Dict{Symbol, Any}(:output1 => 4, :output4 => 1, :output3 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 30), Dict{Symbol, Any}(:output1 => 0, :output4 => 1, :output3 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 35), Dict{Symbol, Any}(:output1 => 0, :output4 => 1, :output3 => 1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => 41), Dict{Symbol, Any}(:output1 => 1, :output4 => 1, :output3 => 1, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 109), Dict{Symbol, Any}(:output1 => 4, :output4 => 4, :output3 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 10000), Dict{Symbol, Any}(:output1 => 0, :output4 => 400, :output3 => 0, :output2 => 0))]) +problem_cut_vector = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[0]), Dict{Symbol, Any}(:output1 => Any[0], :output2 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => Any[10]), Dict{Symbol, Any}(:output1 => Any[10], :output2 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => Any[100]), Dict{Symbol, Any}(:output1 => Any[100], :output2 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => Any[1000]), Dict{Symbol, Any}(:output1 => Any[1000], :output2 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => Any[10000]), Dict{Symbol, Any}(:output1 => Any[10000], :output2 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => Any[2, 129]), Dict{Symbol, Any}(:output1 => Any[2], :output2 => Any[129])), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 40]), Dict{Symbol, Any}(:output1 => Any[0], :output2 => Any[40])), IOExample(Dict{Symbol, Any}(:input1 => Any[9999, 74]), Dict{Symbol, Any}(:output1 => Any[9999], :output2 => Any[74])), IOExample(Dict{Symbol, Any}(:input1 => Any[9879, 9950]), Dict{Symbol, Any}(:output1 => Any[9879], :output2 => Any[9950])), IOExample(Dict{Symbol, Any}(:input1 => Any[9225, 9994]), Dict{Symbol, Any}(:output1 => Any[9225], :output2 => Any[9994]))]) +problem_dice_game = Problem([IOExample(Dict{Symbol, Any}(:input1 => 1, :input2 => 2), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input1 => 2, :input2 => 1), Dict{Symbol, Any}(:output1 => 0.5)), IOExample(Dict{Symbol, Any}(:input1 => 99, :input2 => 100), Dict{Symbol, Any}(:output1 => 0.49)), IOExample(Dict{Symbol, Any}(:input1 => 100, :input2 => 99), Dict{Symbol, Any}(:output1 => 0.5)), IOExample(Dict{Symbol, Any}(:input1 => 1, :input2 => 100), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input1 => 100, :input2 => 1), Dict{Symbol, Any}(:output1 => 0.99)), IOExample(Dict{Symbol, Any}(:input1 => 3, :input2 => 4), Dict{Symbol, Any}(:output1 => 0.25)), IOExample(Dict{Symbol, Any}(:input1 => 4, :input2 => 3), Dict{Symbol, Any}(:output1 => 0.5)), IOExample(Dict{Symbol, Any}(:input1 => 4, :input2 => 6), Dict{Symbol, Any}(:output1 => 0.25)), IOExample(Dict{Symbol, Any}(:input1 => 6, :input2 => 4), Dict{Symbol, Any}(:output1 => 0.5833333)), IOExample(Dict{Symbol, Any}(:input1 => 49, :input2 => 50), Dict{Symbol, Any}(:output1 => 0.48)), IOExample(Dict{Symbol, Any}(:input1 => 50, :input2 => 49), Dict{Symbol, Any}(:output1 => 0.5)), IOExample(Dict{Symbol, Any}(:input1 => 1, :input2 => 1), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input1 => 50, :input2 => 50), Dict{Symbol, Any}(:output1 => 0.49)), IOExample(Dict{Symbol, Any}(:input1 => 100, :input2 => 100), Dict{Symbol, Any}(:output1 => 0.495))]) +problem_find_pair = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[5, 7], :input2 => 12), Dict{Symbol, Any}(:output1 => 5, :output2 => 7)), IOExample(Dict{Symbol, Any}(:input1 => Any[2500, 6352], :input2 => 8852), Dict{Symbol, Any}(:output1 => 2500, :output2 => 6352)), IOExample(Dict{Symbol, Any}(:input1 => Any[-14, 5], :input2 => -9), Dict{Symbol, Any}(:output1 => -14, :output2 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[40, -19], :input2 => 21), Dict{Symbol, Any}(:output1 => 40, :output2 => -19)), IOExample(Dict{Symbol, Any}(:input1 => Any[-4, 4], :input2 => 0), Dict{Symbol, Any}(:output1 => -4, :output2 => 4)), IOExample(Dict{Symbol, Any}(:input1 => Any[-5, -20], :input2 => -25), Dict{Symbol, Any}(:output1 => -5, :output2 => -20)), IOExample(Dict{Symbol, Any}(:input1 => Any[-7831, -3001], :input2 => -10832), Dict{Symbol, Any}(:output1 => -7831, :output2 => -3001)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000, -10000], :input2 => 0), Dict{Symbol, Any}(:output1 => 10000, :output2 => -10000)), IOExample(Dict{Symbol, Any}(:input1 => Any[8310, -8320], :input2 => -10), Dict{Symbol, Any}(:output1 => 8310, :output2 => -8320)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 1234], :input2 => 1234), Dict{Symbol, Any}(:output1 => 0, :output2 => 1234)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 3, 5], :input2 => 4), Dict{Symbol, Any}(:output1 => 1, :output2 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 3, 5], :input2 => 6), Dict{Symbol, Any}(:output1 => 1, :output2 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 3, 5], :input2 => 8), Dict{Symbol, Any}(:output1 => 3, :output2 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[-1, 0, 1], :input2 => 0), Dict{Symbol, Any}(:output1 => -1, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[-1, 0, 1], :input2 => 1), Dict{Symbol, Any}(:output1 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[-1, 0, 1], :input2 => -1), Dict{Symbol, Any}(:output1 => -1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[14, -35, 78], :input2 => -21), Dict{Symbol, Any}(:output1 => 14, :output2 => -35)), IOExample(Dict{Symbol, Any}(:input1 => Any[14, -35, 78], :input2 => 92), Dict{Symbol, Any}(:output1 => 14, :output2 => 78)), IOExample(Dict{Symbol, Any}(:input1 => Any[14, -35, 78], :input2 => 43), Dict{Symbol, Any}(:output1 => -35, :output2 => 78)), IOExample(Dict{Symbol, Any}(:input1 => Any[9492, -3791, -7317], :input2 => 5701), Dict{Symbol, Any}(:output1 => 9492, :output2 => -3791)), IOExample(Dict{Symbol, Any}(:input1 => Any[9492, -3791, -7317], :input2 => 2175), Dict{Symbol, Any}(:output1 => 9492, :output2 => -7317)), IOExample(Dict{Symbol, Any}(:input1 => Any[9492, -3791, -7317], :input2 => -11108), Dict{Symbol, Any}(:output1 => -3791, :output2 => -7317)), IOExample(Dict{Symbol, Any}(:input1 => Any[237, 410, -777], :input2 => 647), Dict{Symbol, Any}(:output1 => 237, :output2 => 410)), IOExample(Dict{Symbol, Any}(:input1 => Any[237, 410, -777], :input2 => -540), Dict{Symbol, Any}(:output1 => 237, :output2 => -777)), IOExample(Dict{Symbol, Any}(:input1 => Any[237, 410, -777], :input2 => -367), Dict{Symbol, Any}(:output1 => 410, :output2 => -777))]) +problem_fizz_buzz = Problem([IOExample(Dict{Symbol, Any}(:input1 => 1), Dict{Symbol, Any}(:output1 => "1")), IOExample(Dict{Symbol, Any}(:input1 => 2), Dict{Symbol, Any}(:output1 => "2")), IOExample(Dict{Symbol, Any}(:input1 => 3), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 4), Dict{Symbol, Any}(:output1 => "4")), IOExample(Dict{Symbol, Any}(:input1 => 5), Dict{Symbol, Any}(:output1 => "Buzz")), IOExample(Dict{Symbol, Any}(:input1 => 6), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 7), Dict{Symbol, Any}(:output1 => "7")), IOExample(Dict{Symbol, Any}(:input1 => 8), Dict{Symbol, Any}(:output1 => "8")), IOExample(Dict{Symbol, Any}(:input1 => 9), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 10), Dict{Symbol, Any}(:output1 => "Buzz")), IOExample(Dict{Symbol, Any}(:input1 => 11), Dict{Symbol, Any}(:output1 => "11")), IOExample(Dict{Symbol, Any}(:input1 => 12), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 13), Dict{Symbol, Any}(:output1 => "13")), IOExample(Dict{Symbol, Any}(:input1 => 14), Dict{Symbol, Any}(:output1 => "14")), IOExample(Dict{Symbol, Any}(:input1 => 15), Dict{Symbol, Any}(:output1 => "FizzBuzz")), IOExample(Dict{Symbol, Any}(:input1 => 16), Dict{Symbol, Any}(:output1 => "16")), IOExample(Dict{Symbol, Any}(:input1 => 17), Dict{Symbol, Any}(:output1 => "17")), IOExample(Dict{Symbol, Any}(:input1 => 18), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 19), Dict{Symbol, Any}(:output1 => "19")), IOExample(Dict{Symbol, Any}(:input1 => 20), Dict{Symbol, Any}(:output1 => "Buzz")), IOExample(Dict{Symbol, Any}(:input1 => 49995), Dict{Symbol, Any}(:output1 => "FizzBuzz")), IOExample(Dict{Symbol, Any}(:input1 => 49998), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 49999), Dict{Symbol, Any}(:output1 => "49999")), IOExample(Dict{Symbol, Any}(:input1 => 50000), Dict{Symbol, Any}(:output1 => "Buzz"))]) +problem_fuel_cost = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[6]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[7]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[8]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[9]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[10]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[11]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[12]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[13]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[14]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[15]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[16]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[17]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[9998]), Dict{Symbol, Any}(:output1 => 3330)), IOExample(Dict{Symbol, Any}(:input1 => Any[9999]), Dict{Symbol, Any}(:output1 => 3331)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000]), Dict{Symbol, Any}(:output1 => 3331)), IOExample(Dict{Symbol, Any}(:input1 => Any[6, 6]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 14]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 15]), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => Any[14, 9]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[15, 9]), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => Any[32, 32]), Dict{Symbol, Any}(:output1 => 16)), IOExample(Dict{Symbol, Any}(:input1 => Any[33, 33]), Dict{Symbol, Any}(:output1 => 18)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000, 9]), Dict{Symbol, Any}(:output1 => 3332)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 10000]), Dict{Symbol, Any}(:output1 => 3332)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000, 10000]), Dict{Symbol, Any}(:output1 => 6662)), IOExample(Dict{Symbol, Any}(:input1 => Any[6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]), Dict{Symbol, Any}(:output1 => 20)), IOExample(Dict{Symbol, Any}(:input1 => Any[10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]), Dict{Symbol, Any}(:output1 => 20)), IOExample(Dict{Symbol, Any}(:input1 => Any[11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11]), Dict{Symbol, Any}(:output1 => 20)), IOExample(Dict{Symbol, Any}(:input1 => Any[12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12]), Dict{Symbol, Any}(:output1 => 40)), IOExample(Dict{Symbol, Any}(:input1 => Any[13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13]), Dict{Symbol, Any}(:output1 => 40)), IOExample(Dict{Symbol, Any}(:input1 => Any[9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998]), Dict{Symbol, Any}(:output1 => 66600)), IOExample(Dict{Symbol, Any}(:input1 => Any[9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999]), Dict{Symbol, Any}(:output1 => 66620)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000]), Dict{Symbol, Any}(:output1 => 66620)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]), Dict{Symbol, Any}(:output1 => 15))]) +problem_gcd = Problem([IOExample(Dict{Symbol, Any}(:input1 => 1, :input2 => 1), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 4, :input2 => 400000), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => 54, :input2 => 24), Dict{Symbol, Any}(:output1 => 6)), IOExample(Dict{Symbol, Any}(:input1 => 4200, :input2 => 3528), Dict{Symbol, Any}(:output1 => 168)), IOExample(Dict{Symbol, Any}(:input1 => 820000, :input2 => 63550), Dict{Symbol, Any}(:output1 => 2050)), IOExample(Dict{Symbol, Any}(:input1 => 123456, :input2 => 654321), Dict{Symbol, Any}(:output1 => 3))]) +problem_indices_of_substring = Problem([IOExample(Dict{Symbol, Any}(:input1 => "a", :input2 => "5"), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => "!", :input2 => "!"), Dict{Symbol, Any}(:output1 => Any[0])), IOExample(Dict{Symbol, Any}(:input1 => "r", :input2 => "nm,xcnwqnd@#\$fwkdjn3"), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => "hi", :input2 => "hihihihihihihihihihi"), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => "############", :input2 => "#"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])), IOExample(Dict{Symbol, Any}(:input1 => "GGGGGGGGGGGGGGGGGGGG", :input2 => "G"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19])), IOExample(Dict{Symbol, Any}(:input1 => "\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$", :input2 => "\$\$"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18])), IOExample(Dict{Symbol, Any}(:input1 => "33333333333333333333", :input2 => "333"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17])), IOExample(Dict{Symbol, Any}(:input1 => "hahahahahahahahahaha", :input2 => "haha"), Dict{Symbol, Any}(:output1 => Any[0, 2, 4, 6, 8, 10, 12, 14, 16])), IOExample(Dict{Symbol, Any}(:input1 => "GCTGCTGCTGCTGCTGCTGC", :input2 => "GCTGC"), Dict{Symbol, Any}(:output1 => Any[0, 3, 6, 9, 12, 15])), IOExample(Dict{Symbol, Any}(:input1 => "bbbbbbb(bb#bbbbbbbb", :input2 => "bbb"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 11, 12, 13, 14, 15, 16])), IOExample(Dict{Symbol, Any}(:input1 => "fa la la la la, la ", :input2 => "la"), Dict{Symbol, Any}(:output1 => Any[3, 6, 9, 12, 16])), IOExample(Dict{Symbol, Any}(:input1 => "start and and with s", :input2 => "s"), Dict{Symbol, Any}(:output1 => Any[0, 19])), IOExample(Dict{Symbol, Any}(:input1 => "tomato", :input2 => "tom"), Dict{Symbol, Any}(:output1 => Any[0])), IOExample(Dict{Symbol, Any}(:input1 => "tomatotomatotomato", :input2 => "tom"), Dict{Symbol, Any}(:output1 => Any[0, 6, 12])), IOExample(Dict{Symbol, Any}(:input1 => "tomatotomatotomato", :input2 => "to"), Dict{Symbol, Any}(:output1 => Any[0, 4, 6, 10, 12, 16])), IOExample(Dict{Symbol, Any}(:input1 => "will be zero", :input2 => "this will be zero"), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => "APPEAR twice APPEAR", :input2 => "APPEAR"), Dict{Symbol, Any}(:output1 => Any[0, 13])), IOExample(Dict{Symbol, Any}(:input1 => "a few ending <3<3<3", :input2 => "<3"), Dict{Symbol, Any}(:output1 => Any[13, 15, 17])), IOExample(Dict{Symbol, Any}(:input1 => "middle of this one", :input2 => "of"), Dict{Symbol, Any}(:output1 => Any[7]))]) +problem_leaders = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[]), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => Any[0]), Dict{Symbol, Any}(:output1 => Any[0])), IOExample(Dict{Symbol, Any}(:input1 => Any[451]), Dict{Symbol, Any}(:output1 => Any[451])), IOExample(Dict{Symbol, Any}(:input1 => Any[1000, 0]), Dict{Symbol, Any}(:output1 => Any[1000, 0])), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 1000]), Dict{Symbol, Any}(:output1 => Any[1000])), IOExample(Dict{Symbol, Any}(:input1 => Any[20, 137, 20]), Dict{Symbol, Any}(:output1 => Any[137, 20])), IOExample(Dict{Symbol, Any}(:input1 => Any[47, 87, 43, 44]), Dict{Symbol, Any}(:output1 => Any[87, 44])), IOExample(Dict{Symbol, Any}(:input1 => Any[5, 5, 5, 5, 5, 5, 5]), Dict{Symbol, Any}(:output1 => Any[5, 5, 5, 5, 5, 5, 5])), IOExample(Dict{Symbol, Any}(:input1 => Any[10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]), Dict{Symbol, Any}(:output1 => Any[10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0])), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), Dict{Symbol, Any}(:output1 => Any[10]))]) +problem_luhn = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3]), Dict{Symbol, Any}(:output1 => 80)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]), Dict{Symbol, Any}(:output1 => 144)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]), Dict{Symbol, Any}(:output1 => 48)), IOExample(Dict{Symbol, Any}(:input1 => Any[4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]), Dict{Symbol, Any}(:output1 => 96)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 0, 2, 0, 4, 3, 2, 1, 0, 4, 1, 2, 3, 4, 2, 1]), Dict{Symbol, Any}(:output1 => 45)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0]), Dict{Symbol, Any}(:output1 => 6)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 8)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 6)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 7)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 9)), IOExample(Dict{Symbol, Any}(:input1 => Any[8, 0, 0, 0, 0, 6, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 22)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 14)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5]), Dict{Symbol, Any}(:output1 => 40)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 9, 8, 7, 6, 6, 7, 8, 9, 9, 8, 7, 6, 5, 5, 6]), Dict{Symbol, Any}(:output1 => 101)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 7, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 10))]) +problem_mastermind = Problem([IOExample(Dict{Symbol, Any}(:input1 => "RRRR", :input2 => "RRRR"), Dict{Symbol, Any}(:output1 => 0, :output2 => 4)), IOExample(Dict{Symbol, Any}(:input1 => "BOYG", :input2 => "GYOB"), Dict{Symbol, Any}(:output1 => 4, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "WYYW", :input2 => "BBOG"), Dict{Symbol, Any}(:output1 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "GGGB", :input2 => "BGGG"), Dict{Symbol, Any}(:output1 => 2, :output2 => 2)), IOExample(Dict{Symbol, Any}(:input1 => "BBBB", :input2 => "OOOO"), Dict{Symbol, Any}(:output1 => 0, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "BWYG", :input2 => "YWBG"), Dict{Symbol, Any}(:output1 => 2, :output2 => 2)), IOExample(Dict{Symbol, Any}(:input1 => "RGOW", :input2 => "OGWR"), Dict{Symbol, Any}(:output1 => 3, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => "YGGB", :input2 => "GYGB"), Dict{Symbol, Any}(:output1 => 2, :output2 => 2)), IOExample(Dict{Symbol, Any}(:input1 => "YGGB", :input2 => "GYBG"), Dict{Symbol, Any}(:output1 => 4, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "GOGY", :input2 => "OGGO"), Dict{Symbol, Any}(:output1 => 2, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => "GOGR", :input2 => "GOYR"), Dict{Symbol, Any}(:output1 => 0, :output2 => 3)), IOExample(Dict{Symbol, Any}(:input1 => "YMOO", :input2 => "YMRG"), Dict{Symbol, Any}(:output1 => 0, :output2 => 2)), IOExample(Dict{Symbol, Any}(:input1 => "GROY", :input2 => "BGOW"), Dict{Symbol, Any}(:output1 => 1, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => "GGYG", :input2 => "BYBB"), Dict{Symbol, Any}(:output1 => 1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "WWWW", :input2 => "BYWR"), Dict{Symbol, Any}(:output1 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => "RBYO", :input2 => "BWBB"), Dict{Symbol, Any}(:output1 => 1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "RBRB", :input2 => "ORBY"), Dict{Symbol, Any}(:output1 => 2, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "WORR", :input2 => "BYOW"), Dict{Symbol, Any}(:output1 => 2, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "YOWW", :input2 => "YWWR"), Dict{Symbol, Any}(:output1 => 1, :output2 => 2)), IOExample(Dict{Symbol, Any}(:input1 => "BRYB", :input2 => "WOGG"), Dict{Symbol, Any}(:output1 => 0, :output2 => 0))]) +problem_middle_character = Problem([IOExample(Dict{Symbol, Any}(:input1 => "Q"), Dict{Symbol, Any}(:output1 => "Q")), IOExample(Dict{Symbol, Any}(:input1 => " "), Dict{Symbol, Any}(:output1 => " ")), IOExample(Dict{Symbol, Any}(:input1 => "\$"), Dict{Symbol, Any}(:output1 => "\$")), IOExample(Dict{Symbol, Any}(:input1 => "E9"), Dict{Symbol, Any}(:output1 => "E9")), IOExample(Dict{Symbol, Any}(:input1 => ")b"), Dict{Symbol, Any}(:output1 => ")b")), IOExample(Dict{Symbol, Any}(:input1 => "DOG"), Dict{Symbol, Any}(:output1 => "O")), IOExample(Dict{Symbol, Any}(:input1 => "OGD"), Dict{Symbol, Any}(:output1 => "G")), IOExample(Dict{Symbol, Any}(:input1 => "test"), Dict{Symbol, Any}(:output1 => "es")), IOExample(Dict{Symbol, Any}(:input1 => "\$3^:1"), Dict{Symbol, Any}(:output1 => "^")), IOExample(Dict{Symbol, Any}(:input1 => "middle"), Dict{Symbol, Any}(:output1 => "dd")), IOExample(Dict{Symbol, Any}(:input1 => " "), Dict{Symbol, Any}(:output1 => " ")), IOExample(Dict{Symbol, Any}(:input1 => "hi ~1"), Dict{Symbol, Any}(:output1 => " ")), IOExample(Dict{Symbol, Any}(:input1 => " hi~1"), Dict{Symbol, Any}(:output1 => "hi")), IOExample(Dict{Symbol, Any}(:input1 => "hi~1 "), Dict{Symbol, Any}(:output1 => "~1")), IOExample(Dict{Symbol, Any}(:input1 => "testing"), Dict{Symbol, Any}(:output1 => "t"))]) +problem_paired_digits = Problem([IOExample(Dict{Symbol, Any}(:input1 => "99"), Dict{Symbol, Any}(:output1 => 9)), IOExample(Dict{Symbol, Any}(:input1 => "88"), Dict{Symbol, Any}(:output1 => 8)), IOExample(Dict{Symbol, Any}(:input1 => "77"), Dict{Symbol, Any}(:output1 => 7)), IOExample(Dict{Symbol, Any}(:input1 => "55"), Dict{Symbol, Any}(:output1 => 5)), IOExample(Dict{Symbol, Any}(:input1 => "44"), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => "22"), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => "00"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "83"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "38"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "71"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "90"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "32"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "05"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "64"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "42"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "999"), Dict{Symbol, Any}(:output1 => 18)), IOExample(Dict{Symbol, Any}(:input1 => "555"), Dict{Symbol, Any}(:output1 => 10)), IOExample(Dict{Symbol, Any}(:input1 => "111"), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => "844"), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => "522"), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => "688"), Dict{Symbol, Any}(:output1 => 8)), IOExample(Dict{Symbol, Any}(:input1 => "233"), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => "660"), Dict{Symbol, Any}(:output1 => 6)), IOExample(Dict{Symbol, Any}(:input1 => "004"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "992"), Dict{Symbol, Any}(:output1 => 9)), IOExample(Dict{Symbol, Any}(:input1 => "123"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "841"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "808"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "454"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "295"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "99999999999999999999"), Dict{Symbol, Any}(:output1 => 171)), IOExample(Dict{Symbol, Any}(:input1 => "88888888885555555555"), Dict{Symbol, Any}(:output1 => 117)), IOExample(Dict{Symbol, Any}(:input1 => "85858585858585858585"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "00000000000000000000"), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => "11111111111111111111"), Dict{Symbol, Any}(:output1 => 19)), IOExample(Dict{Symbol, Any}(:input1 => "11223344556677889900"), Dict{Symbol, Any}(:output1 => 45)), IOExample(Dict{Symbol, Any}(:input1 => "11111888882222266666"), Dict{Symbol, Any}(:output1 => 68)), IOExample(Dict{Symbol, Any}(:input1 => "91181171161151141131"), Dict{Symbol, Any}(:output1 => 6)), IOExample(Dict{Symbol, Any}(:input1 => "77777377777377777377"), Dict{Symbol, Any}(:output1 => 91)), IOExample(Dict{Symbol, Any}(:input1 => "09876543210987654321"), Dict{Symbol, Any}(:output1 => 0))]) +problem_shopping_list = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[50.0], :input2 => Any[100.0]), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input1 => Any[50.0], :input2 => Any[10.0]), Dict{Symbol, Any}(:output1 => 45.0)), IOExample(Dict{Symbol, Any}(:input1 => Any[20.0, 20.0], :input2 => Any[100.0, 50.0]), Dict{Symbol, Any}(:output1 => 10.0)), IOExample(Dict{Symbol, Any}(:input1 => Any[20.0, 20.0], :input2 => Any[20.0, 0.0]), Dict{Symbol, Any}(:output1 => 36.0)), IOExample(Dict{Symbol, Any}(:input1 => Any[10.0, 20.0, 30.0], :input2 => Any[5.0, 10.0, 95.0]), Dict{Symbol, Any}(:output1 => 29.0)), IOExample(Dict{Symbol, Any}(:input1 => Any[43.14, 18.23, 5.33, 1.35, 39.68], :input2 => Any[100.0, 100.0, 100.0, 100.0, 100.0]), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input1 => Any[5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73, 5.73], :input2 => Any[59.19, 91.24, 25.93, 16.18, 24.65, 61.96, 67.91, 43.87, 36.23, 34.3, 96.27, 69.25, 73.78, 0.52, 8.91, 39.18, 79.67, 64.22, 14.15, 52.44]), Dict{Symbol, Any}(:output1 => 59.6)), IOExample(Dict{Symbol, Any}(:input1 => Any[25.43, 43.22, 23.42, 42.09, 25.7], :input2 => Any[0.0, 0.0, 0.0, 0.0, 0.0]), Dict{Symbol, Any}(:output1 => 159.86)), IOExample(Dict{Symbol, Any}(:input1 => Any[0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01], :input2 => Any[85.77, 43.99, 22.78, 34.14, 34.12, 8.54, 11.03]), Dict{Symbol, Any}(:output1 => 0.05)), IOExample(Dict{Symbol, Any}(:input1 => Any[9.99, 9.99, 9.99, 9.99, 9.99, 9.99, 9.99, 9.99, 9.99, 9.99], :input2 => Any[33.65, 33.65, 33.65, 33.65, 33.65, 33.65, 33.65, 33.65, 33.65, 33.65]), Dict{Symbol, Any}(:output1 => 66.28))]) +problem_snow_day = Problem([IOExample(Dict{Symbol, Any}(:input4 => 0.0, :input1 => 0, :input2 => 0.0, :input3 => 0.0), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input4 => 0.15, :input1 => 15, :input2 => 15.0, :input3 => 15.0), Dict{Symbol, Any}(:output1 => 92.5748913763936)), IOExample(Dict{Symbol, Any}(:input4 => 0.999, :input1 => 20, :input2 => 19.99, :input3 => 9.999), Dict{Symbol, Any}(:output1 => 10.00900900900901)), IOExample(Dict{Symbol, Any}(:input4 => 0.0, :input1 => 20, :input2 => 19.99, :input3 => 9.999), Dict{Symbol, Any}(:output1 => 219.96999999999994)), IOExample(Dict{Symbol, Any}(:input4 => 0.0, :input1 => 10, :input2 => 0.0, :input3 => 1.0), Dict{Symbol, Any}(:output1 => 10.0)), IOExample(Dict{Symbol, Any}(:input4 => 0.0, :input1 => 8, :input2 => 10.0, :input3 => 2.0), Dict{Symbol, Any}(:output1 => 26.0)), IOExample(Dict{Symbol, Any}(:input4 => 0.0, :input1 => 13, :input2 => 0.0, :input3 => 0.0), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input4 => 0.0, :input1 => 15, :input2 => 14.56, :input3 => 0.0), Dict{Symbol, Any}(:output1 => 14.56)), IOExample(Dict{Symbol, Any}(:input4 => 0.05, :input1 => 16, :input2 => 18.19, :input3 => 0.0), Dict{Symbol, Any}(:output1 => 8.005904102775611)), IOExample(Dict{Symbol, Any}(:input4 => 0.3, :input1 => 8, :input2 => 11.3, :input3 => 0.5), Dict{Symbol, Any}(:output1 => 2.2220091629999996)), IOExample(Dict{Symbol, Any}(:input4 => 0.05, :input1 => 5, :input2 => 1.3, :input3 => 1.5), Dict{Symbol, Any}(:output1 => 7.792487093749998)), IOExample(Dict{Symbol, Any}(:input4 => 0.0, :input1 => 10, :input2 => 0.0, :input3 => 2.0), Dict{Symbol, Any}(:output1 => 20.0))]) +problem_solve_boolean = Problem([IOExample(Dict{Symbol, Any}(:input1 => "t"), Dict{Symbol, Any}(:output1 => true)), IOExample(Dict{Symbol, Any}(:input1 => "f"), Dict{Symbol, Any}(:output1 => false)), IOExample(Dict{Symbol, Any}(:input1 => "f&f"), Dict{Symbol, Any}(:output1 => false)), IOExample(Dict{Symbol, Any}(:input1 => "f&t"), Dict{Symbol, Any}(:output1 => false)), IOExample(Dict{Symbol, Any}(:input1 => "t&f"), Dict{Symbol, Any}(:output1 => false)), IOExample(Dict{Symbol, Any}(:input1 => "t&t"), Dict{Symbol, Any}(:output1 => true)), IOExample(Dict{Symbol, Any}(:input1 => "f|f"), Dict{Symbol, Any}(:output1 => false)), IOExample(Dict{Symbol, Any}(:input1 => "f|t"), Dict{Symbol, Any}(:output1 => true)), IOExample(Dict{Symbol, Any}(:input1 => "t|f"), Dict{Symbol, Any}(:output1 => true)), IOExample(Dict{Symbol, Any}(:input1 => "t|t"), Dict{Symbol, Any}(:output1 => true))]) +problem_spin_words = Problem([IOExample(Dict{Symbol, Any}(:input1 => ""), Dict{Symbol, Any}(:output1 => "")), IOExample(Dict{Symbol, Any}(:input1 => "a"), Dict{Symbol, Any}(:output1 => "a")), IOExample(Dict{Symbol, Any}(:input1 => "this is a test"), Dict{Symbol, Any}(:output1 => "this is a test")), IOExample(Dict{Symbol, Any}(:input1 => "this is another test"), Dict{Symbol, Any}(:output1 => "this is rehtona test")), IOExample(Dict{Symbol, Any}(:input1 => "hi"), Dict{Symbol, Any}(:output1 => "hi")), IOExample(Dict{Symbol, Any}(:input1 => "cat"), Dict{Symbol, Any}(:output1 => "cat")), IOExample(Dict{Symbol, Any}(:input1 => "walk"), Dict{Symbol, Any}(:output1 => "walk")), IOExample(Dict{Symbol, Any}(:input1 => "jazz"), Dict{Symbol, Any}(:output1 => "jazz")), IOExample(Dict{Symbol, Any}(:input1 => "llama"), Dict{Symbol, Any}(:output1 => "amall")), IOExample(Dict{Symbol, Any}(:input1 => "heart"), Dict{Symbol, Any}(:output1 => "traeh")), IOExample(Dict{Symbol, Any}(:input1 => "pantry"), Dict{Symbol, Any}(:output1 => "yrtnap")), IOExample(Dict{Symbol, Any}(:input1 => "helpful"), Dict{Symbol, Any}(:output1 => "lufpleh")), IOExample(Dict{Symbol, Any}(:input1 => "disrespectful"), Dict{Symbol, Any}(:output1 => "luftcepsersid")), IOExample(Dict{Symbol, Any}(:input1 => "stop spinning these"), Dict{Symbol, Any}(:output1 => "stop gninnips eseht")), IOExample(Dict{Symbol, Any}(:input1 => "couple longer words"), Dict{Symbol, Any}(:output1 => "elpuoc regnol sdrow")), IOExample(Dict{Symbol, Any}(:input1 => "oneloongworrrrrrrrrd"), Dict{Symbol, Any}(:output1 => "drrrrrrrrrowgnooleno")), IOExample(Dict{Symbol, Any}(:input1 => "a b c d e f g h i j"), Dict{Symbol, Any}(:output1 => "a b c d e f g h i j")), IOExample(Dict{Symbol, Any}(:input1 => "ab cd ef gh ij kl mn"), Dict{Symbol, Any}(:output1 => "ab cd ef gh ij kl mn")), IOExample(Dict{Symbol, Any}(:input1 => "abc def gef hij klm"), Dict{Symbol, Any}(:output1 => "abc def gef hij klm")), IOExample(Dict{Symbol, Any}(:input1 => "word less than five"), Dict{Symbol, Any}(:output1 => "word less than five")), IOExample(Dict{Symbol, Any}(:input1 => "abcde fghij klmno"), Dict{Symbol, Any}(:output1 => "edcba jihgf onmlk")), IOExample(Dict{Symbol, Any}(:input1 => "abcdef ghijkl mnopqr"), Dict{Symbol, Any}(:output1 => "fedcba lkjihg rqponm")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefg hijklmn"), Dict{Symbol, Any}(:output1 => "gfedcba nmlkjih")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefgh ijklmnop"), Dict{Symbol, Any}(:output1 => "hgfedcba ponmlkji")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefghi jklmnopqrs"), Dict{Symbol, Any}(:output1 => "ihgfedcba srqponmlkj")), IOExample(Dict{Symbol, Any}(:input1 => "on pineapple island"), Dict{Symbol, Any}(:output1 => "on elppaenip dnalsi")), IOExample(Dict{Symbol, Any}(:input1 => "maybe this isgood"), Dict{Symbol, Any}(:output1 => "ebyam this doogsi")), IOExample(Dict{Symbol, Any}(:input1 => "racecar palindrome"), Dict{Symbol, Any}(:output1 => "racecar emordnilap")), IOExample(Dict{Symbol, Any}(:input1 => "ella is a short pali"), Dict{Symbol, Any}(:output1 => "ella is a trohs pali")), IOExample(Dict{Symbol, Any}(:input1 => "science hi"), Dict{Symbol, Any}(:output1 => "ecneics hi"))]) +problem_square_digits = Problem([IOExample(Dict{Symbol, Any}(:input1 => 0), Dict{Symbol, Any}(:output1 => "0")), IOExample(Dict{Symbol, Any}(:input1 => 1), Dict{Symbol, Any}(:output1 => "1")), IOExample(Dict{Symbol, Any}(:input1 => 2), Dict{Symbol, Any}(:output1 => "4")), IOExample(Dict{Symbol, Any}(:input1 => 3), Dict{Symbol, Any}(:output1 => "9")), IOExample(Dict{Symbol, Any}(:input1 => 4), Dict{Symbol, Any}(:output1 => "16")), IOExample(Dict{Symbol, Any}(:input1 => 5), Dict{Symbol, Any}(:output1 => "25")), IOExample(Dict{Symbol, Any}(:input1 => 7), Dict{Symbol, Any}(:output1 => "49")), IOExample(Dict{Symbol, Any}(:input1 => 9), Dict{Symbol, Any}(:output1 => "81")), IOExample(Dict{Symbol, Any}(:input1 => 10), Dict{Symbol, Any}(:output1 => "10")), IOExample(Dict{Symbol, Any}(:input1 => 12), Dict{Symbol, Any}(:output1 => "14")), IOExample(Dict{Symbol, Any}(:input1 => 16), Dict{Symbol, Any}(:output1 => "136")), IOExample(Dict{Symbol, Any}(:input1 => 24), Dict{Symbol, Any}(:output1 => "416")), IOExample(Dict{Symbol, Any}(:input1 => 35), Dict{Symbol, Any}(:output1 => "925")), IOExample(Dict{Symbol, Any}(:input1 => 46), Dict{Symbol, Any}(:output1 => "1636")), IOExample(Dict{Symbol, Any}(:input1 => 57), Dict{Symbol, Any}(:output1 => "2549")), IOExample(Dict{Symbol, Any}(:input1 => 68), Dict{Symbol, Any}(:output1 => "3664")), IOExample(Dict{Symbol, Any}(:input1 => 79), Dict{Symbol, Any}(:output1 => "4981")), IOExample(Dict{Symbol, Any}(:input1 => 80), Dict{Symbol, Any}(:output1 => "640")), IOExample(Dict{Symbol, Any}(:input1 => 92), Dict{Symbol, Any}(:output1 => "814")), IOExample(Dict{Symbol, Any}(:input1 => 98), Dict{Symbol, Any}(:output1 => "8164")), IOExample(Dict{Symbol, Any}(:input1 => 100), Dict{Symbol, Any}(:output1 => "100")), IOExample(Dict{Symbol, Any}(:input1 => 185), Dict{Symbol, Any}(:output1 => "16425")), IOExample(Dict{Symbol, Any}(:input1 => 231), Dict{Symbol, Any}(:output1 => "491")), IOExample(Dict{Symbol, Any}(:input1 => 372), Dict{Symbol, Any}(:output1 => "9494")), IOExample(Dict{Symbol, Any}(:input1 => 408), Dict{Symbol, Any}(:output1 => "16064")), IOExample(Dict{Symbol, Any}(:input1 => 794), Dict{Symbol, Any}(:output1 => "498116")), IOExample(Dict{Symbol, Any}(:input1 => 321012), Dict{Symbol, Any}(:output1 => "941014")), IOExample(Dict{Symbol, Any}(:input1 => 987654), Dict{Symbol, Any}(:output1 => "816449362516")), IOExample(Dict{Symbol, Any}(:input1 => 999999), Dict{Symbol, Any}(:output1 => "818181818181")), IOExample(Dict{Symbol, Any}(:input1 => 1000000), Dict{Symbol, Any}(:output1 => "1000000"))]) +problem_substitution_cipher = Problem([IOExample(Dict{Symbol, Any}(:input1 => "", :input2 => "", :input3 => ""), Dict{Symbol, Any}(:output1 => "")), IOExample(Dict{Symbol, Any}(:input1 => "a", :input2 => "a", :input3 => "a"), Dict{Symbol, Any}(:output1 => "a")), IOExample(Dict{Symbol, Any}(:input1 => "j", :input2 => "h", :input3 => "j"), Dict{Symbol, Any}(:output1 => "h")), IOExample(Dict{Symbol, Any}(:input1 => "a", :input2 => "z", :input3 => "a"), Dict{Symbol, Any}(:output1 => "z")), IOExample(Dict{Symbol, Any}(:input1 => "e", :input2 => "l", :input3 => "eeeeeeeeee"), Dict{Symbol, Any}(:output1 => "llllllllll")), IOExample(Dict{Symbol, Any}(:input1 => "h", :input2 => "d", :input3 => "hhhhhhhhhhhhhhhhhhhh"), Dict{Symbol, Any}(:output1 => "dddddddddddddddddddd")), IOExample(Dict{Symbol, Any}(:input1 => "o", :input2 => "z", :input3 => "oooooooooooooooooooooooooo"), Dict{Symbol, Any}(:output1 => "zzzzzzzzzzzzzzzzzzzzzzzzzz")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefghijklmnopqrstuvwxyz", :input2 => "zyxwvutsrqponmlkjihgfedcba", :input3 => "bvafvuqgjkkbeccipwdfqttgzl"), Dict{Symbol, Any}(:output1 => "yezuefjtqppyvxxrkdwujggtao")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefghijklmnopqrstuvwxyz", :input2 => "cdqutzayxshgfenjowrkvmpbil", :input3 => "thequickbrownfxjmpsvlazydg"), Dict{Symbol, Any}(:output1 => "kytovxqhdwnpezbsfjrmgcliua")), IOExample(Dict{Symbol, Any}(:input1 => "otghvwmkclidzryxsfqeapnjbu", :input2 => "alpebhxmnrcyiosvtgzjwuqdfk", :input3 => "aaabbbccc"), Dict{Symbol, Any}(:output1 => "wwwfffnnn"))]) +problem_twitter = Problem([IOExample(Dict{Symbol, Any}(:input1 => ""), Dict{Symbol, Any}(:output1 => "You didn't type anything")), IOExample(Dict{Symbol, Any}(:input1 => "1"), Dict{Symbol, Any}(:output1 => "Your tweet has 1 characters")), IOExample(Dict{Symbol, Any}(:input1 => "max length tweet that just contains letters and spaces even SOME CAPITAL LETTERS just to MAKE it INTERESTING now repeeeeeeeeeEEEEEEEeeeat it"), Dict{Symbol, Any}(:output1 => "Your tweet has 140 characters")), IOExample(Dict{Symbol, Any}(:input1 => "40172875*&(&(%^^*!@&#()!@&^(*\$787031264123984721-43214876*%^#!(@^\$_!@^%#\$(!#@%\$(01234~~``)"), Dict{Symbol, Any}(:output1 => "Your tweet has 90 characters")), IOExample(Dict{Symbol, Any}(:input1 => "Tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooolong1"), Dict{Symbol, Any}(:output1 => "Too many characters")), IOExample(Dict{Symbol, Any}(:input1 => "(@)&#N)&#@!M#&17t8023n217830123bn6 BN23780BC3879N01nc3n473N962n9768062BC3718N396b21v8365n9072B638705b097B6*&b%&%b(*B5*&%b7%(*vb&V8%v&(85V80%0857(%v97%(*&%v87c%&*c()0*^c%08v^mN098)vf%9P8V6TfB97b99870)"), Dict{Symbol, Any}(:output1 => "Too many characters"))]) +problem_vector_distance = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[-100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0], :input2 => Any[100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0]), Dict{Symbol, Any}(:output1 => 894.4271909999159)), IOExample(Dict{Symbol, Any}(:input1 => Any[5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32], :input2 => Any[5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32]), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input1 => Any[42.91283], :input2 => Any[-22.134]), Dict{Symbol, Any}(:output1 => 65.04683)), IOExample(Dict{Symbol, Any}(:input1 => Any[1.5, 2.87, 3.3324, 4.654, 5.123, 6.867, 7.5324, 8.534, 9.4132, 10.43], :input2 => Any[-1.534, -2.543, -3.423, -4.13427, -5.714, -6.713, -7.4328, -8.43, -9.73, -10.752]), Dict{Symbol, Any}(:output1 => 42.309638973086265)), IOExample(Dict{Symbol, Any}(:input1 => Any[0.4378, 0.634, 0.1234, 0.764, 0.243], :input2 => Any[-0.254, -0.1223, -0.19582, -0.8971, -0.8743]), Dict{Symbol, Any}(:output1 => 2.2715833329200144))]) \ No newline at end of file diff --git a/src/data/PSB2_2021/example_using_the_benchmark.jl b/src/data/PSB2_2021/example_using_the_benchmark.jl new file mode 100644 index 00000000..d5e6c5d6 --- /dev/null +++ b/src/data/PSB2_2021/example_using_the_benchmark.jl @@ -0,0 +1,18 @@ +# Set the imports +using Pkg +Pkg.activate(temp=true) +Pkg.add(["HerbGrammar", "HerbSpecification", "HerbInterpret", "HerbSearch"]) +using HerbGrammar, HerbSpecification, HerbInterpret, HerbSearch + +include("data.jl") +include("grammars.jl") + +# Define an iterator to use with the correct grammar and starting symbol. Optionally you can define a max_depth +iterator = BFSIterator(minimal_grammar_fizz_buzz, :String, max_depth=5) + +# Synthesize the program for the given problem. Evaluation errors must be allowed for some problems to not fail on trying to divide by 0 for example +program = @time synth(problem_fizz_buzz, iterator, allow_evaluation_errors = true) + +# Note: this might take quite some time as a BFSIterator is not very good on its own. + +println("program: ", program) \ No newline at end of file diff --git a/src/data/PSB2_2021/grammars.jl b/src/data/PSB2_2021/grammars.jl new file mode 100644 index 00000000..582d81bd --- /dev/null +++ b/src/data/PSB2_2021/grammars.jl @@ -0,0 +1,130 @@ +include("base_grammars.jl") + +## Basement problem +input_basement = @csgrammar begin + Integer = -1 | 0 | 1 + List = [] + Sym = :i + Return = Dict(:output1 => Integer) + # Integer = _(rand(-100:100)) +end + +grammar_basement = merge_grammar([input_basement, grammar_integer, grammar_list_integer, grammar_state_integer, grammar_boolean]) + +minimal_grammar_basement = @csgrammar begin + Return = Dict(:output1 => Integer) + List = input1 + Integer = 0 | 1 + Integer = Integer + Integer + Bool = Integer < Integer + Bool = Bool && Bool + Integer = length(List) + Integer = sum(List) + List = getindex(List, Integer:Integer) + Sym = :i + State = Dict(Sym => Integer) + Integer = get(state, Sym, "Key not found") + State = push!(state, Sym => Integer) + Expr = Integer | State + Expr = begin Expr; Expr end + Integer = let state = State; Expr end + State = while Bool; State end +end + +## Coin Sums problem +input_coin_sums = @csgrammar begin + Integer = 0 | 1 | 5 | 10 | 25 + Integer = input1 + Return = Dict(:output1 => Integer, :output2 => Integer, :output3 => Integer, :output4 => Integer) +end + +grammar_coin_sums = merge_grammar([ + input_coin_sums, + grammar_integer, + grammar_boolean +]) + +minimal_grammar_coin_sums = @csgrammar begin + Integer = 0 | 1 | 5 | 10 | 25 + Integer = input1 + Return = Dict(:output1 => Integer, :output2 => Integer, :output3 => Integer, :output4 => Integer) + Integer = floor(Integer) + Integer = Integer / Integer + Integer = Integer % Integer +end + +## FizzBuzz problem +input_fizz_buzz = @csgrammar begin + Integer = input1 + Integer = 0 | 3 | 5 + String = "Fizz" | "Buzz" | "FizzBuzz" + Return = Dict(:output1 => String) +end + +grammar_fizz_buzz = merge_grammar([input_fizz_buzz, grammar_integer, grammar_string, grammar_boolean]) + +minimal_grammar_fizz_buzz = @csgrammar begin + Integer = input1 + Integer = 0 | 3 | 5 + String = "Fizz" | "Buzz" | "FizzBuzz" + String = string(Integer) + Return = Dict(:output1 => String) + Integer = Integer % Integer + Bool = Integer == Integer + Integer = Bool ? Integer : Integer + Bool = Bool && Bool +end + +## Fuel cost problem +input_fuel_cost = @csgrammar begin + Integer = 0 | 1 | 2 | 3 + List = input1 + Return = Dict(:output1 => Integer) + # TODO random noise character +end + +grammar_fuel_cost = merge_grammar([ + input_fuel_cost, + grammar_integer, + grammar_boolean +]) + +minimal_grammar_fuel_cost = @csgrammar begin + List = input1 + Integer = 0 | 1 | 2 | 3 + Float = Integer / Integer + Integer = floor(Float) + Integer = Integer - Integer + Integer = sum(List) + List = map(Func, List) + Func = x -> Integer + Integer = x + Return = Dict(:output1 => Integer) +end + +## GCD problem +input_gcd = @csgrammar begin + Integer = input1 | input2 + Integer = 0 + Sym = :x | :y + Return = Dict(:output1 => Integer) + # Integer = _(rand(1,1000000)) +end + +grammar_gcd = merge_grammar([input_gcd, grammar_integer, grammar_boolean, grammar_state_integer]) + +minimal_grammar_gcd = @csgrammar begin + Integer = input1 | input2 + Return = Dict(:output1 => Integer) + Integer = 0 + Integer = Integer % Integer + Bool = Integer > Integer + Sym = :x | :y + State = Dict(Sym => Integer, Sym => Integer) + Integer = get(state, Sym, "Key not found") + State = merge!(state, State) + Expr = Integer | State + Expr = begin Expr; Expr end + Integer = let state = State; Expr end + State = while Bool; State end +end \ No newline at end of file diff --git a/src/data/PSB2_2021/program_examples.jl b/src/data/PSB2_2021/program_examples.jl new file mode 100644 index 00000000..34024463 --- /dev/null +++ b/src/data/PSB2_2021/program_examples.jl @@ -0,0 +1,68 @@ +function program_basement(input1) + # Takes list of integers and returns 1 integer + Dict(:output1 => + let state = Dict(:i => 1) + while get(state, :i, "key not found") < length(input1) && sum(getindex(input1, 1:get(state, :i, "key not found")+1)) < 0 + push!(state, :i => get!(state, :i, "key not found") + 1) + end + get(state, :i, "key not found") + end + ) +end + +function program_coin_sums(input1) + # Takes one integer and returns 4 integers + Dict( + :output4 => floor(input1 / 25), + :output3 => floor(input1 % 25 / 10), + :output2 => floor(input1 % 25 % 10 / 5), + :output1 => floor(input1 % 25 % 10 % 5)) +end + +function program_fizzbuzz(x) + # Takes an integer and returns a string + Dict(:output1 => + if x % 5 == 0 && x % 3 == 0 + "FizzBuzz" + else + if x % 3 == 0 + "Fizz" + else + if x % 5 == 0 + "Buzz" + else + string(x) + end + end + end + ) +end + +function program_fuel_cost(input1) + # Takes a list of integers and returns an integer + return Dict(:output1 => sum(map(x -> floor(x / 3) - 2, input1))) +end + +function program_gcd(input1, input2) + # Takes two integers and returns one integer + Dict(:output1 => + let state = Dict(:x => input1, :y => input2) + while state[:y] > 0 + merge!(state, Dict(:x => state[:y], :y => state[:x] % state[:y])) + end + get!(state, :x, "key not found") + end + ) +end + +function program_gcd2(input1, input2) + # Takes two integers and returns one integer + Dict(:output1 => + let state = Dict(:x => input1, :y => input2) + while get(state, :y, "key not found") > 0 + merge!(state, Dict(:x => get(state, :y, "key not found"), :y => get(state, :x, "key not found") % get(state, :y, "key not found"))) + end + get(state, :x, "key not found") + end + ) +end \ No newline at end of file diff --git a/src/data/PSB2_2021/psb2_primitives.jl b/src/data/PSB2_2021/psb2_primitives.jl new file mode 100644 index 00000000..ab91cc37 --- /dev/null +++ b/src/data/PSB2_2021/psb2_primitives.jl @@ -0,0 +1,20 @@ +function replace_in_string(str, int, char) + arr = collect(str) + arr[int] = char + join(arr) + return join(arr) +end + + +function merge_grammar(gs::Vector{ContextSensitiveGrammar}) + new_grammar = @csgrammar begin end + for g in gs + for i in eachindex(g.rules) + ex = :($(g.types[i]) = $(g.rules[i])) + add_rule!(new_grammar, ex) + end + end + return new_grammar +end + +# TODO custom interpreter/evaluator? \ No newline at end of file diff --git a/src/data/PSB2_2021/retrieve_all_tasks.jl b/src/data/PSB2_2021/retrieve_all_tasks.jl new file mode 100644 index 00000000..6e40ef26 --- /dev/null +++ b/src/data/PSB2_2021/retrieve_all_tasks.jl @@ -0,0 +1,28 @@ +using Conda, PyCall +using HerbData + +Conda.pip_interop(true) +Conda.pip("install", "psb2") + +psb2 = PyCall.pyimport("psb2") + +function write_psb2_problems_to_file(problems::Vector{String}=String["fizz-buzz"], edge_or_random::String="random", n_train::Int64=200, n_test::Int64=2000, format::String="psb2") + # If no specific problem specified, get all problems in the benchmark + if isempty(problems) + problems = psb2.PROBLEMS + end + for name in problems + if !(name in psb2.PROBLEMS) + throw(ArgumentError("$(name) does not exist in the psb2 problems, the names use '-' as separator, not '_'.")) + else + # This loads the json files to the /datasets/ folder + psb2.fetch_examples(pwd(), name, n_train, n_test, format) + julia_name = replace(name, "-" => "_") + # Reset the file if it exsits, so we can append the data all at once + if isfile("$(pwd())/datasets/$(name)/$(julia_name)data.jl") + rm("$(pwd())/datasets/$(name)/$(julia_name)data.jl") + end + parse_to_julia("$(pwd())/datasets/$(name)/", "$(name)-$(edge_or_random).json", PSB2_2021.parse_line_json, julia_name, "a") + end + end +end \ No newline at end of file From c7a490993727da4d1f593971716bc77c2e334884 Mon Sep 17 00:00:00 2001 From: Issa Hanou Date: Wed, 9 Oct 2024 16:41:43 +0200 Subject: [PATCH 2/6] added ERCs and command_while and restructuring of grammars --- src/data/PSB2_2021/README.md | 34 +++-- src/data/PSB2_2021/base_grammars.jl | 110 ++++++++------- .../PSB2_2021/example_using_the_benchmark.jl | 6 +- src/data/PSB2_2021/grammar.jl | 94 +++++++++++++ src/data/PSB2_2021/grammars.jl | 130 ------------------ src/data/PSB2_2021/problem_grammars.jl | 109 +++++++++++++++ src/data/PSB2_2021/psb2_primitives.jl | 24 +++- 7 files changed, 303 insertions(+), 204 deletions(-) create mode 100644 src/data/PSB2_2021/grammar.jl delete mode 100644 src/data/PSB2_2021/grammars.jl create mode 100644 src/data/PSB2_2021/problem_grammars.jl diff --git a/src/data/PSB2_2021/README.md b/src/data/PSB2_2021/README.md index b6628b18..f033bb82 100644 --- a/src/data/PSB2_2021/README.md +++ b/src/data/PSB2_2021/README.md @@ -1,11 +1,11 @@ # PSB2: The Second Program Synthesis Benchmark Suite -This dataset is comprised of 25 different tasks defined in PSB2. The following table (from [the paper](https://dl.acm.org/doi/abs/10.1145/3449639.3459285?casa_token=biEgaE8LwGkAAAAA%3AyObtJCr1MPh3ObTIh6RQUFP7Sx2E4isZAOpTHNWLkJuCcmOPRGnR94xTCddGkTJLwEbx_LpKfFv8)) describes the tasks. +This dataset is comprised of 25 different problems defined in PSB2. The following table (from [the paper](https://dl.acm.org/doi/abs/10.1145/3449639.3459285?casa_token=biEgaE8LwGkAAAAA%3AyObtJCr1MPh3ObTIh6RQUFP7Sx2E4isZAOpTHNWLkJuCcmOPRGnR94xTCddGkTJLwEbx_LpKfFv8)) describes the problems. afbeelding ## Instruction Set -The instruction set used in the benchmark paper is PushGP, "a stack-based programming language built specifically for use in genetic programming". More information can be found in a paper on [Push3](https://dl.acm.org/doi/10.1145/1068009.1068292) and [PushGP](https://link.springer.com/article/10.1023/A:1014538503543). This directory includes a (work in progress) Julia reimplementation of the required instruction set. The translation from Clojure (the language that Push is implemented in - original implementation can be found [here](https://github.com/thelmuth/Clojush/tree/psb2-v1.0/src/clojush/instructions)) was completed partially by hand, but with the help of ChatGPT and Copilot. +The instruction set used in the benchmark paper is PushGP, "a stack-based programming language built specifically for use in genetic programming". More information can be found in a paper on [Push3](https://dl.acm.org/doi/10.1145/1068009.1068292) and [PushGP](https://link.springer.com/article/10.1023/A:1014538503543). This directory includes a uses Julia native functions without the stack-based implementation. The table below shows the different input sets used for each problem in the benchmark, the sets of grammars that are available, as well as the constants. @@ -17,21 +17,29 @@ For more information, see: ## Structure of benchmark folder -The 25 tasks of the benchmark are added iteratively. The `data.jl` does already contain examples of all tasks, but the rest only for the already implemented ones, see below. +The 25 problems of the benchmark are added iteratively. The `data.jl` does already contain examples of all problems, but the rest only for the already implemented ones, see below. -We do not keep all data in this repository. The `retrieve_all_tasks.jl` functionality can be used to retrieve larger problem examples. The `data.jl` file keeps a small list of IOExamples per task. +We do not keep all data in this repository. The `retrieve_all_tasks.jl` functionality can be used to retrieve more tasks for a problem example. The `data.jl` file keeps a small list of IOExamples per problem. -- `grammars.jl` This file holds the grammars for each of the benchmark problems, it constructs the specific grammars defining the constants, inputs, and outputs, and it merges all necessary grammars into one for each problem. -- `base_grammars.jl` This file includes the base grammars with general functions for integers, strings, characters, booleans, lists, and execution statements. When an ephemeral random constant (ERC) is used, we interpret this as adding one character noise to the grammar. -- `psb2_primitives.jl` This file includes extra grammar functions, like functions used in the base grammars and `merge_grammar`. +- `grammar.jl` This file holds the actual grammars for each of the benchmark problems based on functions. These are combinations of specific input grammars per problem (from `problem_grammars.jl`) and default grammar for different data types (from `base_grammars.jl`). These are merged together to form the `grammar_{problem_name}` object using `merge_grammar()`. + - When an ephemeral random constant (ERC) is used, we interpret this as adding one character noise to the grammar. + - This is added within the function so it is not evaluated within the grammar, therefore there are no grammar objects, only functions + - Each input grammar should have a `Return` symbol as the return type returning a dict with the correct number of outputs +- `problem_grammars.jl` This file constructs the specific grammars defining the constants, inputs, and outputs. +- `base_grammars.jl` This file includes the base grammars with general functions for integers, strings, characters, Booleans, lists, and execution statements. +- `psb2_primitives.jl` This file includes extra grammar functions, like functions used in the base grammars (like a custom `command_while()` that has a limit) and `merge_grammar`. - `data.jl` This file holds some example problems with a small set of IOExamples for each problem in the benchmark. -- `program_examples.jl` This file shows some example programs for each problem: possible outputs of the synthesis task. -- `retrieve_all_tasks.jl` This file shows the functionality for retrieving larger problems from the benchmark, which can be downloaded and written to a JSON file. +- `program_examples.jl` This file shows some example programs for each problem: possible outputs of the synthesis. +- `retrieve_all_tasks.jl` This file shows the functionality for retrieving larger problems from the benchmark, which can be downloaded and written to a JSON file. We distinguish a problem (defined by a set of IOExamples) which can have many tasks (each IOExample). - `example_using_the_benchmark.jl` This file shows an example how to use the benchmarks: where to find the grammar and the data. -### Adding a PSB2 benchmark task +### Adding a PSB2 benchmark problem -To add another task you have to define the following: +To add another problem you have to define the following: - The data format is already specified in `data.jl` which you can use to structure the rest. -- The program example should be added in the `program_examples.jl`, where you can check that the program is possible to make using the grammar you defined for the task. Use the naming convention `program_{problem_name}`. -- The grammar needs to be defined in the `grammars.jl`, which is a merge between functions of the `base_grammars.jl` and an `input_{problem_name}` input grammar defining the constants as defined in the Instruction Set table. If it specifies an ephemeral random constant (ERC) is used, we interpret this as adding one character of noise to the grammar. Use the naming convention `grammar_{problem_name}` for the output of the `merge_grammar` of the subgrammars from `grammars.jl` with the new input grammar. Also define a `minimal_grammar_{problem_name}` containing all the functions used by the program from `program_{problem_name}` as a small test case. \ No newline at end of file +- The program example should be added in the `program_examples.jl`, where you can check that the program is possible to make using the grammar you defined for the problem. Use the naming convention `program_{problem_name}`. +- The grammar needs to be defined in the `grammar.jl`, which should be a function. The function takes in a minimal boolean to allow the inclusion of an ERC in the minimal grammar as well as the full one. The full grammar is a merge between functions of the `base_grammars.jl` and an `input_{problem_name}` defined in `problem_grammars.jl` defining the constants as defined in the Instruction Set table. + - If it specifies an ephemeral random constant (ERC) is used, we interpret this as adding one character of noise to the grammar. This is done by adding a specific rule with the random character. + - Use the naming convention `grammar_{problem_name}` for the output of the `merge_grammar` of the sub-grammars from `grammars.jl` with the new input grammar. + - Define a `minimal_grammar_{problem_name}` in `problem_grammars.jl` containing all the functions used by the program from `program_{problem_name}` as a small test case. + - Check that all required functionality is implemented in the `base_grammars.jl`, considering the list or state changes for different input types (like `String`). \ No newline at end of file diff --git a/src/data/PSB2_2021/base_grammars.jl b/src/data/PSB2_2021/base_grammars.jl index d6da6106..518d9304 100644 --- a/src/data/PSB2_2021/base_grammars.jl +++ b/src/data/PSB2_2021/base_grammars.jl @@ -1,62 +1,60 @@ include("psb2_primitives.jl") -# TODO parsing of integers, rename Int grammar_integer = @csgrammar begin - Int = Int + Int - Int = Int - Int - Int = Int * Int - Int = Int / Int - Int = Int % Int - Int = Int ^ Int - Int = Int ^ 2 - Int = Int + 1 - Int = Int - 1 - Int = Int * -1 - Int = max(Int, Int) - Int = min(Int, Int) - Int = abs(Int) - Int = Integer(Boolean) - Int = Integer(String) - Int = Integer(Float) - Int = Integer(Character) - Int = ceil(Int) - Int = floor(Int) - Boolean = Int > Int - Boolean = Int >= Int - Boolean = Int < Int - Boolean = Int <= Int - Boolean = Int == Int - Boolean = Int != Int - Int = Boolean ? Int : Int - Expression = Int + IntRule = IntRule + IntRule + IntRule = IntRule - IntRule + IntRule = IntRule * IntRule + IntRule = IntRule / IntRule + IntRule = IntRule % IntRule + IntRule = IntRule ^ IntRule + IntRule = IntRule ^ 2 + IntRule = IntRule + 1 + IntRule = IntRule - 1 + IntRule = IntRule * -1 + IntRule = max(IntRule, IntRule) + IntRule = min(IntRule, IntRule) + IntRule = abs(IntRule) + IntRule = Integer(Boolean) + IntRule = Integer(String) + IntRule = Integer(Float) + IntRule = Integer(Character) + IntRule = ceil(IntRule) + IntRule = floor(IntRule) + Boolean = IntRule > IntRule + Boolean = IntRule >= IntRule + Boolean = IntRule < IntRule + Boolean = IntRule <= IntRule + Boolean = IntRule == IntRule + Boolean = IntRule != IntRule + IntRule = Boolean ? IntRule : IntRule + Expression = IntRule Expression = begin Expression; Expression end - Int = Int -> while Bool; Expression end + IntRule = command_while(Boolean, Expression) end grammar_state_integer = @csgrammar begin - # TODO need string/float/int/bool? - State = Dict(Sym => Int) - State = Dict(Sym => Int, Sym => Int) - State = Dict(Sym => Int, Sym => Int, Sym => Int) + State = Dict(Sym => IntRule) + State = Dict(Sym => IntRule, Sym => IntRule) + State = Dict(Sym => IntRule, Sym => IntRule, Sym => IntRule) State = merge!(state, State) - State = push(State, Sym => Int) - Int = get(state, Sym, "Key not found") - Expression = State | Int + State = push(State, Sym => IntRule) + IntRule = get(state, Sym, "Key not found") + Expression = State | IntRule Expression = let state = State; Expression end end grammar_list_integer = @csgrammar begin List = map(Func, List) - Func = (x -> Int) - Int = x - Int = length(List) - Int = sum(List) - Int = indexin(Int, List) - List = getindex(List, Int:Int) + Func = (x -> IntRule) + IntRule = x + IntRule = length(List) + IntRule = sum(List) + IntRule = indexin(IntRule, List) + List = getindex(List, IntRule:IntRule) end grammar_float = @csgrammar begin - Number = Float | Int + Number = Float | IntRule Number = Number + Number Number = Number - Number Number = Number * Number @@ -79,16 +77,16 @@ grammar_float = @csgrammar begin Float = argcos(Number) Float = argsin(Number) Float = argtan(Number) - Float = log(Number, Int) + Float = log(Number, IntRule) Float = log(Number, 2) Float = log(Number, 10) - Float = float(Int) + Float = float(IntRule) Float = float(String) Float = float(Boolean) Expression = Float Expression = begin Expression; Expression end Float = Boolean ? Expression : Expression - Float = Float -> while Bool; Expression end + Float = command_while(Boolean, Expression) end grammar_boolean = @csgrammar begin @@ -96,13 +94,13 @@ grammar_boolean = @csgrammar begin Boolean = Boolean || Boolean Boolean = !Boolean Boolean = Boolean ⊻ Boolean - Boolean = Bool(Int) + Boolean = Bool(IntRule) Boolean = Bool(Float) Boolean = Boolean ? Boolean : Boolean end grammar_character = @csgrammar begin - Character = Char(Int) + Character = Char(IntRule) Character = Char(Float) Character = Char(Boolean) Character = Char(String) @@ -116,13 +114,13 @@ end grammar_string = @csgrammar begin String = string(Character) - String = string(Int) + String = string(IntRule) String = string(Boolean) String = string(Float) String = String * String String = String * string(Character) - String = String[Int] - String = chop(String, head=Int, last=Int) + String = String[IntRule] + String = chop(String, head=IntRule, last=IntRule) String = chop(String, head=0, last=1) String = first(String) String = last(String) @@ -134,15 +132,15 @@ grammar_string = @csgrammar begin String = replace(String, Character=>"") String = uppercase(String) String = lowercase(String) - String = replace_in_string(String, Int, Character) + String = replace_in_string(String, IntRule, Character) String = "" - Int = length(String) - Int = findfirst(Character, String) - Int = count(String, String) + IntRule = length(String) + IntRule = findfirst(Character, String) + IntRule = count(String, String) Boolean = contains(Character, String) Boolean = contains(String, String) String = Boolean ? String : String Expression = String Expression = begin Expression; Expression end - String = String -> while Boolean; Expression end + String = command_while(Boolean, Expression) end \ No newline at end of file diff --git a/src/data/PSB2_2021/example_using_the_benchmark.jl b/src/data/PSB2_2021/example_using_the_benchmark.jl index d5e6c5d6..81af753f 100644 --- a/src/data/PSB2_2021/example_using_the_benchmark.jl +++ b/src/data/PSB2_2021/example_using_the_benchmark.jl @@ -5,13 +5,13 @@ Pkg.add(["HerbGrammar", "HerbSpecification", "HerbInterpret", "HerbSearch"]) using HerbGrammar, HerbSpecification, HerbInterpret, HerbSearch include("data.jl") -include("grammars.jl") +include("grammar.jl") # Define an iterator to use with the correct grammar and starting symbol. Optionally you can define a max_depth -iterator = BFSIterator(minimal_grammar_fizz_buzz, :String, max_depth=5) +iterator = BFSIterator(minimal_grammar_basement, :Return, max_depth=10) # Synthesize the program for the given problem. Evaluation errors must be allowed for some problems to not fail on trying to divide by 0 for example -program = @time synth(problem_fizz_buzz, iterator, allow_evaluation_errors = true) +program = @time synth(problem_basement, iterator, allow_evaluation_errors = true) # Note: this might take quite some time as a BFSIterator is not very good on its own. diff --git a/src/data/PSB2_2021/grammar.jl b/src/data/PSB2_2021/grammar.jl new file mode 100644 index 00000000..65329ef0 --- /dev/null +++ b/src/data/PSB2_2021/grammar.jl @@ -0,0 +1,94 @@ +using Random +using HerbGrammar + +include("problem_grammars.jl") +include("base_grammars.jl") + +function get_grammar_basement(;minimal=true, seed=nothing) + # Basement has a random integer and all integers are in range (-100, 100) + if !isnothing(seed) + Ranndom.seed!(seed) + end + + grammar_basement = merge_grammar([ + input_basement, + grammar_integer, + grammar_list_integer, + grammar_state_integer, + grammar_boolean + ]) + + g = minimal ? minimal_grammar_basement : grammar_basement + g = deepcopy(g) + + int_erc = rand(-100, 100) + HerbGrammar.add_rule!(g, :(IntRule = $int_erc)) + + return g +end + +function get_gramar_coinsums(;minimal=true, seed=nothing) + # Coin sums does not have a random integer + grammar_coin_sums = merge_grammar([ + input_coin_sums, + grammar_integer, + grammar_boolean + ]) + g = minimal ? minimal_grammar_coin_sums : grammar_coin_sums + return g +end + +function get_gramar_fizz_buzz(;minimal=true, seed=nothing) + # Fizz buzz does not have a random character + grammar_fizz_buzz = merge_grammar([ + input_fizz_buzz, + grammar_integer, + grammar_string, + grammar_boolean + ]) + g = minimal ? minimal_grammar_fizz_buzz : grammar_fizz_buzz + return g +end + +function get_grammar_basement(;minimal=true, seed=nothing) + # Fuel cost has a random integer and all integers are in range (6, 1000000) + if !isnothing(seed) + Ranndom.seed!(seed) + end + + grammar_fuel_cost = merge_grammar([ + input_fuel_cost, + grammar_integer, + grammar_boolean + ]) + + g = minimal ? minimal_grammar_fuel_cost : grammar_fuel_cost + g = deepcopy(g) + + int_erc = rand(6, 1000000) + HerbGrammar.add_rule!(g, :(IntRule = $int_erc)) + + return g +end + +function get_grammar_basement(;minimal=true, seed=nothing) + # GCD has a random integer and all integers are in range (1, 1000000) + if !isnothing(seed) + Ranndom.seed!(seed) + end + + grammar_gcd = merge_grammar([ + input_gcd, + grammar_integer, + grammar_boolean, + grammar_state_integer + ]) + + g = minimal ? minimal_grammar_gcd : grammar_gcd + g = deepcopy(g) + + int_erc = rand(1, 1000000) + HerbGrammar.add_rule!(g, :(IntRule = $int_erc)) + + return g +end diff --git a/src/data/PSB2_2021/grammars.jl b/src/data/PSB2_2021/grammars.jl deleted file mode 100644 index 582d81bd..00000000 --- a/src/data/PSB2_2021/grammars.jl +++ /dev/null @@ -1,130 +0,0 @@ -include("base_grammars.jl") - -## Basement problem -input_basement = @csgrammar begin - Integer = -1 | 0 | 1 - List = [] - Sym = :i - Return = Dict(:output1 => Integer) - # Integer = _(rand(-100:100)) -end - -grammar_basement = merge_grammar([input_basement, grammar_integer, grammar_list_integer, grammar_state_integer, grammar_boolean]) - -minimal_grammar_basement = @csgrammar begin - Return = Dict(:output1 => Integer) - List = input1 - Integer = 0 | 1 - Integer = Integer + Integer - Bool = Integer < Integer - Bool = Bool && Bool - Integer = length(List) - Integer = sum(List) - List = getindex(List, Integer:Integer) - Sym = :i - State = Dict(Sym => Integer) - Integer = get(state, Sym, "Key not found") - State = push!(state, Sym => Integer) - Expr = Integer | State - Expr = begin Expr; Expr end - Integer = let state = State; Expr end - State = while Bool; State end -end - -## Coin Sums problem -input_coin_sums = @csgrammar begin - Integer = 0 | 1 | 5 | 10 | 25 - Integer = input1 - Return = Dict(:output1 => Integer, :output2 => Integer, :output3 => Integer, :output4 => Integer) -end - -grammar_coin_sums = merge_grammar([ - input_coin_sums, - grammar_integer, - grammar_boolean -]) - -minimal_grammar_coin_sums = @csgrammar begin - Integer = 0 | 1 | 5 | 10 | 25 - Integer = input1 - Return = Dict(:output1 => Integer, :output2 => Integer, :output3 => Integer, :output4 => Integer) - Integer = floor(Integer) - Integer = Integer / Integer - Integer = Integer % Integer -end - -## FizzBuzz problem -input_fizz_buzz = @csgrammar begin - Integer = input1 - Integer = 0 | 3 | 5 - String = "Fizz" | "Buzz" | "FizzBuzz" - Return = Dict(:output1 => String) -end - -grammar_fizz_buzz = merge_grammar([input_fizz_buzz, grammar_integer, grammar_string, grammar_boolean]) - -minimal_grammar_fizz_buzz = @csgrammar begin - Integer = input1 - Integer = 0 | 3 | 5 - String = "Fizz" | "Buzz" | "FizzBuzz" - String = string(Integer) - Return = Dict(:output1 => String) - Integer = Integer % Integer - Bool = Integer == Integer - Integer = Bool ? Integer : Integer - Bool = Bool && Bool -end - -## Fuel cost problem -input_fuel_cost = @csgrammar begin - Integer = 0 | 1 | 2 | 3 - List = input1 - Return = Dict(:output1 => Integer) - # TODO random noise character -end - -grammar_fuel_cost = merge_grammar([ - input_fuel_cost, - grammar_integer, - grammar_boolean -]) - -minimal_grammar_fuel_cost = @csgrammar begin - List = input1 - Integer = 0 | 1 | 2 | 3 - Float = Integer / Integer - Integer = floor(Float) - Integer = Integer - Integer - Integer = sum(List) - List = map(Func, List) - Func = x -> Integer - Integer = x - Return = Dict(:output1 => Integer) -end - -## GCD problem -input_gcd = @csgrammar begin - Integer = input1 | input2 - Integer = 0 - Sym = :x | :y - Return = Dict(:output1 => Integer) - # Integer = _(rand(1,1000000)) -end - -grammar_gcd = merge_grammar([input_gcd, grammar_integer, grammar_boolean, grammar_state_integer]) - -minimal_grammar_gcd = @csgrammar begin - Integer = input1 | input2 - Return = Dict(:output1 => Integer) - Integer = 0 - Integer = Integer % Integer - Bool = Integer > Integer - Sym = :x | :y - State = Dict(Sym => Integer, Sym => Integer) - Integer = get(state, Sym, "Key not found") - State = merge!(state, State) - Expr = Integer | State - Expr = begin Expr; Expr end - Integer = let state = State; Expr end - State = while Bool; State end -end \ No newline at end of file diff --git a/src/data/PSB2_2021/problem_grammars.jl b/src/data/PSB2_2021/problem_grammars.jl new file mode 100644 index 00000000..8375485c --- /dev/null +++ b/src/data/PSB2_2021/problem_grammars.jl @@ -0,0 +1,109 @@ +include("psb2_primitives.jl") + +## Basement problem +input_basement = @csgrammar begin + IntRule = -1 | 0 | 1 + List = [] + Sym = :i + Return = Dict(:output1 => IntRule) +end + +minimal_grammar_basement = @csgrammar begin + Return = Dict(:output1 => IntRule) + List = input1 + IntRule = 0 | 1 + IntRule = IntRule + IntRule + Boolean = IntRule < IntRule + Boolean = Boolean && Boolean + IntRule = length(List) + IntRule = sum(List) + List = getindex(List, IntRule:IntRule) + Sym = :i + State = Dict(Sym => IntRule) + IntRule = get(state, Sym, "Key not found") + State = push!(state, Sym => IntRule) + Expression = IntRule | State + Expression = begin Expression; Expression end + IntRule = let state = State; Expression end + State = command_while(Boolean, IntRule) +end + +## Coin Sums problem +input_coin_sums = @csgrammar begin + IntRule = 0 | 1 | 5 | 10 | 25 + IntRule = input1 + Return = Dict(:output1 => IntRule, :output2 => IntRule, :output3 => IntRule, :output4 => IntRule) +end + +minimal_grammar_coin_sums = @csgrammar begin + IntRule = 0 | 1 | 5 | 10 | 25 + IntRule = input1 + Return = Dict(:output1 => IntRule, :output2 => IntRule, :output3 => IntRule, :output4 => IntRule) + IntRule = floor(IntRule) + IntRule = IntRule / IntRule + IntRule = IntRule % IntRule +end + +## FizzBuzz problem +input_fizz_buzz = @csgrammar begin + IntRule = input1 + IntRule = 0 | 3 | 5 + String = "Fizz" | "Buzz" | "FizzBuzz" + Return = Dict(:output1 => String) +end + +minimal_grammar_fizz_buzz = @csgrammar begin + IntRule = input1 + IntRule = 0 | 3 | 5 + String = "Fizz" | "Buzz" | "FizzBuzz" + String = string(IntRule) + Return = Dict(:output1 => String) + IntRule = IntRule % IntRule + Boolean = IntRule == IntRule + IntRule = Boolean ? IntRule : IntRule + Boolean = Boolean && Boolean +end + +## Fuel cost problem +input_fuel_cost = @csgrammar begin + IntRule = 0 | 1 | 2 | 3 + List = input1 + Return = Dict(:output1 => IntRule) +end + +minimal_grammar_fuel_cost = @csgrammar begin + List = input1 + IntRule = 0 | 1 | 2 | 3 + Float = IntRule / IntRule + IntRule = floor(Float) + IntRule = IntRule - IntRule + IntRule = sum(List) + List = map(Func, List) + Func = x -> IntRule + IntRule = x + Return = Dict(:output1 => IntRule) +end + +## GCD problem +input_gcd = @csgrammar begin + IntRule = input1 | input2 + IntRule = 0 + Sym = :x | :y + Return = Dict(:output1 => IntRule) +end + +minimal_grammar_gcd = @csgrammar begin + IntRule = input1 | input2 + Return = Dict(:output1 => IntRule) + IntRule = 0 + IntRule = IntRule % IntRule + Boolean = IntRule > IntRule + Sym = :x | :y + State = Dict(Sym => IntRule, Sym => IntRule) + IntRule = get(state, Sym, "Key not found") + State = merge!(state, State) + Expression = IntRule | State + Expression = begin Expression; Expression end + IntRule = let state = State; Expression end + State = command_while(Boolean, State) +end \ No newline at end of file diff --git a/src/data/PSB2_2021/psb2_primitives.jl b/src/data/PSB2_2021/psb2_primitives.jl index ab91cc37..920dff10 100644 --- a/src/data/PSB2_2021/psb2_primitives.jl +++ b/src/data/PSB2_2021/psb2_primitives.jl @@ -1,3 +1,6 @@ +""" +Function for replacing a character in a string. +""" function replace_in_string(str, int, char) arr = collect(str) arr[int] = char @@ -5,7 +8,9 @@ function replace_in_string(str, int, char) return join(arr) end - +""" +Function to merge different grammars. +""" function merge_grammar(gs::Vector{ContextSensitiveGrammar}) new_grammar = @csgrammar begin end for g in gs @@ -17,4 +22,19 @@ function merge_grammar(gs::Vector{ContextSensitiveGrammar}) return new_grammar end -# TODO custom interpreter/evaluator? \ No newline at end of file + +""" +Custom implementation of a while loop with a condition and a body. + +Loop is terminated either when condition is false or when `max_steps` is reached. +""" +function command_while(condition::Any, body::Any, max_steps::Int=1000) + counter = max_steps; + res = nothing + + while condition && counter > 0 + res = eval(body) + counter -= 1 + end + return res +end From c2ad5daa4b278f4ebdea2925214c74a1423c514c Mon Sep 17 00:00:00 2001 From: Issa Hanou Date: Fri, 15 Nov 2024 11:28:30 +0100 Subject: [PATCH 3/6] added grammar objects for each problem file --- src/data/PSB2_2021/grammar.jl | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/data/PSB2_2021/grammar.jl b/src/data/PSB2_2021/grammar.jl index 65329ef0..01318883 100644 --- a/src/data/PSB2_2021/grammar.jl +++ b/src/data/PSB2_2021/grammar.jl @@ -7,7 +7,7 @@ include("base_grammars.jl") function get_grammar_basement(;minimal=true, seed=nothing) # Basement has a random integer and all integers are in range (-100, 100) if !isnothing(seed) - Ranndom.seed!(seed) + Random.seed!(seed) end grammar_basement = merge_grammar([ @@ -21,13 +21,13 @@ function get_grammar_basement(;minimal=true, seed=nothing) g = minimal ? minimal_grammar_basement : grammar_basement g = deepcopy(g) - int_erc = rand(-100, 100) + int_erc = rand((-100, 100)) HerbGrammar.add_rule!(g, :(IntRule = $int_erc)) return g end -function get_gramar_coinsums(;minimal=true, seed=nothing) +function get_grammar_coinsums(;minimal=true) # Coin sums does not have a random integer grammar_coin_sums = merge_grammar([ input_coin_sums, @@ -38,7 +38,7 @@ function get_gramar_coinsums(;minimal=true, seed=nothing) return g end -function get_gramar_fizz_buzz(;minimal=true, seed=nothing) +function get_grammar_fizz_buzz(;minimal=true) # Fizz buzz does not have a random character grammar_fizz_buzz = merge_grammar([ input_fizz_buzz, @@ -50,36 +50,37 @@ function get_gramar_fizz_buzz(;minimal=true, seed=nothing) return g end -function get_grammar_basement(;minimal=true, seed=nothing) +function get_grammar_fuelcost(;minimal=true, seed=nothing) # Fuel cost has a random integer and all integers are in range (6, 1000000) if !isnothing(seed) - Ranndom.seed!(seed) + Random.seed!(seed) end grammar_fuel_cost = merge_grammar([ input_fuel_cost, grammar_integer, - grammar_boolean + grammar_boolean, + grammar_list_integer ]) g = minimal ? minimal_grammar_fuel_cost : grammar_fuel_cost g = deepcopy(g) - int_erc = rand(6, 1000000) + int_erc = rand((6, 1000000)) HerbGrammar.add_rule!(g, :(IntRule = $int_erc)) return g end -function get_grammar_basement(;minimal=true, seed=nothing) +function get_grammar_gcd(;minimal=true, seed=nothing) # GCD has a random integer and all integers are in range (1, 1000000) if !isnothing(seed) - Ranndom.seed!(seed) + Random.seed!(seed) end grammar_gcd = merge_grammar([ - input_gcd, - grammar_integer, + input_gcd, + grammar_integer, grammar_boolean, grammar_state_integer ]) @@ -87,8 +88,14 @@ function get_grammar_basement(;minimal=true, seed=nothing) g = minimal ? minimal_grammar_gcd : grammar_gcd g = deepcopy(g) - int_erc = rand(1, 1000000) + int_erc = rand((1, 1000000)) HerbGrammar.add_rule!(g, :(IntRule = $int_erc)) return g end + +grammar_basement = get_grammar_basement(seed=42) +grammar_coin_sums = get_grammar_coinsums() +grammar_fizz_buzz = get_grammar_fizz_buzz() +grammar_fuel_cost = get_grammar_fuelcost(seed=42) +grammar_gcd = get_grammar_gcd() \ No newline at end of file From e8defb31b0944ce191c4b6068c49ca504f50d5aa Mon Sep 17 00:00:00 2001 From: Issa Hanou Date: Fri, 15 Nov 2024 16:40:18 +0100 Subject: [PATCH 4/6] update PSB2 dataset retriever --- .gitignore | 6 ++ CondaPkg.toml | 2 + Project.toml | 12 ++++ src/data/PSB2_2021/PSB2_2021.jl | 5 +- src/data/PSB2_2021/README.md | 58 +++++++++++++++++-- .../PSB2_2021/example_using_the_benchmark.jl | 19 ++++-- src/data/PSB2_2021/retrieve_all_tasks.jl | 21 ++++--- src/utils/benchmarks_io.jl | 7 +-- 8 files changed, 102 insertions(+), 28 deletions(-) create mode 100644 CondaPkg.toml diff --git a/.gitignore b/.gitignore index 3b30f6e6..32b467b5 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,9 @@ Manifest.toml # Ignore large data files *_large.jl + +# Ignore the CondaPkg files used by PSB2 for retrieving the problem files +.CondaPkg/ + +# Don't commit the datasets file when extracted from the PSB2 online benchmark +**/PSB2_2021/datasets/ \ No newline at end of file diff --git a/CondaPkg.toml b/CondaPkg.toml new file mode 100644 index 00000000..450345c4 --- /dev/null +++ b/CondaPkg.toml @@ -0,0 +1,2 @@ +[pip.deps] +psb2 = "" diff --git a/Project.toml b/Project.toml index 483c7b33..d559b309 100644 --- a/Project.toml +++ b/Project.toml @@ -4,20 +4,32 @@ authors = ["jaapjong ", "Tilman Hinnerichs ## Instruction Set -The instruction set used in the benchmark paper is PushGP, "a stack-based programming language built specifically for use in genetic programming". More information can be found in a paper on [Push3](https://dl.acm.org/doi/10.1145/1068009.1068292) and [PushGP](https://link.springer.com/article/10.1023/A:1014538503543). This directory includes a uses Julia native functions without the stack-based implementation. +The instruction set used in the benchmark paper is PushGP, "a stack-based programming language built specifically for use in genetic programming". More information can be found in a paper on [Push3](https://dl.acm.org/doi/10.1145/1068009.1068292) and [PushGP](https://link.springer.com/article/10.1023/A:1014538503543). This directory uses Julia native functions without the stack-based implementation. The table below shows the different input sets used for each problem in the benchmark, the sets of grammars that are available, as well as the constants. @@ -22,9 +43,9 @@ The 25 problems of the benchmark are added iteratively. The `data.jl` does alrea We do not keep all data in this repository. The `retrieve_all_tasks.jl` functionality can be used to retrieve more tasks for a problem example. The `data.jl` file keeps a small list of IOExamples per problem. - `grammar.jl` This file holds the actual grammars for each of the benchmark problems based on functions. These are combinations of specific input grammars per problem (from `problem_grammars.jl`) and default grammar for different data types (from `base_grammars.jl`). These are merged together to form the `grammar_{problem_name}` object using `merge_grammar()`. - - When an ephemeral random constant (ERC) is used, we interpret this as adding one character noise to the grammar. - - This is added within the function so it is not evaluated within the grammar, therefore there are no grammar objects, only functions + - When an ephemeral random constant (ERC) is used, we interpret this as adding one character noise to the grammar (the range is defined per problem). This is added within the function so it is not evaluated within the grammar, therefore there are no grammar objects, only functions - Each input grammar should have a `Return` symbol as the return type returning a dict with the correct number of outputs + - For each problem, there is an instantiation of the grammar as an object that can be retrieved when using the benchmarks. However, the functions can also be called (especially when you want to see the effect of having a different noise character). - `problem_grammars.jl` This file constructs the specific grammars defining the constants, inputs, and outputs. - `base_grammars.jl` This file includes the base grammars with general functions for integers, strings, characters, Booleans, lists, and execution statements. - `psb2_primitives.jl` This file includes extra grammar functions, like functions used in the base grammars (like a custom `command_while()` that has a limit) and `merge_grammar`. @@ -42,4 +63,31 @@ To add another problem you have to define the following: - If it specifies an ephemeral random constant (ERC) is used, we interpret this as adding one character of noise to the grammar. This is done by adding a specific rule with the random character. - Use the naming convention `grammar_{problem_name}` for the output of the `merge_grammar` of the sub-grammars from `grammars.jl` with the new input grammar. - Define a `minimal_grammar_{problem_name}` in `problem_grammars.jl` containing all the functions used by the program from `program_{problem_name}` as a small test case. - - Check that all required functionality is implemented in the `base_grammars.jl`, considering the list or state changes for different input types (like `String`). \ No newline at end of file + - Check that all required functionality is implemented in the `base_grammars.jl`, considering the list or state changes for different input types (like `String`). + +## Implemented PSB2 problems +- [x] Basement +- [ ] Bouncing Balls +- [ ] Bowling +- [ ] Camel Case +- [x] Coinsums +- [ ] Cut Vector +- [ ] Dice Game +- [ ] Find Pair +- [x] Fizzbuzz +- [x] Fuelcost +- [x] GCD +- [ ] Indices +- [ ] Leaders +- [ ] Luhn +- [ ] Mastermind +- [ ] Middle Character +- [ ] Paired Digits +- [ ] Shopping List +- [ ] Snow Day +- [ ] Solve Boolean +- [ ] Spin Words +- [ ] Square Digits +- [ ] Substitution Cipher +- [ ] Twitter +- [ ] Vector Distance \ No newline at end of file diff --git a/src/data/PSB2_2021/example_using_the_benchmark.jl b/src/data/PSB2_2021/example_using_the_benchmark.jl index 81af753f..ab438291 100644 --- a/src/data/PSB2_2021/example_using_the_benchmark.jl +++ b/src/data/PSB2_2021/example_using_the_benchmark.jl @@ -1,18 +1,27 @@ # Set the imports using Pkg -Pkg.activate(temp=true) Pkg.add(["HerbGrammar", "HerbSpecification", "HerbInterpret", "HerbSearch"]) using HerbGrammar, HerbSpecification, HerbInterpret, HerbSearch include("data.jl") include("grammar.jl") +include("program_examples.jl") -# Define an iterator to use with the correct grammar and starting symbol. Optionally you can define a max_depth -iterator = BFSIterator(minimal_grammar_basement, :Return, max_depth=10) +# Define an iterator to use with the correct grammar and starting symbol. +# Optionally you can define a max_depth +grammar = get_grammar_basement(minimal=true, seed=42) +iterator = BFSIterator(grammar, :Return, max_depth=4) # Synthesize the program for the given problem. Evaluation errors must be allowed for some problems to not fail on trying to divide by 0 for example -program = @time synth(problem_basement, iterator, allow_evaluation_errors = true) +program, result = @time synth(problem_basement, iterator, allow_evaluation_errors = true) +# program = result[1] # Note: this might take quite some time as a BFSIterator is not very good on its own. +println("Result ", result) +println("Found program rulenode: ", program) +println("Found program: ", HerbGrammar.rulenode2expr(program, grammar)) -println("program: ", program) \ No newline at end of file +# Now, we can evaluate a program +input1 = [5,2,3,92] +println("Found program outputs: ", eval(HerbGrammar.rulenode2expr(program, grammar))) +println("Provided example program outputs: ", program_basement(input1)) \ No newline at end of file diff --git a/src/data/PSB2_2021/retrieve_all_tasks.jl b/src/data/PSB2_2021/retrieve_all_tasks.jl index 6e40ef26..414ee23d 100644 --- a/src/data/PSB2_2021/retrieve_all_tasks.jl +++ b/src/data/PSB2_2021/retrieve_all_tasks.jl @@ -1,10 +1,7 @@ -using Conda, PyCall -using HerbData +using PythonCall, CondaPkg +using HerbSpecification -Conda.pip_interop(true) -Conda.pip("install", "psb2") - -psb2 = PyCall.pyimport("psb2") +psb2 = pyimport("psb2") function write_psb2_problems_to_file(problems::Vector{String}=String["fizz-buzz"], edge_or_random::String="random", n_train::Int64=200, n_test::Int64=2000, format::String="psb2") # If no specific problem specified, get all problems in the benchmark @@ -16,13 +13,15 @@ function write_psb2_problems_to_file(problems::Vector{String}=String["fizz-buzz" throw(ArgumentError("$(name) does not exist in the psb2 problems, the names use '-' as separator, not '_'.")) else # This loads the json files to the /datasets/ folder - psb2.fetch_examples(pwd(), name, n_train, n_test, format) + psb2.fetch_examples(dirname(@__FILE__), name, n_train, n_test, format) + # Reset the file if it exists, so we can append the data all at once julia_name = replace(name, "-" => "_") - # Reset the file if it exsits, so we can append the data all at once - if isfile("$(pwd())/datasets/$(name)/$(julia_name)data.jl") - rm("$(pwd())/datasets/$(name)/$(julia_name)data.jl") + dir = joinpath(dirname(@__FILE__), "datasets", name) + if isfile(joinpath(dir, julia_name * "data.jl")) + rm(joinpath(dir, julia_name * "data.jl")) end - parse_to_julia("$(pwd())/datasets/$(name)/", "$(name)-$(edge_or_random).json", PSB2_2021.parse_line_json, julia_name, "a") + # Write contents to file + HerbBenchmarks.parse_to_julia(dir, "$(name)-$(edge_or_random).json", parse_line_json, julia_name * "_") end end end \ No newline at end of file diff --git a/src/utils/benchmarks_io.jl b/src/utils/benchmarks_io.jl index 128fd356..b29534c9 100644 --- a/src/utils/benchmarks_io.jl +++ b/src/utils/benchmarks_io.jl @@ -3,7 +3,7 @@ """ function parse_file(filepath::String, line_parser::Function)::Problem file = open(filepath) - examples::Vector{Example} = Vector{Example}() + examples::AbstractVector{IOExample} = [] for line in eachline(file) line = strip(line) # Remove leading/trailing whitespace @@ -13,7 +13,6 @@ function parse_file(filepath::String, line_parser::Function)::Problem end end - close(file) return Problem(examples) end @@ -68,8 +67,8 @@ end Parses a single problem file given a line parser and writes the Herb'ed problem to the output_path. """ function parse_to_julia(path::String, filename::String, line_parser::Function, prefix::String="")::Problem - problem = parse_file(path*filename, line_parser) - write_problem(path*"$(prefix)data.jl", problem, prefix) + problem = parse_file(joinpath(path, filename), line_parser) + write_problem(joinpath(path, "$(prefix)data.jl"), problem, prefix) end """ From 127f4ab1926ce6ffb6b580a98a29466d700b2949 Mon Sep 17 00:00:00 2001 From: Issa Hanou Date: Fri, 15 Nov 2024 16:28:28 +0100 Subject: [PATCH 5/6] commit 9999999 working PSB2 benchmark, fixed some bugs, create grammar placeholder for each problem --- src/data/PSB2_2021/PSB2_2021.jl | 3 + src/data/PSB2_2021/README.md | 1 + src/data/PSB2_2021/base_grammars.jl | 2 +- src/data/PSB2_2021/data.jl | 6 +- .../PSB2_2021/example_using_the_benchmark.jl | 2 +- src/data/PSB2_2021/grammar.jl | 122 +++++++++++++++++- src/data/PSB2_2021/program_examples.jl | 14 +- src/data/PSB2_2021/psb2_primitives.jl | 10 +- src/utils/benchmarks_io.jl | 2 +- test/runtests.jl | 1 + test/test_general.jl | 5 +- test/test_psb2_2021.jl | 97 ++++++++++++++ 12 files changed, 246 insertions(+), 19 deletions(-) create mode 100644 test/test_psb2_2021.jl diff --git a/src/data/PSB2_2021/PSB2_2021.jl b/src/data/PSB2_2021/PSB2_2021.jl index ae1b998c..9226a6d9 100644 --- a/src/data/PSB2_2021/PSB2_2021.jl +++ b/src/data/PSB2_2021/PSB2_2021.jl @@ -5,6 +5,9 @@ using HerbSpecification include("data.jl") include("retrieve_all_tasks.jl") +include("grammar.jl") +include("program_examples.jl") + export parse_line_json diff --git a/src/data/PSB2_2021/README.md b/src/data/PSB2_2021/README.md index 3225df12..83b75c4d 100644 --- a/src/data/PSB2_2021/README.md +++ b/src/data/PSB2_2021/README.md @@ -59,6 +59,7 @@ We do not keep all data in this repository. The `retrieve_all_tasks.jl` function To add another problem you have to define the following: - The data format is already specified in `data.jl` which you can use to structure the rest. - The program example should be added in the `program_examples.jl`, where you can check that the program is possible to make using the grammar you defined for the problem. Use the naming convention `program_{problem_name}`. + - Also write a test for this program - The grammar needs to be defined in the `grammar.jl`, which should be a function. The function takes in a minimal boolean to allow the inclusion of an ERC in the minimal grammar as well as the full one. The full grammar is a merge between functions of the `base_grammars.jl` and an `input_{problem_name}` defined in `problem_grammars.jl` defining the constants as defined in the Instruction Set table. - If it specifies an ephemeral random constant (ERC) is used, we interpret this as adding one character of noise to the grammar. This is done by adding a specific rule with the random character. - Use the naming convention `grammar_{problem_name}` for the output of the `merge_grammar` of the sub-grammars from `grammars.jl` with the new input grammar. diff --git a/src/data/PSB2_2021/base_grammars.jl b/src/data/PSB2_2021/base_grammars.jl index 518d9304..905475d7 100644 --- a/src/data/PSB2_2021/base_grammars.jl +++ b/src/data/PSB2_2021/base_grammars.jl @@ -37,7 +37,7 @@ grammar_state_integer = @csgrammar begin State = Dict(Sym => IntRule, Sym => IntRule) State = Dict(Sym => IntRule, Sym => IntRule, Sym => IntRule) State = merge!(state, State) - State = push(State, Sym => IntRule) + State = push!(State, Sym => IntRule) IntRule = get(state, Sym, "Key not found") Expression = State | IntRule Expression = let state = State; Expression end diff --git a/src/data/PSB2_2021/data.jl b/src/data/PSB2_2021/data.jl index 645384cc..3fb0c12e 100644 --- a/src/data/PSB2_2021/data.jl +++ b/src/data/PSB2_2021/data.jl @@ -8,7 +8,7 @@ problem_dice_game = Problem([IOExample(Dict{Symbol, Any}(:input1 => 1, :input2 = problem_find_pair = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[5, 7], :input2 => 12), Dict{Symbol, Any}(:output1 => 5, :output2 => 7)), IOExample(Dict{Symbol, Any}(:input1 => Any[2500, 6352], :input2 => 8852), Dict{Symbol, Any}(:output1 => 2500, :output2 => 6352)), IOExample(Dict{Symbol, Any}(:input1 => Any[-14, 5], :input2 => -9), Dict{Symbol, Any}(:output1 => -14, :output2 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[40, -19], :input2 => 21), Dict{Symbol, Any}(:output1 => 40, :output2 => -19)), IOExample(Dict{Symbol, Any}(:input1 => Any[-4, 4], :input2 => 0), Dict{Symbol, Any}(:output1 => -4, :output2 => 4)), IOExample(Dict{Symbol, Any}(:input1 => Any[-5, -20], :input2 => -25), Dict{Symbol, Any}(:output1 => -5, :output2 => -20)), IOExample(Dict{Symbol, Any}(:input1 => Any[-7831, -3001], :input2 => -10832), Dict{Symbol, Any}(:output1 => -7831, :output2 => -3001)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000, -10000], :input2 => 0), Dict{Symbol, Any}(:output1 => 10000, :output2 => -10000)), IOExample(Dict{Symbol, Any}(:input1 => Any[8310, -8320], :input2 => -10), Dict{Symbol, Any}(:output1 => 8310, :output2 => -8320)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 1234], :input2 => 1234), Dict{Symbol, Any}(:output1 => 0, :output2 => 1234)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 3, 5], :input2 => 4), Dict{Symbol, Any}(:output1 => 1, :output2 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 3, 5], :input2 => 6), Dict{Symbol, Any}(:output1 => 1, :output2 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 3, 5], :input2 => 8), Dict{Symbol, Any}(:output1 => 3, :output2 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[-1, 0, 1], :input2 => 0), Dict{Symbol, Any}(:output1 => -1, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[-1, 0, 1], :input2 => 1), Dict{Symbol, Any}(:output1 => 0, :output2 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[-1, 0, 1], :input2 => -1), Dict{Symbol, Any}(:output1 => -1, :output2 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[14, -35, 78], :input2 => -21), Dict{Symbol, Any}(:output1 => 14, :output2 => -35)), IOExample(Dict{Symbol, Any}(:input1 => Any[14, -35, 78], :input2 => 92), Dict{Symbol, Any}(:output1 => 14, :output2 => 78)), IOExample(Dict{Symbol, Any}(:input1 => Any[14, -35, 78], :input2 => 43), Dict{Symbol, Any}(:output1 => -35, :output2 => 78)), IOExample(Dict{Symbol, Any}(:input1 => Any[9492, -3791, -7317], :input2 => 5701), Dict{Symbol, Any}(:output1 => 9492, :output2 => -3791)), IOExample(Dict{Symbol, Any}(:input1 => Any[9492, -3791, -7317], :input2 => 2175), Dict{Symbol, Any}(:output1 => 9492, :output2 => -7317)), IOExample(Dict{Symbol, Any}(:input1 => Any[9492, -3791, -7317], :input2 => -11108), Dict{Symbol, Any}(:output1 => -3791, :output2 => -7317)), IOExample(Dict{Symbol, Any}(:input1 => Any[237, 410, -777], :input2 => 647), Dict{Symbol, Any}(:output1 => 237, :output2 => 410)), IOExample(Dict{Symbol, Any}(:input1 => Any[237, 410, -777], :input2 => -540), Dict{Symbol, Any}(:output1 => 237, :output2 => -777)), IOExample(Dict{Symbol, Any}(:input1 => Any[237, 410, -777], :input2 => -367), Dict{Symbol, Any}(:output1 => 410, :output2 => -777))]) problem_fizz_buzz = Problem([IOExample(Dict{Symbol, Any}(:input1 => 1), Dict{Symbol, Any}(:output1 => "1")), IOExample(Dict{Symbol, Any}(:input1 => 2), Dict{Symbol, Any}(:output1 => "2")), IOExample(Dict{Symbol, Any}(:input1 => 3), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 4), Dict{Symbol, Any}(:output1 => "4")), IOExample(Dict{Symbol, Any}(:input1 => 5), Dict{Symbol, Any}(:output1 => "Buzz")), IOExample(Dict{Symbol, Any}(:input1 => 6), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 7), Dict{Symbol, Any}(:output1 => "7")), IOExample(Dict{Symbol, Any}(:input1 => 8), Dict{Symbol, Any}(:output1 => "8")), IOExample(Dict{Symbol, Any}(:input1 => 9), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 10), Dict{Symbol, Any}(:output1 => "Buzz")), IOExample(Dict{Symbol, Any}(:input1 => 11), Dict{Symbol, Any}(:output1 => "11")), IOExample(Dict{Symbol, Any}(:input1 => 12), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 13), Dict{Symbol, Any}(:output1 => "13")), IOExample(Dict{Symbol, Any}(:input1 => 14), Dict{Symbol, Any}(:output1 => "14")), IOExample(Dict{Symbol, Any}(:input1 => 15), Dict{Symbol, Any}(:output1 => "FizzBuzz")), IOExample(Dict{Symbol, Any}(:input1 => 16), Dict{Symbol, Any}(:output1 => "16")), IOExample(Dict{Symbol, Any}(:input1 => 17), Dict{Symbol, Any}(:output1 => "17")), IOExample(Dict{Symbol, Any}(:input1 => 18), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 19), Dict{Symbol, Any}(:output1 => "19")), IOExample(Dict{Symbol, Any}(:input1 => 20), Dict{Symbol, Any}(:output1 => "Buzz")), IOExample(Dict{Symbol, Any}(:input1 => 49995), Dict{Symbol, Any}(:output1 => "FizzBuzz")), IOExample(Dict{Symbol, Any}(:input1 => 49998), Dict{Symbol, Any}(:output1 => "Fizz")), IOExample(Dict{Symbol, Any}(:input1 => 49999), Dict{Symbol, Any}(:output1 => "49999")), IOExample(Dict{Symbol, Any}(:input1 => 50000), Dict{Symbol, Any}(:output1 => "Buzz"))]) problem_fuel_cost = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[6]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[7]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[8]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[9]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[10]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[11]), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => Any[12]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[13]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[14]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[15]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[16]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[17]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[9998]), Dict{Symbol, Any}(:output1 => 3330)), IOExample(Dict{Symbol, Any}(:input1 => Any[9999]), Dict{Symbol, Any}(:output1 => 3331)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000]), Dict{Symbol, Any}(:output1 => 3331)), IOExample(Dict{Symbol, Any}(:input1 => Any[6, 6]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 14]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 15]), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => Any[14, 9]), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => Any[15, 9]), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => Any[32, 32]), Dict{Symbol, Any}(:output1 => 16)), IOExample(Dict{Symbol, Any}(:input1 => Any[33, 33]), Dict{Symbol, Any}(:output1 => 18)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000, 9]), Dict{Symbol, Any}(:output1 => 3332)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 10000]), Dict{Symbol, Any}(:output1 => 3332)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000, 10000]), Dict{Symbol, Any}(:output1 => 6662)), IOExample(Dict{Symbol, Any}(:input1 => Any[6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]), Dict{Symbol, Any}(:output1 => 20)), IOExample(Dict{Symbol, Any}(:input1 => Any[10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]), Dict{Symbol, Any}(:output1 => 20)), IOExample(Dict{Symbol, Any}(:input1 => Any[11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11]), Dict{Symbol, Any}(:output1 => 20)), IOExample(Dict{Symbol, Any}(:input1 => Any[12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12]), Dict{Symbol, Any}(:output1 => 40)), IOExample(Dict{Symbol, Any}(:input1 => Any[13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13]), Dict{Symbol, Any}(:output1 => 40)), IOExample(Dict{Symbol, Any}(:input1 => Any[9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998, 9998]), Dict{Symbol, Any}(:output1 => 66600)), IOExample(Dict{Symbol, Any}(:input1 => Any[9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999]), Dict{Symbol, Any}(:output1 => 66620)), IOExample(Dict{Symbol, Any}(:input1 => Any[10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000]), Dict{Symbol, Any}(:output1 => 66620)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]), Dict{Symbol, Any}(:output1 => 15))]) -problem_gcd = Problem([IOExample(Dict{Symbol, Any}(:input1 => 1, :input2 => 1), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 4, :input2 => 400000), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => 54, :input2 => 24), Dict{Symbol, Any}(:output1 => 6)), IOExample(Dict{Symbol, Any}(:input1 => 4200, :input2 => 3528), Dict{Symbol, Any}(:output1 => 168)), IOExample(Dict{Symbol, Any}(:input1 => 820000, :input2 => 63550), Dict{Symbol, Any}(:output1 => 2050)), IOExample(Dict{Symbol, Any}(:input1 => 123456, :input2 => 654321), Dict{Symbol, Any}(:output1 => 3))]) +problem_gcd = Problem([IOExample(Dict{Symbol, Any}(:input1 => 1, :input2 => 1), Dict{Symbol, Any}(:output1 => 1)), IOExample(Dict{Symbol, Any}(:input1 => 4, :input2 => 400000), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => 54, :input2 => 24), Dict{Symbol, Any}(:output1 => 6)), IOExample(Dict{Symbol, Any}(:input1 => 4200, :input2 => 3528), Dict{Symbol, Any}(:output1 => 168)), IOExample(Dict{Symbol, Any}(:input1 => 820000, :input2 => 63550), Dict{Symbol, Any}(:output1 => 2050)), IOExample(Dict{Symbol, Any}(:input1 => 123456, :input2 => 654321), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => 13, :input2 => 39), Dict{Symbol, Any}(:output1 => 13)), IOExample(Dict{Symbol, Any}(:input1 => 123321, :input2 => 12321), Dict{Symbol, Any}(:output1 => 111)), IOExample(Dict{Symbol, Any}(:input1 => 47091, :input2 => 36561), Dict{Symbol, Any}(:output1 => 3)), IOExample(Dict{Symbol, Any}(:input1 => 1701, :input2 => 3768), Dict{Symbol, Any}(:output1 => 3))]) problem_indices_of_substring = Problem([IOExample(Dict{Symbol, Any}(:input1 => "a", :input2 => "5"), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => "!", :input2 => "!"), Dict{Symbol, Any}(:output1 => Any[0])), IOExample(Dict{Symbol, Any}(:input1 => "r", :input2 => "nm,xcnwqnd@#\$fwkdjn3"), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => "hi", :input2 => "hihihihihihihihihihi"), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => "############", :input2 => "#"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])), IOExample(Dict{Symbol, Any}(:input1 => "GGGGGGGGGGGGGGGGGGGG", :input2 => "G"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19])), IOExample(Dict{Symbol, Any}(:input1 => "\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$\$", :input2 => "\$\$"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18])), IOExample(Dict{Symbol, Any}(:input1 => "33333333333333333333", :input2 => "333"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17])), IOExample(Dict{Symbol, Any}(:input1 => "hahahahahahahahahaha", :input2 => "haha"), Dict{Symbol, Any}(:output1 => Any[0, 2, 4, 6, 8, 10, 12, 14, 16])), IOExample(Dict{Symbol, Any}(:input1 => "GCTGCTGCTGCTGCTGCTGC", :input2 => "GCTGC"), Dict{Symbol, Any}(:output1 => Any[0, 3, 6, 9, 12, 15])), IOExample(Dict{Symbol, Any}(:input1 => "bbbbbbb(bb#bbbbbbbb", :input2 => "bbb"), Dict{Symbol, Any}(:output1 => Any[0, 1, 2, 3, 4, 11, 12, 13, 14, 15, 16])), IOExample(Dict{Symbol, Any}(:input1 => "fa la la la la, la ", :input2 => "la"), Dict{Symbol, Any}(:output1 => Any[3, 6, 9, 12, 16])), IOExample(Dict{Symbol, Any}(:input1 => "start and and with s", :input2 => "s"), Dict{Symbol, Any}(:output1 => Any[0, 19])), IOExample(Dict{Symbol, Any}(:input1 => "tomato", :input2 => "tom"), Dict{Symbol, Any}(:output1 => Any[0])), IOExample(Dict{Symbol, Any}(:input1 => "tomatotomatotomato", :input2 => "tom"), Dict{Symbol, Any}(:output1 => Any[0, 6, 12])), IOExample(Dict{Symbol, Any}(:input1 => "tomatotomatotomato", :input2 => "to"), Dict{Symbol, Any}(:output1 => Any[0, 4, 6, 10, 12, 16])), IOExample(Dict{Symbol, Any}(:input1 => "will be zero", :input2 => "this will be zero"), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => "APPEAR twice APPEAR", :input2 => "APPEAR"), Dict{Symbol, Any}(:output1 => Any[0, 13])), IOExample(Dict{Symbol, Any}(:input1 => "a few ending <3<3<3", :input2 => "<3"), Dict{Symbol, Any}(:output1 => Any[13, 15, 17])), IOExample(Dict{Symbol, Any}(:input1 => "middle of this one", :input2 => "of"), Dict{Symbol, Any}(:output1 => Any[7]))]) problem_leaders = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[]), Dict{Symbol, Any}(:output1 => Any[])), IOExample(Dict{Symbol, Any}(:input1 => Any[0]), Dict{Symbol, Any}(:output1 => Any[0])), IOExample(Dict{Symbol, Any}(:input1 => Any[451]), Dict{Symbol, Any}(:output1 => Any[451])), IOExample(Dict{Symbol, Any}(:input1 => Any[1000, 0]), Dict{Symbol, Any}(:output1 => Any[1000, 0])), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 1000]), Dict{Symbol, Any}(:output1 => Any[1000])), IOExample(Dict{Symbol, Any}(:input1 => Any[20, 137, 20]), Dict{Symbol, Any}(:output1 => Any[137, 20])), IOExample(Dict{Symbol, Any}(:input1 => Any[47, 87, 43, 44]), Dict{Symbol, Any}(:output1 => Any[87, 44])), IOExample(Dict{Symbol, Any}(:input1 => Any[5, 5, 5, 5, 5, 5, 5]), Dict{Symbol, Any}(:output1 => Any[5, 5, 5, 5, 5, 5, 5])), IOExample(Dict{Symbol, Any}(:input1 => Any[10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]), Dict{Symbol, Any}(:output1 => Any[10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0])), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), Dict{Symbol, Any}(:output1 => Any[10]))]) problem_luhn = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3]), Dict{Symbol, Any}(:output1 => 80)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9]), Dict{Symbol, Any}(:output1 => 144)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 0)), IOExample(Dict{Symbol, Any}(:input1 => Any[5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]), Dict{Symbol, Any}(:output1 => 48)), IOExample(Dict{Symbol, Any}(:input1 => Any[4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]), Dict{Symbol, Any}(:output1 => 96)), IOExample(Dict{Symbol, Any}(:input1 => Any[1, 0, 2, 0, 4, 3, 2, 1, 0, 4, 1, 2, 3, 4, 2, 1]), Dict{Symbol, Any}(:output1 => 45)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 2)), IOExample(Dict{Symbol, Any}(:input1 => Any[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 4)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0]), Dict{Symbol, Any}(:output1 => 6)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 8)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 6)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 5)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 7)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 9)), IOExample(Dict{Symbol, Any}(:input1 => Any[8, 0, 0, 0, 0, 6, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 22)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 14)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5, 0, 5]), Dict{Symbol, Any}(:output1 => 40)), IOExample(Dict{Symbol, Any}(:input1 => Any[9, 9, 8, 7, 6, 6, 7, 8, 9, 9, 8, 7, 6, 5, 5, 6]), Dict{Symbol, Any}(:output1 => 101)), IOExample(Dict{Symbol, Any}(:input1 => Any[0, 0, 0, 0, 0, 7, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0]), Dict{Symbol, Any}(:output1 => 10))]) @@ -21,5 +21,5 @@ problem_solve_boolean = Problem([IOExample(Dict{Symbol, Any}(:input1 => "t"), Di problem_spin_words = Problem([IOExample(Dict{Symbol, Any}(:input1 => ""), Dict{Symbol, Any}(:output1 => "")), IOExample(Dict{Symbol, Any}(:input1 => "a"), Dict{Symbol, Any}(:output1 => "a")), IOExample(Dict{Symbol, Any}(:input1 => "this is a test"), Dict{Symbol, Any}(:output1 => "this is a test")), IOExample(Dict{Symbol, Any}(:input1 => "this is another test"), Dict{Symbol, Any}(:output1 => "this is rehtona test")), IOExample(Dict{Symbol, Any}(:input1 => "hi"), Dict{Symbol, Any}(:output1 => "hi")), IOExample(Dict{Symbol, Any}(:input1 => "cat"), Dict{Symbol, Any}(:output1 => "cat")), IOExample(Dict{Symbol, Any}(:input1 => "walk"), Dict{Symbol, Any}(:output1 => "walk")), IOExample(Dict{Symbol, Any}(:input1 => "jazz"), Dict{Symbol, Any}(:output1 => "jazz")), IOExample(Dict{Symbol, Any}(:input1 => "llama"), Dict{Symbol, Any}(:output1 => "amall")), IOExample(Dict{Symbol, Any}(:input1 => "heart"), Dict{Symbol, Any}(:output1 => "traeh")), IOExample(Dict{Symbol, Any}(:input1 => "pantry"), Dict{Symbol, Any}(:output1 => "yrtnap")), IOExample(Dict{Symbol, Any}(:input1 => "helpful"), Dict{Symbol, Any}(:output1 => "lufpleh")), IOExample(Dict{Symbol, Any}(:input1 => "disrespectful"), Dict{Symbol, Any}(:output1 => "luftcepsersid")), IOExample(Dict{Symbol, Any}(:input1 => "stop spinning these"), Dict{Symbol, Any}(:output1 => "stop gninnips eseht")), IOExample(Dict{Symbol, Any}(:input1 => "couple longer words"), Dict{Symbol, Any}(:output1 => "elpuoc regnol sdrow")), IOExample(Dict{Symbol, Any}(:input1 => "oneloongworrrrrrrrrd"), Dict{Symbol, Any}(:output1 => "drrrrrrrrrowgnooleno")), IOExample(Dict{Symbol, Any}(:input1 => "a b c d e f g h i j"), Dict{Symbol, Any}(:output1 => "a b c d e f g h i j")), IOExample(Dict{Symbol, Any}(:input1 => "ab cd ef gh ij kl mn"), Dict{Symbol, Any}(:output1 => "ab cd ef gh ij kl mn")), IOExample(Dict{Symbol, Any}(:input1 => "abc def gef hij klm"), Dict{Symbol, Any}(:output1 => "abc def gef hij klm")), IOExample(Dict{Symbol, Any}(:input1 => "word less than five"), Dict{Symbol, Any}(:output1 => "word less than five")), IOExample(Dict{Symbol, Any}(:input1 => "abcde fghij klmno"), Dict{Symbol, Any}(:output1 => "edcba jihgf onmlk")), IOExample(Dict{Symbol, Any}(:input1 => "abcdef ghijkl mnopqr"), Dict{Symbol, Any}(:output1 => "fedcba lkjihg rqponm")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefg hijklmn"), Dict{Symbol, Any}(:output1 => "gfedcba nmlkjih")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefgh ijklmnop"), Dict{Symbol, Any}(:output1 => "hgfedcba ponmlkji")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefghi jklmnopqrs"), Dict{Symbol, Any}(:output1 => "ihgfedcba srqponmlkj")), IOExample(Dict{Symbol, Any}(:input1 => "on pineapple island"), Dict{Symbol, Any}(:output1 => "on elppaenip dnalsi")), IOExample(Dict{Symbol, Any}(:input1 => "maybe this isgood"), Dict{Symbol, Any}(:output1 => "ebyam this doogsi")), IOExample(Dict{Symbol, Any}(:input1 => "racecar palindrome"), Dict{Symbol, Any}(:output1 => "racecar emordnilap")), IOExample(Dict{Symbol, Any}(:input1 => "ella is a short pali"), Dict{Symbol, Any}(:output1 => "ella is a trohs pali")), IOExample(Dict{Symbol, Any}(:input1 => "science hi"), Dict{Symbol, Any}(:output1 => "ecneics hi"))]) problem_square_digits = Problem([IOExample(Dict{Symbol, Any}(:input1 => 0), Dict{Symbol, Any}(:output1 => "0")), IOExample(Dict{Symbol, Any}(:input1 => 1), Dict{Symbol, Any}(:output1 => "1")), IOExample(Dict{Symbol, Any}(:input1 => 2), Dict{Symbol, Any}(:output1 => "4")), IOExample(Dict{Symbol, Any}(:input1 => 3), Dict{Symbol, Any}(:output1 => "9")), IOExample(Dict{Symbol, Any}(:input1 => 4), Dict{Symbol, Any}(:output1 => "16")), IOExample(Dict{Symbol, Any}(:input1 => 5), Dict{Symbol, Any}(:output1 => "25")), IOExample(Dict{Symbol, Any}(:input1 => 7), Dict{Symbol, Any}(:output1 => "49")), IOExample(Dict{Symbol, Any}(:input1 => 9), Dict{Symbol, Any}(:output1 => "81")), IOExample(Dict{Symbol, Any}(:input1 => 10), Dict{Symbol, Any}(:output1 => "10")), IOExample(Dict{Symbol, Any}(:input1 => 12), Dict{Symbol, Any}(:output1 => "14")), IOExample(Dict{Symbol, Any}(:input1 => 16), Dict{Symbol, Any}(:output1 => "136")), IOExample(Dict{Symbol, Any}(:input1 => 24), Dict{Symbol, Any}(:output1 => "416")), IOExample(Dict{Symbol, Any}(:input1 => 35), Dict{Symbol, Any}(:output1 => "925")), IOExample(Dict{Symbol, Any}(:input1 => 46), Dict{Symbol, Any}(:output1 => "1636")), IOExample(Dict{Symbol, Any}(:input1 => 57), Dict{Symbol, Any}(:output1 => "2549")), IOExample(Dict{Symbol, Any}(:input1 => 68), Dict{Symbol, Any}(:output1 => "3664")), IOExample(Dict{Symbol, Any}(:input1 => 79), Dict{Symbol, Any}(:output1 => "4981")), IOExample(Dict{Symbol, Any}(:input1 => 80), Dict{Symbol, Any}(:output1 => "640")), IOExample(Dict{Symbol, Any}(:input1 => 92), Dict{Symbol, Any}(:output1 => "814")), IOExample(Dict{Symbol, Any}(:input1 => 98), Dict{Symbol, Any}(:output1 => "8164")), IOExample(Dict{Symbol, Any}(:input1 => 100), Dict{Symbol, Any}(:output1 => "100")), IOExample(Dict{Symbol, Any}(:input1 => 185), Dict{Symbol, Any}(:output1 => "16425")), IOExample(Dict{Symbol, Any}(:input1 => 231), Dict{Symbol, Any}(:output1 => "491")), IOExample(Dict{Symbol, Any}(:input1 => 372), Dict{Symbol, Any}(:output1 => "9494")), IOExample(Dict{Symbol, Any}(:input1 => 408), Dict{Symbol, Any}(:output1 => "16064")), IOExample(Dict{Symbol, Any}(:input1 => 794), Dict{Symbol, Any}(:output1 => "498116")), IOExample(Dict{Symbol, Any}(:input1 => 321012), Dict{Symbol, Any}(:output1 => "941014")), IOExample(Dict{Symbol, Any}(:input1 => 987654), Dict{Symbol, Any}(:output1 => "816449362516")), IOExample(Dict{Symbol, Any}(:input1 => 999999), Dict{Symbol, Any}(:output1 => "818181818181")), IOExample(Dict{Symbol, Any}(:input1 => 1000000), Dict{Symbol, Any}(:output1 => "1000000"))]) problem_substitution_cipher = Problem([IOExample(Dict{Symbol, Any}(:input1 => "", :input2 => "", :input3 => ""), Dict{Symbol, Any}(:output1 => "")), IOExample(Dict{Symbol, Any}(:input1 => "a", :input2 => "a", :input3 => "a"), Dict{Symbol, Any}(:output1 => "a")), IOExample(Dict{Symbol, Any}(:input1 => "j", :input2 => "h", :input3 => "j"), Dict{Symbol, Any}(:output1 => "h")), IOExample(Dict{Symbol, Any}(:input1 => "a", :input2 => "z", :input3 => "a"), Dict{Symbol, Any}(:output1 => "z")), IOExample(Dict{Symbol, Any}(:input1 => "e", :input2 => "l", :input3 => "eeeeeeeeee"), Dict{Symbol, Any}(:output1 => "llllllllll")), IOExample(Dict{Symbol, Any}(:input1 => "h", :input2 => "d", :input3 => "hhhhhhhhhhhhhhhhhhhh"), Dict{Symbol, Any}(:output1 => "dddddddddddddddddddd")), IOExample(Dict{Symbol, Any}(:input1 => "o", :input2 => "z", :input3 => "oooooooooooooooooooooooooo"), Dict{Symbol, Any}(:output1 => "zzzzzzzzzzzzzzzzzzzzzzzzzz")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefghijklmnopqrstuvwxyz", :input2 => "zyxwvutsrqponmlkjihgfedcba", :input3 => "bvafvuqgjkkbeccipwdfqttgzl"), Dict{Symbol, Any}(:output1 => "yezuefjtqppyvxxrkdwujggtao")), IOExample(Dict{Symbol, Any}(:input1 => "abcdefghijklmnopqrstuvwxyz", :input2 => "cdqutzayxshgfenjowrkvmpbil", :input3 => "thequickbrownfxjmpsvlazydg"), Dict{Symbol, Any}(:output1 => "kytovxqhdwnpezbsfjrmgcliua")), IOExample(Dict{Symbol, Any}(:input1 => "otghvwmkclidzryxsfqeapnjbu", :input2 => "alpebhxmnrcyiosvtgzjwuqdfk", :input3 => "aaabbbccc"), Dict{Symbol, Any}(:output1 => "wwwfffnnn"))]) -problem_twitter = Problem([IOExample(Dict{Symbol, Any}(:input1 => ""), Dict{Symbol, Any}(:output1 => "You didn't type anything")), IOExample(Dict{Symbol, Any}(:input1 => "1"), Dict{Symbol, Any}(:output1 => "Your tweet has 1 characters")), IOExample(Dict{Symbol, Any}(:input1 => "max length tweet that just contains letters and spaces even SOME CAPITAL LETTERS just to MAKE it INTERESTING now repeeeeeeeeeEEEEEEEeeeat it"), Dict{Symbol, Any}(:output1 => "Your tweet has 140 characters")), IOExample(Dict{Symbol, Any}(:input1 => "40172875*&(&(%^^*!@&#()!@&^(*\$787031264123984721-43214876*%^#!(@^\$_!@^%#\$(!#@%\$(01234~~``)"), Dict{Symbol, Any}(:output1 => "Your tweet has 90 characters")), IOExample(Dict{Symbol, Any}(:input1 => "Tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooolong1"), Dict{Symbol, Any}(:output1 => "Too many characters")), IOExample(Dict{Symbol, Any}(:input1 => "(@)&#N)&#@!M#&17t8023n217830123bn6 BN23780BC3879N01nc3n473N962n9768062BC3718N396b21v8365n9072B638705b097B6*&b%&%b(*B5*&%b7%(*vb&V8%v&(85V80%0857(%v97%(*&%v87c%&*c()0*^c%08v^mN098)vf%9P8V6TfB97b99870)"), Dict{Symbol, Any}(:output1 => "Too many characters"))]) -problem_vector_distance = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[-100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0], :input2 => Any[100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0]), Dict{Symbol, Any}(:output1 => 894.4271909999159)), IOExample(Dict{Symbol, Any}(:input1 => Any[5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32], :input2 => Any[5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32]), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input1 => Any[42.91283], :input2 => Any[-22.134]), Dict{Symbol, Any}(:output1 => 65.04683)), IOExample(Dict{Symbol, Any}(:input1 => Any[1.5, 2.87, 3.3324, 4.654, 5.123, 6.867, 7.5324, 8.534, 9.4132, 10.43], :input2 => Any[-1.534, -2.543, -3.423, -4.13427, -5.714, -6.713, -7.4328, -8.43, -9.73, -10.752]), Dict{Symbol, Any}(:output1 => 42.309638973086265)), IOExample(Dict{Symbol, Any}(:input1 => Any[0.4378, 0.634, 0.1234, 0.764, 0.243], :input2 => Any[-0.254, -0.1223, -0.19582, -0.8971, -0.8743]), Dict{Symbol, Any}(:output1 => 2.2715833329200144))]) \ No newline at end of file +problem_twitter = Problem([IOExample(Dict{Symbol, Any}(:input1 => ""), Dict{Symbol, Any}(:output1 => "You didn't type anything")), IOExample(Dict{Symbol, Any}(:input1 => "1"), Dict{Symbol, Any}(:output1 => "Your tweet has 1 characters")), IOExample(Dict{Symbol, Any}(:input1 => "max length tweet that just contains letters and spaces even SOME CAPITAL LETTERS just to MAKE it INTERESTING now repeeeeeeeeeEEEEEEEeeeat it"), Dict{Symbol, Any}(:output1 => "Your tweet has 140 characters")), IOExample(Dict{Symbol, Any}(:input1 => "40172875*&(&(%^^*!@&#()!@&^(*\$787031264123984721-43214876*%^#!(@^\$_!@^%#\$(!#@%\$(01234~~``)"), Dict{Symbol, Any}(:output1 => "Your tweet has 90 characters")), IOExample(Dict{Symbol, Any}(:input1 => "Tooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooolong1"), Dict{Symbol, Any}(:output1 => "Too many characters")), IOExample(Dict{Symbol, Any}(:input1 => "(@)&#N)&#@!M#&17t8023n217830123bn6 BN23780BC3879N01nc3n473N962n9768062BC3718N396b21v8365n9072B638705b097B6*&b%&%b(*B5*&%b7%(*vb&V8%v&(85V80%0857(%v97%(*&%v87c%&*c()0*^c%08v^mN098)vf%9P8V6TfB97b99870)"), Dict{Symbol, Any}(:output1 => "Too many characters")), IOExample(Dict{Symbol, Any}(:input1 => "{PY0\"o)U0bXwz"), Dict{Symbol, Any}(:output1 => "Your tweet has 13 characters")), IOExample(Dict{Symbol, Any}(:input1 => "&((-|1c&Q\\+r*P8}h(HT,<(_Gq=[z!KaOATI[f;n[6I&17i~1UrPGX-DVG%6Z7d9:w\"Z3VD,.v6+u2rn@Gq&jIM{i{H2D\$Nqo@6Y_p~W\$\"c41<(=zC'Y"), Dict{Symbol, Any}(:output1 => "Your tweet has 128 characters")), IOExample(Dict{Symbol, Any}(:input1 => "Ob0MDUD#.\$&!s=g\$#/FX_w&PSsQf0#\"Bow"), Dict{Symbol, Any}(:output1 => "Your tweet has 34 characters")), IOExample(Dict{Symbol, Any}(:input1 => "U7s"), Dict{Symbol, Any}(:output1 => "Your tweet has 3 characters")), IOExample(Dict{Symbol, Any}(:input1 => ";;H0laFtpCw/3W+6z`tA:?!GY#jH'Syp*Kyz}:ky(&b[fxM|2@73cR6=U1Xk-ay@oF<\$oHx54daCHTGKo;iM|9_^a=+Zt2U:PDCB5\$#uPW2vdYN].Af2RaIWcf)0*@n1!]%>j@]G/&D} "Too many characters"))]) +problem_vector_distance = Problem([IOExample(Dict{Symbol, Any}(:input1 => Any[-100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0], :input2 => Any[100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0]), Dict{Symbol, Any}(:output1 => 894.4271909999159)), IOExample(Dict{Symbol, Any}(:input1 => Any[5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32], :input2 => Any[5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32, 5.32]), Dict{Symbol, Any}(:output1 => 0.0)), IOExample(Dict{Symbol, Any}(:input1 => Any[42.91283], :input2 => Any[-22.134]), Dict{Symbol, Any}(:output1 => 65.04683)), IOExample(Dict{Symbol, Any}(:input1 => Any[1.5, 2.87, 3.3324, 4.654, 5.123, 6.867, 7.5324, 8.534, 9.4132, 10.43], :input2 => Any[-1.534, -2.543, -3.423, -4.13427, -5.714, -6.713, -7.4328, -8.43, -9.73, -10.752]), Dict{Symbol, Any}(:output1 => 42.309638973086265)), IOExample(Dict{Symbol, Any}(:input1 => Any[0.4378, 0.634, 0.1234, 0.764, 0.243], :input2 => Any[-0.254, -0.1223, -0.19582, -0.8971, -0.8743]), Dict{Symbol, Any}(:output1 => 2.2715833329200144)), IOExample(Dict{Symbol, Any}(:input1 => Any[-41.83223695854199], :input2 => Any[81.43459569023676]), Dict{Symbol, Any}(:output1 => 123.26683264877875)), IOExample(Dict{Symbol, Any}(:input1 => Any[-45.57661301376075], :input2 => Any[59.52672225595944]), Dict{Symbol, Any}(:output1 => 105.10333526972019)), IOExample(Dict{Symbol, Any}(:input1 => Any[-19.37591081936432], :input2 => Any[92.46338927044178]), Dict{Symbol, Any}(:output1 => 111.8393000898061)), IOExample(Dict{Symbol, Any}(:input1 => Any[57.24745475398481], :input2 => Any[-37.96836543530524]), Dict{Symbol, Any}(:output1 => 95.21582018929004)), IOExample(Dict{Symbol, Any}(:input1 => Any[99.86899705927289], :input2 => Any[-67.62510731730127]), Dict{Symbol, Any}(:output1 => 167.49410437657417)), IOExample(Dict{Symbol, Any}(:input1 => Any[59.45605485065619], :input2 => Any[-93.7540353776902]), Dict{Symbol, Any}(:output1 => 153.2100902283464))]) \ No newline at end of file diff --git a/src/data/PSB2_2021/example_using_the_benchmark.jl b/src/data/PSB2_2021/example_using_the_benchmark.jl index ab438291..f0be3212 100644 --- a/src/data/PSB2_2021/example_using_the_benchmark.jl +++ b/src/data/PSB2_2021/example_using_the_benchmark.jl @@ -22,6 +22,6 @@ println("Found program rulenode: ", program) println("Found program: ", HerbGrammar.rulenode2expr(program, grammar)) # Now, we can evaluate a program -input1 = [5,2,3,92] +input1 = [5,-2,-4,92] println("Found program outputs: ", eval(HerbGrammar.rulenode2expr(program, grammar))) println("Provided example program outputs: ", program_basement(input1)) \ No newline at end of file diff --git a/src/data/PSB2_2021/grammar.jl b/src/data/PSB2_2021/grammar.jl index 01318883..241d4383 100644 --- a/src/data/PSB2_2021/grammar.jl +++ b/src/data/PSB2_2021/grammar.jl @@ -27,6 +27,21 @@ function get_grammar_basement(;minimal=true, seed=nothing) return g end +function get_grammar_bouncing_balls(;minimal=true) + @warn "Not yet implemented grammar bouncing balls" + return ContextSensitiveGrammar() +end + +function get_grammar_bowling(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar bowling" + return ContextSensitiveGrammar() +end + +function get_grammar_camel_case(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar camel case" + return ContextSensitiveGrammar() +end + function get_grammar_coinsums(;minimal=true) # Coin sums does not have a random integer grammar_coin_sums = merge_grammar([ @@ -38,6 +53,21 @@ function get_grammar_coinsums(;minimal=true) return g end +function get_grammar_cut_vector(;minimal=true) + @warn "Not yet implemented grammar cut vector" + return ContextSensitiveGrammar() +end + +function get_grammar_dice_game(;minimal=true) + @warn "Not yet implemented grammar dice game" + return ContextSensitiveGrammar() +end + +function get_grammar_find_pair(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar find pair" + return ContextSensitiveGrammar() +end + function get_grammar_fizz_buzz(;minimal=true) # Fizz buzz does not have a random character grammar_fizz_buzz = merge_grammar([ @@ -94,8 +124,98 @@ function get_grammar_gcd(;minimal=true, seed=nothing) return g end +function get_grammar_indices_of_substring(;minimal=true) + @warn "Not yet implemented grammar indices of substring" + return ContextSensitiveGrammar() +end + +function get_grammar_leaders(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar leaders" + return ContextSensitiveGrammar() +end + +function get_grammar_luhn(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar luhn" + return ContextSensitiveGrammar() +end + +function get_grammar_mastermind(;minimal=true) + @warn "Not yet implemented grammar mastermind" + return ContextSensitiveGrammar() +end + +function get_grammar_middle_character(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar middle character" + return ContextSensitiveGrammar() +end + +function get_grammar_paired_digits(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar paired digits" + return ContextSensitiveGrammar() +end + +function get_grammar_shopping_list(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar shopping list" + return ContextSensitiveGrammar() +end + +function get_grammar_snow_day(;minimal=true) + @warn "Not yet implemented grammar snow day" + return ContextSensitiveGrammar() +end + +function get_grammar_solve_boolean(;minimal=true) + @warn "Not yet implemented grammar solve boolean" + return ContextSensitiveGrammar() +end + +function get_grammar_spin_words(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar spin words" + return ContextSensitiveGrammar() +end + +function get_grammar_square_digits(;minimal=true, seed=nothing) + @warn "Not yet implemented grammar square digits" + return ContextSensitiveGrammar() +end + +function get_grammar_substitution_cipher(;minimal=true) + @warn "Not yet implemented grammar substitution cipher" + return ContextSensitiveGrammar() +end + +function get_grammar_twitter(;minimal=true) + @warn "Not yet implemented grammar twitter" + return ContextSensitiveGrammar() +end + +function get_grammar_vector_distance(;minimal=true) + @warn "Not yet implemented grammar vector distance" + return ContextSensitiveGrammar() +end + grammar_basement = get_grammar_basement(seed=42) +grammar_bouncing_balls = get_grammar_bouncing_balls() +grammar_bowling = get_grammar_bowling(seed=42) +grammar_camel_case = get_grammar_camel_case(seed=42) grammar_coin_sums = get_grammar_coinsums() +grammar_cut_vector = get_grammar_cut_vector() +grammar_dice_game = get_grammar_dice_game() +grammar_find_pair = get_grammar_find_pair(seed=42) grammar_fizz_buzz = get_grammar_fizz_buzz() grammar_fuel_cost = get_grammar_fuelcost(seed=42) -grammar_gcd = get_grammar_gcd() \ No newline at end of file +grammar_gcd = get_grammar_gcd() +grammar_indices_of_substring = get_grammar_indices_of_substring() +grammar_leaders = get_grammar_leaders(seed=42) +grammar_luhn = get_grammar_luhn(seed=42) +grammar_mastermind = get_grammar_mastermind() +grammar_middle_character = get_grammar_middle_character(seed=42) +grammar_paired_digits = get_grammar_paired_digits(seed=42) +grammar_shopping_list = get_grammar_shopping_list(seed=42) +grammar_snow_day = get_grammar_snow_day() +grammar_solve_boolean = get_grammar_solve_boolean() +grammar_spin_words = get_grammar_spin_words(seed=42) +grammar_square_digits = get_grammar_square_digits(seed=42) +grammar_substitution_cipher = get_grammar_substitution_cipher() +grammar_twitter = get_grammar_twitter() +grammar_vector_distance = get_grammar_vector_distance() \ No newline at end of file diff --git a/src/data/PSB2_2021/program_examples.jl b/src/data/PSB2_2021/program_examples.jl index 34024463..0b7c39e2 100644 --- a/src/data/PSB2_2021/program_examples.jl +++ b/src/data/PSB2_2021/program_examples.jl @@ -1,11 +1,15 @@ function program_basement(input1) # Takes list of integers and returns 1 integer Dict(:output1 => - let state = Dict(:i => 1) - while get(state, :i, "key not found") < length(input1) && sum(getindex(input1, 1:get(state, :i, "key not found")+1)) < 0 - push!(state, :i => get!(state, :i, "key not found") + 1) + if getindex(input1, 1) < 0 + 0 + else + let state = Dict(:i => 0) + while get(state, :i, "key not found") < length(input1) && sum(getindex(input1, 1:get(state, :i, "key not found"))) < 0 + push!(state, :i => get(state, :i, "key not found")) + end + get(state, :i, "key not found") end - get(state, :i, "key not found") end ) end @@ -50,7 +54,7 @@ function program_gcd(input1, input2) while state[:y] > 0 merge!(state, Dict(:x => state[:y], :y => state[:x] % state[:y])) end - get!(state, :x, "key not found") + get(state, :x, "key not found") end ) end diff --git a/src/data/PSB2_2021/psb2_primitives.jl b/src/data/PSB2_2021/psb2_primitives.jl index 920dff10..7de053b9 100644 --- a/src/data/PSB2_2021/psb2_primitives.jl +++ b/src/data/PSB2_2021/psb2_primitives.jl @@ -1,5 +1,5 @@ """ -Function for replacing a character in a string. +Function for replacing the character at index 'int' with new character 'char'. """ function replace_in_string(str, int, char) arr = collect(str) @@ -12,14 +12,14 @@ end Function to merge different grammars. """ function merge_grammar(gs::Vector{ContextSensitiveGrammar}) - new_grammar = @csgrammar begin end + grammar_to_construct = ContextSensitiveGrammar() for g in gs for i in eachindex(g.rules) ex = :($(g.types[i]) = $(g.rules[i])) - add_rule!(new_grammar, ex) + add_rule!(grammar_to_construct, ex) end end - return new_grammar + return grammar_to_construct end @@ -32,7 +32,7 @@ function command_while(condition::Any, body::Any, max_steps::Int=1000) counter = max_steps; res = nothing - while condition && counter > 0 + while eval(condition) && counter > 0 res = eval(body) counter -= 1 end diff --git a/src/utils/benchmarks_io.jl b/src/utils/benchmarks_io.jl index b29534c9..869c8904 100644 --- a/src/utils/benchmarks_io.jl +++ b/src/utils/benchmarks_io.jl @@ -66,7 +66,7 @@ end Parses a single problem file given a line parser and writes the Herb'ed problem to the output_path. """ -function parse_to_julia(path::String, filename::String, line_parser::Function, prefix::String="")::Problem +function parse_to_julia(path::String, filename::String, line_parser::Function, prefix::String="") problem = parse_file(joinpath(path, filename), line_parser) write_problem(joinpath(path, "$(prefix)data.jl"), problem, prefix) end diff --git a/test/runtests.jl b/test/runtests.jl index 919e26a9..79947d78 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,4 +10,5 @@ using Test include("test_pixels_2020.jl") include("test_abstract_reasoning_2019.jl") include("test_problem_fetcher.jl") + include("test_psb2_2021.jl") end diff --git a/test/test_general.jl b/test/test_general.jl index 73944d76..a605af26 100644 --- a/test/test_general.jl +++ b/test/test_general.jl @@ -2,7 +2,7 @@ input_rules(grammar::AbstractGrammar) = findall(rule -> occursin("_arg_", string @testset verbose=true "General tests on all submodules" begin modules = get_submodules(HerbBenchmarks) - for mod in modules + for mod in modules @testset verbose=true "Module $mod" begin problems = get_all_problems(mod) if length(problems) == 0 @@ -15,7 +15,8 @@ input_rules(grammar::AbstractGrammar) = findall(rule -> occursin("_arg_", string if mod ∉ [HerbBenchmarks.String_transformations_2020, HerbBenchmarks.Pixels_2020, - HerbBenchmarks.Robots_2020] + HerbBenchmarks.Robots_2020, + HerbBenchmarks.PSB2_2021] @testset verbose=true "Inputs align in grammar and problem" begin pairs = get_all_problem_grammar_pairs(mod) diff --git a/test/test_psb2_2021.jl b/test/test_psb2_2021.jl new file mode 100644 index 00000000..41d36fb5 --- /dev/null +++ b/test/test_psb2_2021.jl @@ -0,0 +1,97 @@ +using HerbBenchmarks.PSB2_2021 +using HerbGrammar + +@testset verbose = true "PSB2_2021" begin + @testset verbose = true "General tests" begin + domains = get_all_identifiers(PSB2_2021) + for dom in domains + @testset verbose = true "Domain $(dom)" begin + domain = replace(dom, "-" => "_") + problem = get_problem(PSB2_2021, domain) + grammar = get_grammar(PSB2_2021, domain) + @test typeof(problem) <: HerbSpecification.Problem + @test typeof(problem.spec[1]) == HerbSpecification.IOExample + @test length(problem.spec) >= 10 + end + end + end + + @testset verbose = true "Program examples" begin + result_basement = program_basement(problem_basement.spec[1].in[:input1]) + expected_basement = problem_basement.spec[1].out + @test result_basement == expected_basement + + result_coinsums = program_coin_sums(problem_coin_sums.spec[1].in[:input1]) + expected_coinsums = problem_coin_sums.spec[1].out + @test result_coinsums == expected_coinsums + + result_fizzbuzz = program_fizzbuzz(problem_fizz_buzz.spec[1].in[:input1]) + expected_fizzbuzz = problem_fizz_buzz.spec[1].out + @test result_fizzbuzz == expected_fizzbuzz + + result_fuelcost = program_fuel_cost(problem_fuel_cost.spec[1].in[:input1]) + expected_fuelcost = problem_fuel_cost.spec[1].out + @test result_fuelcost == expected_fuelcost + + result_gcd1 = program_gcd(problem_gcd.spec[1].in[:input1], problem_gcd.spec[1].in[:input2]) + expected_gcd1 = problem_gcd.spec[1].out + @test result_gcd1 == expected_gcd1 + + result_gcd2 = program_gcd2(problem_gcd.spec[1].in[:input1], problem_gcd.spec[1].in[:input2]) + expected_gcd2 = problem_gcd.spec[1].out + @test result_gcd2 == expected_gcd2 + end + + @testset verbose = true "Grammars" begin + @testset verbose = true "Grammar primitives" begin + @testset verbose = true "replace_in_string" begin + test_string = "test101" + check_string = "tesT101" + result = replace_in_string(test_string, 4, 'T') + @test result == check_string + end + + @testset verbose = true "merge_grammar" begin + test_g1 = @csgrammar begin + Rule = 1 + Rule = Rule + Rule + end + test_g2 = @csgrammar begin + Rule = 2 + end + test_g3 = @csgrammar begin + Rule = 3 + end + test_full_grammar = merge_grammar([test_g1, test_g2, test_g3]) + @test length(test_full_grammar.rules) == 4 + @test eval(rulenode2expr(RuleNode(2, [RuleNode(3), RuleNode(4)]), test_full_grammar)) == 5 + end + + @testset verbose = true "command_while" begin + # TODO Idea use the SymbolTable + @warn "TODO test the command_while function using the correct scope" + end + end + @testset verbose = true "Grammar boolean" begin + @warn "Not yet implemented boolean grammar functionality tests" + end + @testset verbose = true "Grammar integer" begin + @warn "Not yet implemented integer grammar functionality tests" + end + @testset verbose = true "Grammar state integer" begin + @warn "Not yet implemented state integer grammar functionality tests" + end + @testset verbose = true "Grammar list integer" begin + @warn "Not yet implemented list integer grammar functionality tests" + end + @testset verbose = true "Grammar float" begin + @warn "Not yet implemented float grammar functionality tests" + end + @testset verbose = true "Grammar string" begin + @warn "Not yet implemented string grammar functionality tests" + end + @testset verbose = true "Grammar char" begin + @warn "Not yet implemented char grammar functionality tests" + end + end +end From f3739b5cabb63da2c009ea4b886ad109972b0f92 Mon Sep 17 00:00:00 2001 From: Issa Hanou Date: Fri, 15 Nov 2024 16:38:33 +0100 Subject: [PATCH 6/6] update PSB2 dataset retriever --- src/data/PSB2_2021/PSB2_2021.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/data/PSB2_2021/PSB2_2021.jl b/src/data/PSB2_2021/PSB2_2021.jl index 9226a6d9..224ac1e5 100644 --- a/src/data/PSB2_2021/PSB2_2021.jl +++ b/src/data/PSB2_2021/PSB2_2021.jl @@ -8,7 +8,6 @@ include("retrieve_all_tasks.jl") include("grammar.jl") include("program_examples.jl") - export parse_line_json write_psb2_problems_to_file