Checkpoint recent developments

.github/workflows/CI.yml

@@ -19,6 +19,7 @@ jobs:
       matrix:
         version:
           - '1.8'
+          - '1'
           - 'nightly'
         os:
           - ubuntu-latest
@@ -33,3 +34,11 @@ jobs:
       - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: Herb-AI/HerbSearch.jl
+          files: lcov.info
+          fail_ci_if_error: false
+

Project.toml

@@ -1,28 +1,29 @@
 name = "HerbSearch"
 uuid = "3008d8e8-f9aa-438a-92ed-26e9c7b4829f"
 authors = ["Sebastijan Dumancic <[email protected]>", "Jaap de Jong <[email protected]>", "Nicolae Filat <[email protected]>", "Piotr Cichoń <[email protected]>", "Tilman Hinnerichs <[email protected]>"]
-version = "0.1.1"
+version = "0.2.0"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 HerbConstraints = "1fa96474-3206-4513-b4fa-23913f296dfc"
 HerbCore = "2b23ba43-8213-43cb-b5ea-38c12b45bd45"
-HerbData = "495a3ad3-8034-41b3-a087-aacf2fd71098"
 HerbGrammar = "4ef9e186-2fe5-4b24-8de7-9f7291f24af7"
 HerbInterpret = "5bbddadd-02c5-4713-84b8-97364418cca7"
+HerbSpecification = "6d54aada-062f-46d8-85cf-a1ceaf058a06"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+MLStyle = "d8e11817-5142-5d16-987a-aa16d5891078"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 
 [compat]
 DataStructures = "0.17,0.18"
-HerbConstraints = "0.1.0"
-HerbCore = "0.1.1"
-HerbData = "0.1.1"
-HerbGrammar = "0.1.0"
-HerbInterpret = "0.1.0"
-StatsBase = "0.34"
-julia = "1.8"
+HerbConstraints = "^0.1.0"
+HerbCore = "^0.2.0"
+HerbGrammar = "^0.2.0"
+HerbInterpret = "^0.1.1"
+HerbSpecification = "^0.1.0"
+StatsBase = "^0.34"
+julia = "^1.8"
 
 [extras]
 LegibleLambdas = "f1f30506-32fe-5131-bd72-7c197988f9e5"

README.md

@@ -1,14 +1,17 @@
-# Search.jl
-
+[![codecov](https://codecov.io/gh/Herb-AI/HerbSearch.jl/graph/badge.svg?token=VUK6MXLCU4)](https://codecov.io/gh/Herb-AI/HerbSearch.jl)
 [![Build Status](https://github.com/Herb-AI/HerbSearch.jl/actions/workflows/CI.yml/badge.svg?branch=master)](https://github.com/Herb-AI/HerbSearch.jl/actions/workflows/CI.yml?query=branch%3Amaster)
+[![Dev-Docs](https://img.shields.io/badge/docs-latest-blue.svg)](https://Herb-AI.github.io/Herb.jl/dev)
+
+# HerbSearch.jl
 
 This package contains search procedure implementations for the Herb Program Synthesis framework. 
 
-For full documentation please see the `Herb.jl` documentation
+For full documentation please see the [`Herb.jl` documentation](https://herb-ai.github.io/Herb.jl/dev/).
 
 ## Getting started
-To use this project, initialize the project with 
+For a quick tutorial on how to get started with using `HerbSearch.jl` have a look at our [introductory tutorial](https://herb-ai.github.io/Herb.jl/dev/get_started/) or [Advanced search procedures](https://herb-ai.github.io/Herb.jl/dev/tutorials/advanced_search/).
 
+If you want to help developing this project, initialize the project with 
 ```shell
 julia --project=. -e 'using Pkg; Pkg.instantiate()'
 ```

src/HerbSearch.jl

@@ -5,70 +5,68 @@ using DataStructures
 using HerbCore
 using HerbGrammar
 using HerbConstraints
-using HerbData
 using HerbInterpret
+using HerbSpecification
+using MLStyle
 
 include("sampling_grammar.jl")
-include("enumerator_constructors.jl")
 
-include("expression_iterator.jl")
+include("program_iterator.jl")
 include("count_expressions.jl")
 
-include("csg_priority_enumerator.jl")
-include("cfg_priority_enumerator.jl")
-
 include("heuristics.jl")
 
-include("stochastic_search_iterator.jl")
+include("top_down_iterator.jl")
+
+include("evaluate.jl")
+
 include("search_procedure.jl")
-include("stochastic_functions/cost_functions.jl")
 
+include("stochastic_iterator.jl")
+include("stochastic_functions/cost_functions.jl")
 include("stochastic_functions/neighbourhood.jl")
 include("stochastic_functions/propose.jl")
 include("stochastic_functions/accept.jl")
 include("stochastic_functions/temperature.jl")
-include("stochastic_enumerators.jl")
 
 include("genetic_functions/fitness.jl")
 include("genetic_functions/mutation.jl")
 include("genetic_functions/crossover.jl")
 include("genetic_functions/select_parents.jl")
 include("genetic_search_iterator.jl")
-include("genetic_enumerators.jl")
 
 export 
   count_expressions,
-  ExpressionIterator,
+  ProgramIterator,
+  @programiterator,
 
   ContextSensitivePriorityEnumerator,
-  ContextFreePriorityEnumerator,
 
   heuristic_leftmost,
   heuristic_rightmost,
   heuristic_random,
   heuristic_smallest_domain,
 
-  search_rulenode,
-  search,
-  search_best,
+  synth,
+  SynthResult,
+  optimal_program,
+  suboptimal_program,
+
+  TopDownIterator,
+  BFSIterator,
+  DFSIterator,
+  MLFSIterator,
+
+  MHSearchIterator,
+  VLSNSearchIterator,
+  SASearchIterator,
 
-  bfs_priority_function,
-  get_bfs_enumerator,
-  get_mh_enumerator,
-  get_vlsn_enumerator,
-  get_sa_enumerator,
-  get_genetic_enumerator,
   mean_squared_error,
   misclassification,
-  mse_error_function,
 
-  dfs_priority_function,
-  get_dfs_enumerator,
-
-  most_likely_priority_function,
-  get_most_likely_first_enumerator,
-  mutate_random!,
-  crossover_swap_children_2,
+  GeneticSearchIterator,
+  misclassification,
+  validate_iterator,
   sample,
   rand
 end # module HerbSearch

src/count_expressions.jl

@@ -1,20 +1,20 @@
 """
-    count_expressions(grammar::Grammar, max_depth::Int, max_size::Int, sym::Symbol)
+    count_expressions(grammar::AbstractGrammar, max_depth::Int, max_size::Int, sym::Symbol)
 
 Counts and returns the number of possible expressions of a grammar up to max_depth with start symbol sym.
 """
-function count_expressions(grammar::Grammar, max_depth::Int, max_size::Int, sym::Symbol)
+function count_expressions(grammar::AbstractGrammar, max_depth::Int, max_size::Int, sym::Symbol)
     l = 0
     # Calculate length without storing all expressions
-    for _ ∈ get_bfs_enumerator(grammar, max_depth, max_size, sym)
+    for _ ∈ BFSIterator(grammar, sym, max_depth=max_depth, max_size=max_size)
         l += 1
     end
     return l
 end
 
 """
-    count_expressions(iter::ExpressionIterator)    
+    count_expressions(iter::ProgramIterator)    
 
 Counts and returns the number of possible expressions in the expression iterator. The Iterator is not modified.
 """
-count_expressions(iter::ExpressionIterator) = count_expressions(iter.grammar, iter.max_depth, iter.max_size, iter.sym)
+count_expressions(iter::ProgramIterator) = count_expressions(iter.grammar, iter.max_depth, iter.max_size, iter.sym)

src/evaluate.jl

@@ -0,0 +1,46 @@
+struct EvaluationError <: Exception
+    expr::Expr
+    input::Dict{Symbol, Any}
+    error::Exception
+end
+
+Base.showerror(io::IO, e::EvaluationError) = print(io, "An exception was thrown while evaluating the expression $(e.expr) on input $(e.input): $(e.error)")
+
+"""
+    evaluate(problem::Problem{Vector{IOExample}}, expr::Any, tab::SymbolTable; allow_evaluation_errors::Bool=false)
+
+Evaluate the expression on the examples.
+
+Optional parameters:
+
+    - `shortcircuit` - Whether to stop evaluating after finding single example fails, to speed up the [synth](@ref) procedure. If true, the returned score is an underapproximation of the actual score.
+    - `allow_evaluation_errors` - Whether the search should continue if an exception is thrown in the evaluation or throw the error
+
+Returns a score in the interval [0, 1]
+"""
+function evaluate(problem::Problem{Vector{IOExample}}, expr::Any, symboltable::SymbolTable; shortcircuit::Bool=true, allow_evaluation_errors::Bool=false)::Number
+    number_of_satisfied_examples = 0
+
+    crashed = false
+    for example ∈ problem.spec
+        try
+            output = execute_on_input(symboltable, expr, example.in)
+            if (output == example.out)
+                number_of_satisfied_examples += 1
+            elseif (shortcircuit)
+                break;
+            end
+        catch e
+            # You could also decide to handle less severe errors (such as index out of range) differently,
+            # for example by just increasing the error value and keeping the program as a candidate.
+            crashed = true
+            # Throw the error again if evaluation errors aren't allowed
+            eval_error = EvaluationError(expr, example.in, e)
+            allow_evaluation_errors || throw(eval_error)
+            break
+        end
+    end
+
+    return number_of_satisfied_examples/length(problem.spec);
+end
+