FixedShapedSolver

Project.toml

@@ -1,7 +1,7 @@
 name = "HerbSearch"
 uuid = "3008d8e8-f9aa-438a-92ed-26e9c7b4829f"
 authors = ["Sebastijan Dumancic <[email protected]>", "Jaap de Jong <[email protected]>", "Nicolae Filat <[email protected]>", "Piotr Cichoń <[email protected]>", "Tilman Hinnerichs <[email protected]>"]
-version = "0.1.1"
+version = "0.2.1"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
@@ -17,13 +17,14 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 
 [compat]
 DataStructures = "0.17,0.18"
-HerbConstraints = "0.1.0"
-HerbCore = "0.1.1"
-HerbGrammar = "0.1.0"
-HerbInterpret = "0.1.0"
-HerbSpecification = "0.1.0"
+HerbConstraints = "^0.2.0"
+HerbCore = "^0.2.0"
+HerbGrammar = "^0.2.1"
+HerbInterpret = "0.1.2"
+HerbSpecification = "^0.1.0"
+MLStyle = "^0.4.17"
 StatsBase = "0.34"
-julia = "1.8"
+julia = "^1.8"
 
 [extras]
 LegibleLambdas = "f1f30506-32fe-5131-bd72-7c197988f9e5"

src/HerbSearch.jl

@@ -16,6 +16,7 @@ include("count_expressions.jl")
 
 include("heuristics.jl")
 
+include("fixed_shaped_iterator.jl")
 include("top_down_iterator.jl")
 
 include("evaluate.jl")
@@ -52,6 +53,8 @@ export
   optimal_program,
   suboptimal_program,
 
+  FixedShapedIterator,
+
   TopDownIterator,
   BFSIterator,
   DFSIterator,

src/count_expressions.jl

@@ -1,9 +1,9 @@
 """
-    count_expressions(grammar::Grammar, max_depth::Int, max_size::Int, sym::Symbol)
+    count_expressions(grammar::AbstractGrammar, max_depth::Int, max_size::Int, sym::Symbol)
 
 Counts and returns the number of possible expressions of a grammar up to max_depth with start symbol sym.
 """
-function count_expressions(grammar::Grammar, max_depth::Int, max_size::Int, sym::Symbol)
+function count_expressions(grammar::AbstractGrammar, max_depth::Int, max_size::Int, sym::Symbol)
     l = 0
     # Calculate length without storing all expressions
     for _ ∈ BFSIterator(grammar, sym, max_depth=max_depth, max_size=max_size)

src/fixed_shaped_iterator.jl

@@ -0,0 +1,110 @@
+Base.@doc """
+    @programiterator FixedShapedIterator()
+
+Enumerates all programs that extend from the provided fixed shaped tree.
+The [Solver](@ref) is required to be in a state without any [VariableShapedHole](@ref)s 
+""" FixedShapedIterator
+@programiterator FixedShapedIterator()
+
+"""
+    priority_function(::FixedShapedIterator, g::AbstractGrammar, tree::AbstractRuleNode, parent_value::Union{Real, Tuple{Vararg{Real}}})
+
+Assigns a priority value to a `tree` that needs to be considered later in the search. Trees with the lowest priority value are considered first.
+
+- `g`: The grammar used for enumeration
+- `tree`: The tree that is about to be stored in the priority queue
+- `parent_value`: The priority value of the parent [`State`](@ref)
+"""
+function priority_function(
+    ::FixedShapedIterator, 
+    g::AbstractGrammar, 
+    tree::AbstractRuleNode, 
+    parent_value::Union{Real, Tuple{Vararg{Real}}}
+)
+    parent_value + 1;
+end
+
+
+"""
+    hole_heuristic(::FixedShapedIterator, node::AbstractRuleNode, max_depth::Int)::Union{ExpandFailureReason, HoleReference}
+
+Defines a heuristic over fixed shaped holes. Returns a [`HoleReference`](@ref) once a hole is found.
+"""
+function hole_heuristic(::FixedShapedIterator, node::AbstractRuleNode, max_depth::Int)::Union{ExpandFailureReason, HoleReference}
+    return heuristic_leftmost_fixed_shaped_hole(node, max_depth);
+end
+
+"""
+    Base.iterate(iter::FixedShapedIterator)
+
+Describes the iteration for a given [`TopDownIterator`](@ref) over the grammar. The iteration constructs a [`PriorityQueue`](@ref) first and then prunes it propagating the active constraints. Recursively returns the result for the priority queue.
+"""
+function Base.iterate(iter::FixedShapedIterator)
+    # Priority queue with number of nodes in the program
+    pq :: PriorityQueue{State, Union{Real, Tuple{Vararg{Real}}}} = PriorityQueue()
+
+    solver = iter.solver
+    @assert !contains_variable_shaped_hole(get_tree(iter.solver)) "A FixedShapedIterator cannot iterate partial programs with VariableShapedHoles"
+
+    if isfeasible(solver)
+        enqueue!(pq, get_state(solver), priority_function(iter, get_grammar(solver), get_tree(solver), 0))
+    end
+    return _find_next_complete_tree(solver, pq, iter)
+end
+
+
+"""
+    Base.iterate(iter::FixedShapedIterator, pq::DataStructures.PriorityQueue)
+
+Describes the iteration for a given [`TopDownIterator`](@ref) and a [`PriorityQueue`](@ref) over the grammar without enqueueing new items to the priority queue. Recursively returns the result for the priority queue.
+"""
+function Base.iterate(iter::FixedShapedIterator, pq::DataStructures.PriorityQueue)
+    return _find_next_complete_tree(iter.solver, pq, iter)
+end
+
+"""
+    _find_next_complete_tree(solver::Solver, pq::PriorityQueue, iter::FixedShapedIterator)::Union{Tuple{RuleNode, PriorityQueue}, Nothing}
+
+Takes a priority queue and returns the smallest AST from the grammar it can obtain from the queue or by (repeatedly) expanding trees that are in the queue.
+Returns `nothing` if there are no trees left within the depth limit.
+"""
+function _find_next_complete_tree(
+    solver::Solver, 
+    pq::PriorityQueue,
+    iter::FixedShapedIterator
+)::Union{Tuple{RuleNode, PriorityQueue}, Nothing}
+    while length(pq) ≠ 0
+        (state, priority_value) = dequeue_pair!(pq)
+        load_state!(solver, state)
+
+        hole_res = hole_heuristic(iter, get_tree(solver), typemax(Int))
+        if hole_res ≡ already_complete
+            #the tree is complete
+            return (get_tree(solver), pq)
+        elseif hole_res ≡ limit_reached
+            # The maximum depth is reached
+            continue
+        elseif hole_res isa HoleReference
+            # Fixed Shaped Hole was found
+            # TODO: problem. this 'hole' is tied to a target state. it should be state independent
+            (; hole, path) = hole_res
+
+            rules = findall(hole.domain)
+            number_of_rules = length(rules)
+            for (i, rule_index) ∈ enumerate(findall(hole.domain))
+                if i < number_of_rules
+                    state = save_state!(solver)
+                end
+                @assert isfeasible(solver) "Attempting to expand an infeasible tree: $(get_tree(solver))"
+                remove_all_but!(solver, path, rule_index)
+                if isfeasible(solver)
+                    enqueue!(pq, get_state(solver), priority_function(iter, get_grammar(solver), get_tree(solver), priority_value))
+                end
+                if i < number_of_rules
+                    load_state!(solver, state)
+                end
+            end
+        end
+    end
+    return nothing
+end

src/genetic_functions/mutation.jl

@@ -1,9 +1,9 @@
 """
-    mutate_random!(program::RuleNode, grammar::Grammar, max_depth::Int64 = 2)
+    mutate_random!(program::RuleNode, grammar::AbstractGrammar, max_depth::Int64 = 2)
 
 Mutates the given program by inserting a randomly generated sub-program at a random location.
 """
-function mutate_random!(program::RuleNode, grammar::Grammar, max_depth::Int64 = 2)
+function mutate_random!(program::RuleNode, grammar::AbstractGrammar, max_depth::Int64 = 2)
     node_location::NodeLoc = sample(NodeLoc, program)
     subprogram = get(program, node_location)
     symbol = return_type(grammar, subprogram)

src/genetic_search_iterator.jl

@@ -46,11 +46,11 @@ cross_over(::GeneticSearchIterator, parent_1::RuleNode, parent_2::RuleNode) = cr
 
 
 """
-    mutate!(::GeneticSearchIterator, program::RuleNode, grammar::Grammar, max_depth::Int = 2)
+    mutate!(::GeneticSearchIterator, program::RuleNode, grammar::AbstractGrammar, max_depth::Int = 2)
 
 Mutates the program of an invididual.
 """
-mutate!(::GeneticSearchIterator, program::RuleNode, grammar::Grammar, max_depth::Int = 2) = mutate_random!(program, grammar, max_depth)
+mutate!(::GeneticSearchIterator, program::RuleNode, grammar::AbstractGrammar, max_depth::Int = 2) = mutate_random!(program, grammar, max_depth)
 
 """
     select_parents(::GeneticSearchIterator, population::Array{RuleNode}, fitness_array::Array{<:Real})

src/heuristics.jl

@@ -1,13 +1,42 @@
 using Random
 
+"""
+    heuristic_leftmost_fixed_shaped_hole(node::AbstractRuleNode, max_depth::Int)::Union{ExpandFailureReason, HoleReference}
+
+Defines a heuristic over [FixedShapeHole](@ref)s, where the left-most hole always gets considered first. Returns a [`HoleReference`](@ref) once a hole is found. This is the default option for enumerators.
+"""
+function heuristic_leftmost_fixed_shaped_hole(node::AbstractRuleNode, max_depth::Int)::Union{ExpandFailureReason, HoleReference}
+    function leftmost(node::AbstractRuleNode, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+        if max_depth == 0 return limit_reached end
+
+        for (i, child) in enumerate(node.children)
+            new_path = push!(copy(path), i)
+            hole_res = leftmost(child, max_depth-1, new_path)
+            if (hole_res == limit_reached) || (hole_res isa HoleReference)
+                return hole_res
+            end
+        end
+
+        return already_complete
+    end
+
+    #TODO: refactor this. this method should be merged with `heuristic_leftmost`. The only difference is the `FixedShapedHole` typing in the signature below:
+    function leftmost(hole::FixedShapedHole, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+        if max_depth == 0 return limit_reached end
+        return HoleReference(hole, path)
+    end
+
+    return leftmost(node, max_depth, Vector{Int}())
+end
+
 
 """
     heuristic_leftmost(node::AbstractRuleNode, max_depth::Int)::Union{ExpandFailureReason, HoleReference}
 
 Defines a heuristic over holes, where the left-most hole always gets considered first. Returns a [`HoleReference`](@ref) once a hole is found. This is the default option for enumerators.
 """
 function heuristic_leftmost(node::AbstractRuleNode, max_depth::Int)::Union{ExpandFailureReason, HoleReference}
-    function leftmost(node::RuleNode, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+    function leftmost(node::AbstractRuleNode, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
         if max_depth == 0 return limit_reached end
 
         for (i, child) in enumerate(node.children)
@@ -21,7 +50,7 @@ function heuristic_leftmost(node::AbstractRuleNode, max_depth::Int)::Union{Expan
         return already_complete
     end
 
-    function leftmost(hole::Hole, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+    function leftmost(hole::VariableShapedHole, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
         if max_depth == 0 return limit_reached end
         return HoleReference(hole, path)
     end
@@ -35,7 +64,7 @@ end
 Defines a heuristic over holes, where the right-most hole always gets considered first. Returns a [`HoleReference`](@ref) once a hole is found. 
 """
 function heuristic_rightmost(node::AbstractRuleNode, max_depth::Int)::Union{ExpandFailureReason, HoleReference}
-    function rightmost(node::RuleNode, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+    function rightmost(node::AbstractRuleNode, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
         if max_depth == 0 return limit_reached end
 
         for (i, child) in Iterators.reverse(enumerate(node.children))
@@ -49,7 +78,7 @@ function heuristic_rightmost(node::AbstractRuleNode, max_depth::Int)::Union{Expa
         return already_complete
     end
 
-    function rightmost(hole::Hole, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+    function rightmost(hole::VariableShapedHole, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
         if max_depth == 0 return limit_reached end
         return HoleReference(hole, path)
     end
@@ -64,7 +93,7 @@ end
 Defines a heuristic over holes, where random holes get chosen randomly using random exploration. Returns a [`HoleReference`](@ref) once a hole is found.
 """
 function heuristic_random(node::AbstractRuleNode, max_depth::Int)::Union{ExpandFailureReason, HoleReference}
-    function random(node::RuleNode, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+    function random(node::AbstractRuleNode, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
         if max_depth == 0 return limit_reached end
 
         for (i, child) in shuffle(collect(enumerate(node.children)))
@@ -78,7 +107,7 @@ function heuristic_random(node::AbstractRuleNode, max_depth::Int)::Union{ExpandF
         return already_complete
     end
 
-    function random(hole::Hole, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+    function random(hole::VariableShapedHole, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
         if max_depth == 0 return limit_reached end
         return HoleReference(hole, path)
     end
@@ -92,7 +121,7 @@ end
 Defines a heuristic over all available holes in the unfinished AST, by considering the size of their respective domains. A domain here describes the number of possible derivations with respect to the constraints. Returns a [`HoleReference`](@ref) once a hole is found. 
 """
 function heuristic_smallest_domain(node::AbstractRuleNode, max_depth::Int)::Union{ExpandFailureReason, HoleReference}
-    function smallest_domain(node::RuleNode, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+    function smallest_domain(node::AbstractRuleNode, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
         if max_depth == 0 return limit_reached end
 
         smallest_size::Int = typemax(Int)
@@ -119,7 +148,7 @@ function heuristic_smallest_domain(node::AbstractRuleNode, max_depth::Int)::Unio
         return smallest_result
     end
 
-    function smallest_domain(hole::Hole, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
+    function smallest_domain(hole::VariableShapedHole, max_depth::Int, path::Vector{Int})::Union{ExpandFailureReason, HoleReference}
         if max_depth == 0 return limit_reached end
         return HoleReference(hole, path)
     end

src/program_iterator.jl

@@ -15,7 +15,8 @@ abstract type ProgramIterator end
 
 Base.IteratorSize(::ProgramIterator) = Base.SizeUnknown()
 
-Base.eltype(::ProgramIterator) = RuleNode
+#TODO: currently, ProgramIterator will not create `StateFixedShapedHole` yet, but this should be possible
+Base.eltype(::ProgramIterator) = Union{RuleNode, StateFixedShapedHole}
 
 """
     @programiterator
@@ -49,7 +50,7 @@ macro programiterator(ex)
     generate_iterator(__module__, ex)
 end
 
-function generate_iterator(mod::Module, ex::Expr, mut::Bool=false)
+function generate_iterator(mod::Module, ex::Expr, mut::Bool=true)
     Base.remove_linenums!(ex)
 
     @match ex begin
@@ -71,7 +72,8 @@ processdecl(mod::Module, mut::Bool, decl::Expr, super=nothing) = @match decl beg
             Expr(:kw, :(max_depth::Int), typemax(Int)), 
             Expr(:kw, :(max_size::Int), typemax(Int)), 
             Expr(:kw, :(max_time::Int), typemax(Int)), 
-            Expr(:kw, :(max_enumerations::Int), typemax(Int))
+            Expr(:kw, :(max_enumerations::Int), typemax(Int)),
+            Expr(:kw, :(solver::Union{Solver, Nothing}), nothing)
         ]
 
         head = Expr(:(<:), name, isnothing(super) ? :(HerbSearch.ProgramIterator) : :($mod.$super))
@@ -82,6 +84,7 @@ processdecl(mod::Module, mut::Bool, decl::Expr, super=nothing) = @match decl beg
             max_size::Int
             max_time::Int
             max_enumerations::Int
+            solver::Union{Solver, Nothing}
         end)
 
         map!(ex -> processkwarg!(kwargs, ex), extrafields, extrafields)