Frangel v2

src/HerbSearch.jl

@@ -38,6 +38,19 @@ include("genetic_search_iterator.jl")
 
 include("random_iterator.jl")
 
+include("fragments/fragment_grammar_utils.jl")
+include("fragments/mining_fragments.jl")
+
+include("angelic_conditions/generate_angelic.jl")
+include("angelic_conditions/angelic_replacement_strategies.jl")
+include("angelic_conditions/long_hash_map.jl")
+
+include("frangel/frangel.jl")
+include("frangel/frangel_utils.jl")
+include("frangel/frangel_generation.jl")
+include("frangel/frangel_random_iterator.jl")
+
+
 export 
   ProgramIterator,
   @programiterator,
@@ -69,13 +82,48 @@ export
   MHSearchIterator,
   VLSNSearchIterator,
   SASearchIterator,
-
+  
   mean_squared_error,
   misclassification,
 
   GeneticSearchIterator,
   misclassification,
   validate_iterator,
   sample,
-  rand
+  rand,
+
+  FrAngelConfig,
+  FrAngelConfigGeneration,
+  frangel,
+
+  replace_first_angelic!,
+  replace_last_angelic!,
+
+  generate_random_program,
+  modify_and_replace_program_fragments!,
+  add_angelic_conditions!,
+  resolve_angelic!,
+
+  mine_fragments,
+  remember_programs!,
+
+  add_fragments_prob!,
+  setup_grammar_with_fragments!,
+  add_fragment_base_rules!,
+  add_fragment_rules!,
+  updateGrammarWithFragments!,
+
+  FrAngelRandomIterator,
+  FrAngelRandomIteratorState,
+
+  simplify_quick,
+  _simplify_quick_once,
+  symbols_minsize,
+  rules_minsize,
+
+  LongHashMap,
+  init_long_hash_map,
+  lhm_put!,
+  lhm_contains
+
 end # module HerbSearch

src/angelic_conditions/angelic_replacement_strategies.jl

@@ -0,0 +1,83 @@
+"""
+    replace_first_angelic!(program::RuleNode, boolean_expr::RuleNode, angelic_rulenode::RuleNode, angelic_conditions::Dict{UInt16,UInt8})
+        ::Union{Tuple{RuleNode,Int,AbstractHole},Nothing}
+
+Replaces the first `AbstractHole` node in the `program` with the `boolean_expr` node. 
+The 'first' is defined here as the first node visited by pre-order traversal, left-to-right. The program is modified in-place.
+
+# Arguments
+- `program`: The program to resolve angelic conditions in.
+- `boolean_expr`: The boolean expression node to replace the `AbstractHole` node with.
+- `angelic_rulenode`: The angelic rulenode. Used to compare against nodes in the program.
+- `angelic_conditions`: A dictionary mapping indices of angelic condition candidates, to the child index that may be changed.
+
+# Returns
+The parent node, the index of its modified child, and the modification. Used to clear the changes if replacement is unsuccessful.
+
+"""
+function replace_first_angelic!(
+    program::RuleNode,
+    boolean_expr::RuleNode,
+    angelic_rulenode::RuleNode,
+    angelic_conditions::Dict{UInt16,UInt8}
+)::Union{Tuple{RuleNode,Int,AbstractHole},Nothing}
+    angelic_index = get(angelic_conditions, program.ind, -1)
+    for (child_index, child) in enumerate(program.children)
+        if child_index == angelic_index && child isa AbstractHole
+            program.children[child_index] = boolean_expr
+            return (program, child_index, child)
+        else
+            res = replace_first_angelic!(child, boolean_expr, angelic_rulenode, angelic_conditions)
+            if res !== nothing
+                return res
+            end
+        end
+    end
+    return nothing
+end
+
+
+"""
+    replace_last_angelic!(program::RuleNode, boolean_expr::RuleNode, angelic_rulenode::RuleNode, angelic_conditions::Dict{UInt16,UInt8})
+        ::Union{Tuple{RuleNode,Int,AbstractHole},Nothing}
+
+Replaces the last `AbstractHole` node in the `program` with the `boolean_expr` node. 
+The 'last' is defined here as the first node visited by reversed pre-order traversal (right-to-left). The program is modified in-place.
+
+# Arguments
+- `program`: The program to resolve angelic conditions in.
+- `boolean_expr`: The boolean expression node to replace the `AbstractHole` node with.
+- `angelic_rulenode`: The angelic rulenode. Used to compare against nodes in the program.
+- `angelic_conditions`: A dictionary mapping indices of angelic condition candidates, to the child index that may be changed.
+
+# Returns
+The parent node, the index of its modified child, and the modification. Used to clear the changes if replacement is unsuccessful.
+
+"""
+function replace_last_angelic!(
+    program::RuleNode,
+    boolean_expr::RuleNode,
+    angelic_rulenode::RuleNode,
+    angelic_conditions::Dict{UInt16,UInt8}
+)::Union{Tuple{RuleNode,Int,AbstractHole},Nothing}
+    angelic_index = get(angelic_conditions, program.ind, -1)
+    # Store indices to go over them backwards later
+    indices = Vector{Int}([])
+    for child_index in reverse(eachindex(program.children))
+        child = program.children[child_index]
+        if child_index == angelic_index && child isa AbstractHole
+            program.children[child_index] = boolean_expr
+            return (program, child_index, child)
+        else
+            push!(indices, child_index)
+        end
+    end
+    # If no angelic in this layer, continue onto lower layers, in reverse
+    for child_index in indices
+        res = replace_last_angelic!(program.children[child_index], boolean_expr, angelic_rulenode, angelic_conditions)
+        if res !== nothing
+            return res
+        end
+    end
+    return nothing
+end

src/angelic_conditions/generate_angelic.jl

@@ -0,0 +1,113 @@
+"""
+    resolve_angelic!(
+        program::RuleNode, passing_tests::BitVector, grammar::AbstractGrammar, symboltable::SymbolTable, tests::AbstractVector{<:IOExample}, 
+        replacement_func::Function, angelic_conditions::Dict{UInt16,UInt8}, angelic_config::ConfigAngelic, evaluation_grammar::AbstractGrammar)::RuleNode
+
+Resolve angelic conditions in the given program by generating random boolean expressions and replacing the holes in the expression.
+The program is modified in-place. All replacement strategies are attempted sequentially as provided.
+
+# Arguments
+- `program`: The program to resolve angelic conditions in.
+- `passing_tests`: A BitVector representing the tests that the program has already passed.
+- `grammar`: The grammar rules of the program to be used for sampling angelic condition candidates.
+- `symboltable`: A symbol table for the grammar.
+- `tests`: A vector of `IOExample` objects representing the input-output test cases.
+- `replacement_func`: The function to use for replacement -> either `replace_first_angelic!` or `replace_last_angelic!`.
+- `angelic_conditions`: A dictionary mapping indices of angelic condition candidates, to the child index that may be changed.
+- `angelic_config`: The configuration for angelic conditions.
+- `evaluation_grammar`: The grammar rules of the program to be used for evaluation. Usually the same as `grammar`, or augmented with fragments.
+
+# Returns
+The resolved program with angelic values replaced, or an unresolved program if it times out.
+
+"""
+function resolve_angelic!(
+    program::RuleNode,
+    passing_tests::BitVector,
+    grammar::AbstractGrammar,
+    symboltable::SymbolTable,
+    tests::AbstractVector{<:IOExample},
+    replacement_funcs::Vector{Function},
+    angelic_conditions::Dict{UInt16,UInt8},
+    angelic_config::ConfigAngelic,
+    evaluation_grammar::AbstractGrammar=grammar
+)::RuleNode
+    num_holes = number_of_holes(program)
+    # Try each replacement strategy
+    for replacement_strategy in replacement_funcs
+        new_tests = BitVector([false for _ in tests])
+        # Continue resolution until all holes are filled
+        while num_holes != 0
+            success = false
+            start_time = time()
+            # Keep track of visited replacements - avoid duplicates
+            visited = init_long_hash_map()
+            while time() - start_time < angelic_config.max_time
+                # Generate a replacement
+                boolean_expr = rand(RuleNode, grammar, :Bool, angelic_config.boolean_expr_max_depth)
+                program_hash = hash(boolean_expr)
+                if lhm_contains(visited, program_hash)
+                    continue
+                end
+                lhm_put!(visited, program_hash)
+                # Either replace 'first' or 'last' hole
+                changed = replacement_strategy(program, boolean_expr, angelic_config.angelic_rulenode, angelic_conditions)
+                update_passed_tests!(program, evaluation_grammar, symboltable, tests, new_tests, angelic_conditions, angelic_config)
+                # If the new program passes all the tests the original program did, replacement is successful
+                if all(passing_tests .== (passing_tests .& new_tests))
+                    passing_tests = new_tests
+                    success = true
+                    break
+                else
+                    # Undo replacement changes
+                    changed[1].children[changed[2]] = changed[3]
+                end
+            end
+            if success
+                num_holes -= 1
+            else
+                break
+            end
+        end
+    end
+    return program
+end
+
+"""
+    add_angelic_conditions!(program::RuleNode, grammar::AbstractGrammar, angelic_conditions::Dict{UInt16,UInt8})::RuleNode
+
+Add angelic conditions to a program. This is done by replacing some of the nodes indicated by `angelic_conditions`` with holes.
+
+# Arguments
+- `program`: The program to modify.
+- `grammar`: The grammar rules of the program.
+- `angelic_conditions`: A dictionary mapping indices of angelic condition candidates, to the child index that may be changed.
+
+# Returns
+The modified program with angelic conditions added.
+
+"""
+function add_angelic_conditions!(program::RuleNode, grammar::AbstractGrammar, angelic_conditions::Dict{UInt16,UInt8})::RuleNode
+    if isterminal(grammar, program.ind)
+        return program
+    end
+    # If the current node has an angelic child, look for it
+    if haskey(angelic_conditions, program.ind)
+        angelic_condition_ind = angelic_conditions[program.ind]
+        for (index, child) in enumerate(program.children)
+            # Traverse children for angelic condition candidates
+            if index != angelic_condition_ind
+                program.children[index] = add_angelic_conditions!(child, grammar, angelic_conditions)
+                # A hole represents the angelic condition's location - to be replaced by angelic rulenode before evaluation
+            else
+                program.children[index] = Hole(grammar.domains[grammar.childtypes[program.ind][angelic_condition_ind]])
+            end
+        end
+        # Traverse the node's children for angelic condition candidates
+    else
+        for (index, child) in enumerate(program.children)
+            program.children[index] = add_angelic_conditions!(child, grammar, angelic_conditions)
+        end
+    end
+    program
+end

src/angelic_conditions/long_hash_map.jl

@@ -0,0 +1,82 @@
+const DEFAULT_CAPACITY = 32768
+const LOAD_FACTOR = 0.8
+
+"""
+An implementation of bare-bones hashmap that only contains the hash keys, i.e. it does not store the elements, only whether they have been seen/added before.
+This is used to keep track of programs that have been generated before, so that they are not generated again, but also does not store them to save on space.
+
+A bloom filter would usually be used for this purpose, but we did not want to introduce other dependencies, and a from-scratch implementation is not trivial.
+
+The "hashmap" only stores the hashes of visited programs, and uses linear probing for collision resolution.
+
+"""
+mutable struct LongHashMap
+    keys::Vector{UInt64}
+    occupied::Vector{Bool}
+    size::Int
+end
+
+function init_long_hash_map()
+    keys = Vector{UInt64}(undef, DEFAULT_CAPACITY)
+    occupied = Vector{Bool}(undef, DEFAULT_CAPACITY)
+    fill!(occupied, false)
+    LongHashMap(keys, occupied, 0)
+end
+
+function lhm_hash(key::UInt64, capacity::Int64)
+    return (key >>> 32) % capacity
+end
+
+# Resizes if the load factor is exceeded
+function lhm_resize!(map::LongHashMap)
+    new_capacity = length(map.keys) * 2
+    new_keys = Vector{UInt64}(undef, new_capacity)
+    new_occupied = Vector{Bool}(undef, new_capacity)
+    fill!(new_occupied, false)
+
+    for i in 1:length(map.keys)
+        if map.occupied[i]
+            index = lhm_hash(map.keys[i], new_capacity)
+            while new_occupied[index + 1]
+                index = (index + 1) % new_capacity
+            end
+            new_keys[index + 1] = map.keys[i]
+            new_occupied[index + 1] = true
+        end
+    end
+
+    map.keys = new_keys
+    map.occupied = new_occupied
+end
+
+# Adding keys to the hashmap
+function lhm_put!(map::LongHashMap, key::UInt64)
+    # Resize if load factor is exceeded
+    if (map.size / length(map.keys)) >= LOAD_FACTOR
+        lhm_resize!(map)
+    end
+    index = lhm_hash(key, length(map.keys))
+    # Collision - linear probing
+    while map.occupied[index + 1]
+        if map.keys[index + 1] == key
+            return
+        end
+        index = (index + 1) % length(map.keys)
+    end
+    map.keys[index + 1] = key
+    map.occupied[index + 1] = true
+    map.size += 1
+end
+
+# Checking if it has been seen before
+function lhm_contains(map::LongHashMap, key::UInt64)
+    index = lhm_hash(key, length(map.keys))
+    # Collision - linear probing
+    while map.occupied[index + 1]
+        if map.keys[index + 1] == key
+            return true
+        end
+        index = (index + 1) % length(map.keys)
+    end
+    return false
+end