Merge pull request #95 from Herb-AI/contains-subtree

Add tests for changes in HerbConstraints

src/program_iterator.jl

@@ -122,7 +122,7 @@ processdecl(mod::Module, mut::Bool, decl::Expr, super=nothing) = @match decl beg
                 end
 
                 # solver with grammar and initial rulenode to start with
-                function $(escaped_name)(grammar::AbstractGrammar, initial_node::RuleNode, $(notkwargs...) ;
+                function $(escaped_name)(grammar::AbstractGrammar, initial_node::AbstractRuleNode, $(notkwargs...) ;
                                         max_size = typemax(Int), max_depth = typemax(Int), $(kwargs_fields...) )
                     return $(escaped_name)(GenericSolver(grammar, initial_node, max_size = max_size, max_depth = max_depth), $(field_names...))
                 end

test/runtests.jl

@@ -22,8 +22,11 @@ Random.seed!(1234)
     include("test_forbidden.jl")
     include("test_ordered.jl")
     include("test_contains.jl")
+    include("test_contains_subtree.jl")
     include("test_unique.jl")
 
+    include("test_constraints.jl")
+
     # Excluded because it contains long tests
     # include("test_realistic_searches.jl")
 end

test/test_constraints.jl

@@ -0,0 +1,120 @@
+using HerbCore, HerbGrammar, HerbConstraints
+
+@testset verbose=true "Constraints" begin
+
+    function new_grammar()
+        grammar = @csgrammar begin
+            Int = 1
+            Int = x
+            Int = -Int
+            Int = Int + Int
+            Int = Int * Int
+        end
+        clearconstraints!(grammar)
+        return grammar
+    end
+
+    contains_subtree = ContainsSubtree(RuleNode(4, [
+        RuleNode(1),
+        RuleNode(1)
+    ]))
+
+    contains_subtree2 = ContainsSubtree(RuleNode(4, [
+        RuleNode(4, [
+            VarNode(:a),
+            RuleNode(2)
+        ]),
+        VarNode(:a)
+    ]))
+
+    contains = Contains(2)
+
+    forbidden_sequence = ForbiddenSequence([4, 5])
+
+    forbidden_sequence2 = ForbiddenSequence([4, 5], ignore_if=[3])
+
+    forbidden = Forbidden(RuleNode(3, [RuleNode(3, [VarNode(:a)])]))
+
+    forbidden2 = Forbidden(RuleNode(4, [
+        VarNode(:a),
+        VarNode(:a)
+    ]))
+
+    ordered = Ordered(RuleNode(5, [
+        VarNode(:a),
+        VarNode(:b)
+    ]), [:a, :b])
+
+    unique = Unique(2)
+
+    @testset "fix_point_running related bug" begin
+        # post contains_subtree2
+        # propagate contains_subtree2
+        #     schedule forbidden2
+        # propagate forbidden2
+
+        grammar = new_grammar()
+        addconstraint!(grammar, contains_subtree)
+        addconstraint!(grammar, contains_subtree2)
+        addconstraint!(grammar, forbidden2)
+
+        partial_program = UniformHole(BitVector((0, 0, 0, 1, 1)), [
+            UniformHole(BitVector((0, 0, 0, 1, 1)), [
+                UniformHole(BitVector((1, 1, 0, 0, 0)), []),
+                UniformHole(BitVector((1, 1, 0, 0, 0)), [])
+            ]),
+            UniformHole(BitVector((0, 0, 0, 1, 1)), [
+                UniformHole(BitVector((0, 0, 0, 1, 1)), [
+                    UniformHole(BitVector((1, 1, 0, 0, 0)), []),
+                    UniformHole(BitVector((1, 1, 0, 0, 0)), [])
+                ])
+                UniformHole(BitVector((1, 1, 0, 0, 0)), [])
+            ])
+        ])
+
+        iterator = BFSIterator(grammar, partial_program, max_size=9) 
+        @test length(iterator) == 0
+    end
+
+    all_constraints = [
+        ("ContainsSubtree", contains_subtree),
+        ("ContainsSubtree2", contains_subtree2),
+        ("Contains", contains),
+        ("ForbiddenSequence", forbidden_sequence),
+        ("ForbiddenSequence2", forbidden_sequence2),
+        ("Forbidden", forbidden),
+        ("Forbidden2", forbidden2),
+        ("Ordered", ordered),
+        ("Unique", unique)
+    ]
+
+    @testset "1 constraint" begin
+        # test all constraints individually, the constraints are chosen to prune the program space non-trivially
+        @testset "$name" for (name, constraint) ∈ all_constraints
+            test_constraint!(new_grammar(), constraint, max_size=6, allow_trivial=false)
+        end
+    end
+
+    @testset "$n constraints" for n ∈ 2:5
+        # test constraint interactions by randomly sampling constraints
+        for _ ∈ 1:10
+            indices = randperm(length(all_constraints))[1:n]
+            names = [name for (name, _) ∈ all_constraints[indices]]
+            constraints = [constraint for (_, constraint) ∈ all_constraints[indices]]
+
+            @testset "$names" begin
+                test_constraints!(new_grammar(), constraints, max_size=6, allow_trivial=true)
+            end
+        end
+    end
+
+    @testset "all constraints" begin
+        # all constraints combined, no valid solution exists
+        grammar = new_grammar()
+        for (_, constraint) ∈ all_constraints
+            addconstraint!(grammar, constraint)
+        end
+        iter = BFSIterator(grammar, :Int, max_size=10)
+        @test length(iter) == 0
+    end
+end

test/test_contains.jl

@@ -16,6 +16,6 @@ using HerbCore, HerbGrammar, HerbConstraints
 
         # There are 5! = 120 permutations of 5 distinct elements
         iter = BFSIterator(grammar, :Permutation)
-        @test length(collect(iter)) == 120
+        @test length(iter) == 120
     end
 end

test/test_contains_subtree.jl

@@ -0,0 +1,97 @@
+using HerbCore, HerbGrammar, HerbConstraints
+
+@testset verbose=true "ContainsSubtree" begin
+    @testset "Minimal Example" begin
+        grammar = @csgrammar begin
+            Int = x
+            Int = Int + Int
+            Int = Int + Int
+            Int = 1
+        end
+
+        constraint = ContainsSubtree(
+            RuleNode(2, [
+                RuleNode(1),
+                RuleNode(2, [
+                    RuleNode(1),
+                    RuleNode(1)
+                ])
+            ])
+        )
+
+        test_constraint!(grammar, constraint, max_size=6)
+    end
+
+    @testset "1 VarNode" begin
+        grammar = @csgrammar begin
+            Int = x
+            Int = Int + Int
+            Int = Int + Int
+            Int = 1
+        end
+
+        constraint = ContainsSubtree(
+            RuleNode(2, [
+                RuleNode(1),
+                VarNode(:x)
+            ])
+        )
+
+        test_constraint!(grammar, constraint, max_size=6)
+    end
+
+    @testset "2 VarNodes" begin
+        grammar = @csgrammar begin
+            Int = x
+            Int = Int + Int
+            Int = Int + Int
+            Int = 1
+        end
+
+        constraint = ContainsSubtree(
+            RuleNode(2, [
+                VarNode(:x),
+                VarNode(:x)
+            ])
+        )
+
+        test_constraint!(grammar, constraint, max_size=6)
+    end
+
+
+    @testset "No StateHoles" begin
+        grammar = @csgrammar begin
+            Int = x
+            Int = Int + Int
+        end
+
+        constraint = ContainsSubtree(
+            RuleNode(2, [
+                RuleNode(1),
+                RuleNode(2, [
+                    RuleNode(1),
+                    RuleNode(1)
+                ])
+            ])
+        )
+
+        test_constraint!(grammar, constraint, max_size=6)
+    end
+
+    @testset "Permutations" begin
+        # A grammar that represents all permutations of (1, 2, 3, 4, 5)
+        grammar = @csgrammar begin
+            N = |(1:5)
+            Permutation = (N, N, N, N, N)
+        end
+        addconstraint!(grammar, ContainsSubtree(RuleNode(1)))
+        addconstraint!(grammar, ContainsSubtree(RuleNode(2)))
+        addconstraint!(grammar, ContainsSubtree(RuleNode(3)))
+        addconstraint!(grammar, ContainsSubtree(RuleNode(4)))
+        addconstraint!(grammar, ContainsSubtree(RuleNode(5)))
+
+        # There are 5! = 120 permutations of 5 distinct elements
+        iter = BFSIterator(grammar, :Permutation)
+        @test length(iter) == 120
+    end
+end

test/test_helpers.jl

@@ -25,3 +25,57 @@ function create_problem(f, range=20)
     examples = [IOExample(Dict(:x => x), f(x)) for x ∈ 1:range]
     return Problem(examples), examples
 end
+
+"""
+    function test_constraints!(grammar::AbstractGrammar, constraints::Vector{AbstractGrammarConstraint}; max_size=typemax(Int), max_depth=typemax(Int), allow_trivial=false)
+
+Tests if propagating the constraints during a top-down iteration yields the correct number of programs.
+
+Does two searches and tests if they have the same amount of programs:
+- without the constraints and retrospectively applying the constraints
+- propagating the constraints during search
+
+If `allow_trivial = false`, it is tested that:
+-  at least 1 program satisfies the constraint
+-  at least 1 program violates the constraint
+"""
+function test_constraints!(grammar::AbstractGrammar, constraints::Vector{<:AbstractGrammarConstraint}; max_size=typemax(Int), max_depth=typemax(Int), allow_trivial=false)
+    starting_symbol = grammar.types[1]
+    iter = BFSIterator(grammar, starting_symbol, max_size = max_size, max_depth = max_depth)
+    alltrees = 0
+    validtrees = 0
+    for p ∈ iter
+        if all(check_tree(constraint, p) for constraint ∈ constraints)
+            validtrees += 1
+        end
+        alltrees += 1
+    end
+
+    for constraint ∈ constraints
+        addconstraint!(grammar, constraint)
+    end
+    constraint_iter = BFSIterator(grammar, starting_symbol, max_size = max_size, max_depth = max_depth)
+
+    @test length(constraint_iter) == validtrees
+    if !allow_trivial
+        @test validtrees > 0
+        @test validtrees < alltrees
+    end
+end
+
+"""
+    test_constraint!(grammar::AbstractGrammar, constraint::AbstractGrammarConstraint; max_size=typemax(Int), max_depth=typemax(Int))
+
+Tests if propagating the constraint during a top-down iteration yields the correct number of programs.
+
+Does two searches and tests if they have the same amount of programs:
+- without the constraint and retrospectively applying the constraint
+- propagating the constraint during search
+
+If `allow_trivial = false`, it is tested that:
+-  at least 1 program satisfies the constraint
+-  at least 1 program violates the constraint
+"""
+function test_constraint!(grammar::AbstractGrammar, constraint::AbstractGrammarConstraint; max_size=typemax(Int), max_depth=typemax(Int), allow_trivial=false)
+    test_constraints!(grammar, [constraint], max_size = max_size, max_depth = max_depth, allow_trivial=allow_trivial)
+end

test/test_ordered.jl

@@ -2,7 +2,7 @@ using HerbCore, HerbGrammar, HerbConstraints
 
 @testset verbose=true "Ordered" begin
 
-    function get_grammar_and_constraint1()
+    @testset "Number of candidate programs" begin
         grammar = @csgrammar begin
             Number = 1
             Number = x
@@ -12,10 +12,8 @@ using HerbCore, HerbGrammar, HerbConstraints
             VarNode(:a),
             VarNode(:b)
         ]), [:a, :b])
-        return grammar, constraint
-    end
+        test_constraint!(grammar, constraint, max_size=6)
 
-    function get_grammar_and_constraint2()
         grammar = @csgrammar begin
             Number = Number + Number
             Number = 1
@@ -26,28 +24,7 @@ using HerbCore, HerbGrammar, HerbConstraints
             RuleNode(3, [VarNode(:a)]) ,
             RuleNode(3, [VarNode(:b)])
         ]), [:a, :b])
-        return grammar, constraint
-    end
-
-    @testset "Number of candidate programs" begin
-        for (grammar, constraint) in [get_grammar_and_constraint1(), get_grammar_and_constraint2()]
-            iter = BFSIterator(grammar, :Number, max_size=6)
-            alltrees = 0
-            validtrees = 0
-            for p ∈ iter
-                if check_tree(constraint, p)
-                    validtrees += 1
-                end
-                alltrees += 1
-            end
-
-            addconstraint!(grammar, constraint)
-            constraint_iter = BFSIterator(grammar, :Number, max_size=6)
-
-            @test validtrees > 0
-            @test validtrees < alltrees
-            @test length([freeze_state(p) for p ∈ constraint_iter]) == validtrees
-        end
+        test_constraint!(grammar, constraint, max_size=6)
     end
 
     @testset "DomainRuleNode" begin