Merge pull request #7 from RelationalAI/dba-large-error-messages

Truncate large error messages and allow testing for more types of errors

src/SolverAPI.jl

@@ -176,9 +176,7 @@ gracefully and included in `Response`.
 """
 function solve(fn, json::Request, solver::MOI.AbstractOptimizer)
     errors = validate(json)
-    if length(errors) > 0
-        return response(; errors)
-    end
+    isempty(errors) || return response(; errors)
 
     try
         T, solver_info, model = initialize(json, solver)
@@ -190,20 +188,23 @@ function solve(fn, json::Request, solver::MOI.AbstractOptimizer)
         end
         return response(json, model, solver)
     catch e
+        _err(E) = response(Error(E, sprint(Base.showerror, e)))
         if e isa MOI.UnsupportedError
-            throw(Error(Unsupported, sprint(Base.showerror, e)))
+            return _err(Unsupported)
         elseif e isa MOI.NotAllowedError
-            throw(Error(NotAllowed, sprint(Base.showerror, e)))
+            return _err(NotAllowed)
         elseif e isa MOI.InvalidIndex ||
                e isa MOI.ResultIndexBoundsError ||
                e isa MOI.ScalarFunctionConstantNotZero ||
                e isa MOI.LowerBoundAlreadySet ||
                e isa MOI.UpperBoundAlreadySet ||
                e isa MOI.OptimizeInProgress ||
                e isa MOI.InvalidCallbackUsage
-            throw(Error(Domain, sprint(Base.showerror, e)))
+            return _err(Domain)
         elseif e isa ErrorException
-            throw(Error(Other, e.msg))
+            return _err(Other)
+        elseif e isa Error
+            return response(e)
         else
             rethrow()
         end
@@ -353,16 +354,6 @@ function validate(json::Request)#::Vector{Error}
         end
     end
 
-    for con in json.constraints
-        if first(con) == "range"
-            if length(con) != 5
-                _err("The `range` constraint expects 4 arguments.")
-            elseif con[4] != 1
-                _err("The `range` constraint expects a step size of 1.")
-            end
-        end
-    end
-
     return out
 end
 
@@ -487,7 +478,7 @@ function nl_to_aff_or_quad(::Type{T}, f::MOI.ScalarNonlinearFunction) where {T<:
             isnothing(h) || return MOI.Utilities.operate(h, T, args...)
         end
     end
-    throw(Error(Domain, "Function $f cannot be converted to linear or quadratic form."))
+    return error() # Gets caught by canonicalize_SNF.
 end
 
 nl_to_aff_or_quad(::Type{<:Real}, f::MOI.VariableIndex) = f
@@ -522,7 +513,8 @@ function add_obj!(
     g = canonicalize_SNF(T, json_to_snf(a, vars_map))
     g_type = MOI.ObjectiveFunction{typeof(g)}()
     if !MOI.supports(model, g_type)
-        throw(Error(Unsupported, "Objective function $g isn't supported by this solver."))
+        msg = "Objective function $(trunc_str(g)) isn't supported by this solver."
+        throw(Error(Unsupported, msg))
     end
     MOI.set(model, g_type, g)
     return nothing
@@ -536,14 +528,6 @@ function add_cons!(
     solver_info::Dict,
 ) where {T<:Real}
     head = a[1]
-
-    function _check_v_type(v)
-        if !(v isa MOI.VariableIndex)
-            msg = "Variable $v must be of type MOI.VariableIndex, not $(typeof(v))."
-            throw(Error(InvalidModel, msg))
-        end
-    end
-
     if head == "and"
         for i in eachindex(a)
             i == 1 && continue
@@ -576,13 +560,17 @@ function add_cons!(
         f = MOI.ScalarNonlinearFunction(:abs, Any[v])
         MOI.add_constraint(model, f, MOI.EqualTo(1))
     elseif head == "range"
+        if length(a) != 5
+            throw(Error(InvalidModel, "The `range` constraint expects 4 arguments."))
+        end
         v = json_to_snf(a[5], vars_map)
-
+        _check_v_type(v)
         if !(a[2] isa Int && a[3] isa Int)
             throw(Error(InvalidModel, "The `range` constraint expects integer bounds."))
         end
-        _check_v_type(v)
-
+        if a[4] != 1
+            throw(Error(InvalidModel, "The `range` constraint expects a step size of 1."))
+        end
         MOI.add_constraint(model, v, MOI.Integer())
         MOI.add_constraint(model, v, MOI.Interval{T}(a[2], a[3]))
     elseif head == "implies" && solver_info[:use_indicator]
@@ -626,7 +614,7 @@ function add_cons!(
             g = shift_terms(T, f.args)
             s = S(zero(T))
             if !MOI.supports_constraint(model, typeof(g), typeof(s))
-                msg = "Constraint $g in $s isn't supported by this solver."
+                msg = "Constraint $(trunc_str(g)) in $(trunc_str(s)) isn't supported by this solver."
                 throw(Error(Unsupported, msg))
             end
             ci = MOI.Utilities.normalize_and_add_constraint(model, g, s)
@@ -635,6 +623,10 @@ function add_cons!(
     return nothing
 end
 
+_check_v_type(::MOI.VariableIndex) = nothing
+_check_v_type(_) =
+    throw(Error(InvalidModel, "$v must be a `MOI.VariableIndex`, not $(typeof(v))."))
+
 ineq_to_moi = Dict(:<= => MOI.LessThan, :>= => MOI.GreaterThan, :(==) => MOI.EqualTo)
 
 function shift_terms(::Type{T}, args::Vector) where {T<:Real}
@@ -644,4 +636,13 @@ function shift_terms(::Type{T}, args::Vector) where {T<:Real}
     return MOI.Utilities.operate(-, T, g1, g2)
 end
 
+# Convert object to string and truncate string length if too long.
+function trunc_str(f::Union{MOI.AbstractScalarFunction,MOI.AbstractScalarSet})
+    f_str = string(f)
+    if length(f_str) > 256
+        f_str = f_str[1:256] * " ... (truncated)"
+    end
+    return f_str
+end
+
 end # module SolverAPI

test/all_tests.jl

@@ -75,35 +75,59 @@ end
     end
 end
 
-@testitem "validate" setup = [SolverSetup] begin
-    using SolverAPI: deserialize, validate
+@testitem "errors" setup = [SolverSetup] begin
+    using SolverAPI
     import JSON3
 
     # scenarios with incorrect format
-    format_err_json_names = [
-        # TODO fix: error not thrown for "unsupported_print_format"
-        # "unsupported_print_format",     # print format not supported
-        "feas_with_obj",                # objective provided for a feasibility problem
-        "min_no_obj",                   # no objective function specified for a minimization problem
-        "unsupported_sense",            # unsupported sense such as 'feasiblity'
-        "obj_len_greater_than_1",       # length of objective greater than 1
-        "incorrect_range_num_params",   # number of parameters not equal to 4
-        "incorrect_range_step_not_1",   # step not one in range definition
-        "vars_is_not_str",              # field variables is not a string
-        "vars_is_not_arr",              # field variables is not an array
-        "objs_is_not_arr",              # field objectives is not an array
-        "cons_is_not_arr",              # field constraints is not an array
-        "missing_vars",                 # missing field variables
-        "missing_cons",                 # missing field constraints
-        "missing_objs",                 # missing field objectives
-        "missing_sense",                # missing field sense
-        "missing_version",              # missing field version
+    json_names_and_errors = [
+        # missing field variables
+        ("missing_vars", "InvalidFormat"),
+        # missing field constraints
+        ("missing_cons", "InvalidFormat"),
+        # missing field objectives
+        ("missing_objs", "InvalidFormat"),
+        # missing field sense
+        ("missing_sense", "InvalidFormat"),
+        # missing field version
+        ("missing_version", "InvalidFormat"),
+        # field variables is not a string
+        ("vars_is_not_str", "InvalidFormat"),
+        # field variables is not an array
+        ("vars_is_not_arr", "InvalidFormat"),
+        # field objectives is not an array
+        ("objs_is_not_arr", "InvalidFormat"),
+        # field constraints is not an array
+        ("cons_is_not_arr", "InvalidFormat"),
+        # length of objective greater than 1
+        ("obj_len_greater_than_1", "InvalidFormat"),
+        # objective provided for a feasibility problem
+        ("feas_with_obj", "InvalidFormat"),
+        # no objective function specified for a minimization problem
+        ("min_no_obj", "InvalidFormat"),
+        # unsupported sense such as 'feasibility'
+        ("unsupported_sense", "InvalidFormat"),
+        # range: wrong number of args
+        ("incorrect_range_num_params", "InvalidModel"),
+        # range: step not one
+        ("incorrect_range_step_not_1", "InvalidModel"),
+        # unsupported objective function type
+        ("unsupported_obj_type", "Unsupported"),
+        # unsupported constraint function type
+        ("unsupported_con_type", "Unsupported"),
+        # unsupported constraint sign
+        ("unsupported_con_sign", "Unsupported"),
+        # unsupported operator
+        ("unsupported_operator", "Unsupported"),
+        # unsupported solver option
+        ("unsupported_solver_option", "Unsupported"),
+        # print format not supported
+        ("unsupported_print_format", "Unsupported"),
     ]
 
-    @testset "$j" for j in format_err_json_names
-        input = deserialize(read_json("inputs", j))
-        errors = validate(input)
-        @test errors isa Vector{SolverAPI.Error}
-        @test length(errors) >= 1
+    @testset "$j" for (j, es...) in json_names_and_errors
+        result = JSON3.read(run_solve(read_json("inputs", j)))
+        @test haskey(result, :errors) && length(result.errors) >= 1
+        @test Set(e.type for e in result.errors) == Set(es)
     end
 end

test/inputs/feas_range.json

@@ -1 +1 @@
-{"version":"0.1","sense":"feas","variables":["x"],"constraints":[["range",9, 9, 1, "x"],["Float","x"]],"objectives":[],"options":{"solver":"MiniZinc"}}
+{"version":"0.1","sense":"feas","variables":["x"],"constraints":[["range",9,9,1,"x"],["Float","x"]],"objectives":[],"options":{"solver":"MiniZinc"}}

test/inputs/incorrect_range_num_params.json

@@ -1 +1 @@
-{"version":"0.1","sense":"feas","variables":["x"],"constraints":[["range", 9, 9, "x"],["Int","x"]],"objectives":[],"options":{"solver":"MiniZinc"}}
+{"version":"0.1","sense":"feas","variables":["x"],"constraints":[["range",9,9,"x"],["Int","x"]],"objectives":[],"options":{"solver":"MiniZinc"}}

test/inputs/incorrect_range_step_not_1.json

@@ -1 +1 @@
-{"version":"0.1","sense":"feas","variables":["x"],"constraints":[["range",9, 9, 2, "x"],["Int","x"]],"objectives":[],"options":{"solver":"MiniZinc"}}
+{"version":"0.1","sense":"feas","variables":["x"],"constraints":[["range",9,9,2,"x"],["Int","x"]],"objectives":[],"options":{"solver":"MiniZinc"}}

test/inputs/unsupported_con_sign.json

@@ -0,0 +1 @@
+{"version":"0.1","sense":"min","variables":["x","y"],"constraints":[["and",["!=",["+","x",["*",3,"y"]],1],[">=",["+","x","y"],1]],["and",["Int","x"],["Nonneg","x"]],["Int","y"]],"objectives":[["+",["*",2,"x"],"y"]],"options":{"solver":"HiGHS"}}

test/inputs/unsupported_con_type.json

@@ -0,0 +1 @@
+{"version":"0.1","sense":"min","variables":["x","y"],"constraints":[["and",["==",["*","x",["*","x","y"]],1],[">=",["+","x","y"],1]],["and",["Int","x"],["Nonneg","x"]],["Int","y"]],"objectives":[["+",["*",2,"x"],"y"]],"options":{"solver":"HiGHS"}}

test/inputs/unsupported_obj_type.json

@@ -0,0 +1 @@
+{"version":"0.1","sense":"min","variables":["x","y"],"constraints":[["and",["==",["+","x",["*",3,"y"]],1],[">=",["+","x","y"],1]],["and",["Int","x"],["Nonneg","x"]],["Int","y"]],"objectives":[["/",["*",2,"x"],"y"]],"options":{"solver":"HiGHS"}}

test/inputs/unsupported_operator.json

@@ -0,0 +1 @@
+{"version":"0.1","sense":"feas","variables":["x"],"constraints":[["fake_operator","x",1],["Int","x"]],"objectives":[],"options":{"solver":"MiniZinc"}}

test/inputs/unsupported_solver_option.json

@@ -0,0 +1 @@
+{"version":"0.1","sense":"min","variables":["x"],"constraints":[["==","x",1],["Int","x"]],"objectives":["x"],"options":{"time_limit_sec":60,"solver":"HiGHS","fake_option":0}}