From e96d3a6483eaf46fc11dc699d7c554342f91ddb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Wed, 10 Jul 2024 19:08:42 +0200 Subject: [PATCH 1/8] Add support for VariableIndex-in-EqualTo constraint --- src/MOI_wrapper.jl | 54 +++++++++++++++++++++++++--- test/simple_lowrank.jl | 8 +++++ test/{simple.jl => simple_sparse.jl} | 0 test/test_simple.jl | 32 ++++++++++++++--- 4 files changed, 85 insertions(+), 9 deletions(-) create mode 100644 test/simple_lowrank.jl rename test/{simple.jl => simple_sparse.jl} (100%) diff --git a/src/MOI_wrapper.jl b/src/MOI_wrapper.jl index 04179a8..05b8527 100644 --- a/src/MOI_wrapper.jl +++ b/src/MOI_wrapper.jl @@ -20,6 +20,8 @@ const SupportedSets = mutable struct Optimizer <: MOI.AbstractOptimizer objective_constant::Float64 objective_sign::Int + is_scalar_product::BitSet + variable_constraint::Vector{Int} blksz::Vector{Cptrdiff_t} blktype::Vector{Cchar} varmap::Vector{Tuple{Int,Int,Int}} # Variable Index vi -> blk, i, j @@ -48,6 +50,8 @@ mutable struct Optimizer <: MOI.AbstractOptimizer return new( 0.0, 1, + BitSet(), + Int[], Cptrdiff_t[], Cchar[], Tuple{Int,Int,Int}[], @@ -142,6 +146,7 @@ function _new_block(model::Optimizer, set::MOI.Nonnegatives) blk = length(model.blksz) for i in 1:MOI.dimension(set) push!(model.varmap, (blk, i, i)) + push!(model.variable_constraint, 0) end return end @@ -153,6 +158,7 @@ function _new_block(model::Optimizer, set::MOI.PositiveSemidefiniteConeTriangle) for j in 1:set.side_dimension for i in 1:j push!(model.varmap, (blk, i, j)) + push!(model.variable_constraint, 0) end end return @@ -184,7 +190,7 @@ end function MOI.supports_constraint( ::Optimizer, - ::Type{MOI.ScalarAffineFunction{Cdouble}}, + ::Union{Type{MOI.SingleVariable},Type{MOI.ScalarAffineFunction{Cdouble}}}, ::Type{MOI.EqualTo{Cdouble}}, ) return true @@ -253,12 +259,11 @@ function _fill!( @assert length(type) == length(model.blksz) end -function MOI.add_constraint( +function _add_constraint( model::Optimizer, func::MOI.ScalarAffineFunction{Cdouble}, set::MOI.EqualTo{Cdouble}, ) - reset_solution!(model) push!(model.Ainfo_entptr, Csize_t[]) push!(model.Ainfo_type, Cchar[]) _fill!( @@ -271,7 +276,34 @@ function MOI.add_constraint( func, ) push!(model.b, MOI.constant(set) - MOI.constant(func)) - return MOI.ConstraintIndex{typeof(func),typeof(set)}(length(model.b)) + return length(model.b) +end + +function MOI.add_constraint( + model::Optimizer, + func::MOI.ScalarAffineFunction{Cdouble}, + set::MOI.EqualTo{Cdouble}, +) + reset_solution!(model) + index = _add_constraint(model, func, set) + return MOI.ConstraintIndex{typeof(func),typeof(set)}(index) +end + +function MOI.add_constraint( + model::Optimizer, + vi::MOI.VariableIndex, + set::MOI.EqualTo{Cdouble}, +) + + if !iszero(model.variable_constraint) + S = typeof(set) + throw(MOI.LowerBoundAlreadySet{S,S}(vi)) + end + reset_solution!(model) + func = convert(MOI.ScalarAffineFunction{Cdouble}, vi) + index = _add_constraint(model, func, set) + model.variable_constraint[vi.value] = index + return MOI.ConstraintIndex{typeof(vi),typeof(set)}(vi.value) end MOI.supports_incremental_interface(::Optimizer) = true @@ -367,6 +399,8 @@ end function MOI.empty!(optimizer::Optimizer) optimizer.objective_constant = 0.0 optimizer.objective_sign = 1 + empty!(optimizer.is_scalar_product) + empty!(optimizer.variable_constraint) empty!(optimizer.blksz) empty!(optimizer.blktype) empty!(optimizer.varmap) @@ -497,3 +531,15 @@ function MOI.get( MOI.check_result_index_bounds(optimizer, attr) return optimizer.lambda[ci.value] end + +function MOI.get( + optimizer::Optimizer, + attr::MOI.ConstraintDual, + ci::MOI.ConstraintIndex{ + MOI.VariableIndex, + MOI.EqualTo{Cdouble}, + }, +) + MOI.check_result_index_bounds(optimizer, attr) + return optimizer.lambda[optimizer.variable_constraint[ci.value]] +end diff --git a/test/simple_lowrank.jl b/test/simple_lowrank.jl new file mode 100644 index 0000000..3149e01 --- /dev/null +++ b/test/simple_lowrank.jl @@ -0,0 +1,8 @@ +blksz = Cptrdiff_t[2] +blktype = Cchar['s'] +b = Cdouble[1] +CAinfo_entptr = Csize_t[0, 2, 8] +CAent = Cdouble[1, 1, -0.25, 0.25, -1, 1, 1, 1] +CArow = Csize_t[1, 2, 1, 2, 1, 2, 1, 2] +CAcol = Csize_t[1, 2, 1, 2, 1, 1, 2, 2] +CAinfo_type = Cchar['s', 'l'] diff --git a/test/simple.jl b/test/simple_sparse.jl similarity index 100% rename from test/simple.jl rename to test/simple_sparse.jl diff --git a/test/test_simple.jl b/test/test_simple.jl index 491cd9b..2b8870b 100644 --- a/test/test_simple.jl +++ b/test/test_simple.jl @@ -4,8 +4,8 @@ import Random import MathOptInterface as MOI # This is `test_conic_PositiveSemidefiniteConeTriangle_VectorOfVariables` -@testset "Solve simple with sdplrlib" begin - include("simple.jl") +@testset "Solve simple with sdplrlib with $file" for file in ["simple_sparse.jl", "simple_lowrank.jl"] + include(file) # The `925` seed is taken from SDPLR's `main.c` Random.seed!(925) ret, R, lambda, ranks, pieces = SDPLR.solve( @@ -26,7 +26,7 @@ import MathOptInterface as MOI @test ranks == Csize_t[2] end -function simple_model() +function simple_sparse_model() model = SDPLR.Optimizer() X, _ = MOI.add_constrained_variables( model, @@ -40,6 +40,28 @@ function simple_model() return model, X, c end +function simple_lowrank_model() + model = SDPLR.Optimizer() + A = MOI.LowRankMatrix( + [-1/4, 1/4], + [-1.0 1.0 + 1.0 1.0] + ) + X, _ = MOI.add_constrained_variables( + model, + MOI.SetWithDotProducts( + MOI.PositiveSemidefiniteConeTriangle(2), + [MOI.TriangleVectorization(A)], + ), + ) + c = MOI.add_constraint(model, X[4], MOI.EqualTo(1.0)) + MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE) + obj = 1.0 * X[1] + 1.0 * X[3] + MOI.set(model, MOI.ObjectiveFunction{typeof(obj)}(), obj) + @test MOI.get(model, MOI.TerminationStatus()) == MOI.OPTIMIZE_NOT_CALLED + return model, X[1:3], c +end + function simple_test(model, X, c) atol = rtol = 1e-2 @test MOI.get(model, MOI.TerminationStatus()) == MOI.LOCALLY_SOLVED @@ -54,8 +76,8 @@ function simple_test(model, X, c) @test σ ≈ sigma end -@testset "MOI wrapper" begin - model, X, c = simple_model() +@testset "MOI wrapper for $f" for f in [simple_sparse_model] #, simple_lowrank_model] + model, X, c = f() MOI.optimize!(model) simple_test(model, X, c) end From a6699cf8f5e7ecddea7987c3a440588bf6c4b58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Wed, 10 Jul 2024 20:08:31 +0200 Subject: [PATCH 2/8] Fixes --- src/MOI_wrapper.jl | 115 +++++++++++++++++++++----------------------- test/test_simple.jl | 12 ++--- 2 files changed, 62 insertions(+), 65 deletions(-) diff --git a/src/MOI_wrapper.jl b/src/MOI_wrapper.jl index 05b8527..05b0ca7 100644 --- a/src/MOI_wrapper.jl +++ b/src/MOI_wrapper.jl @@ -14,14 +14,21 @@ const PIECES_MAP = Dict{String,Int}( "overallsc" => 8, ) -const SupportedSets = - Union{MOI.Nonnegatives,MOI.PositiveSemidefiniteConeTriangle} +const _SetWithDotProd = MOI.SetWithDotProducts{ + MOI.PositiveSemidefiniteConeTriangle, + MOI.TriangleVectorization{MOI.LowRankMatrix{Cdouble}}, +} + +const SupportedSets = Union{ + MOI.Nonnegatives, + MOI.PositiveSemidefiniteConeTriangle, + _SetWithDotProd, +} mutable struct Optimizer <: MOI.AbstractOptimizer objective_constant::Float64 objective_sign::Int - is_scalar_product::BitSet - variable_constraint::Vector{Int} + dot_product::Vector{Union{Nothing,MOI.LowRankMatrix{Cdouble}}} blksz::Vector{Cptrdiff_t} blktype::Vector{Cchar} varmap::Vector{Tuple{Int,Int,Int}} # Variable Index vi -> blk, i, j @@ -50,8 +57,7 @@ mutable struct Optimizer <: MOI.AbstractOptimizer return new( 0.0, 1, - BitSet(), - Int[], + Union{Nothing,MOI.LowRankMatrix{Cdouble}}[], Cptrdiff_t[], Cchar[], Tuple{Int,Int,Int}[], @@ -146,7 +152,7 @@ function _new_block(model::Optimizer, set::MOI.Nonnegatives) blk = length(model.blksz) for i in 1:MOI.dimension(set) push!(model.varmap, (blk, i, i)) - push!(model.variable_constraint, 0) + push!(model.dot_product, nothing) end return end @@ -158,12 +164,22 @@ function _new_block(model::Optimizer, set::MOI.PositiveSemidefiniteConeTriangle) for j in 1:set.side_dimension for i in 1:j push!(model.varmap, (blk, i, j)) - push!(model.variable_constraint, 0) + push!(model.dot_product, nothing) end end return end +function _new_block(model::Optimizer, set::_SetWithDotProd) + blk = length(model.blksz) + 1 + for i in eachindex(set.vectors) + push!(model.varmap, (-blk, i, i)) + push!(model.dot_product, set.vectors[i].matrix) + end + _new_block(model, set.set) + return +end + function MOI.add_constrained_variables(model::Optimizer, set::SupportedSets) reset_solution!(model) offset = length(model.varmap) @@ -190,7 +206,7 @@ end function MOI.supports_constraint( ::Optimizer, - ::Union{Type{MOI.SingleVariable},Type{MOI.ScalarAffineFunction{Cdouble}}}, + ::Type{MOI.ScalarAffineFunction{Cdouble}}, ::Type{MOI.EqualTo{Cdouble}}, ) return true @@ -245,25 +261,46 @@ function _fill!( ) for t in MOI.Utilities.canonical(func).terms blk, i, j = model.varmap[t.variable.value] - _fill_until(model, blk, entptr, type, length(ent)) - coef = t.coefficient - if i != j - coef /= 2 + _fill_until(model, abs(blk), entptr, type, length(ent)) + if type[end] == Cchar('l') + error("Can either have one dot product variable or several normal variables in the same constraint") + end + if blk < 0 + type[end] = Cchar('l') + mat = model.dot_product[t.variable.value] + for i in eachindex(mat.diagonal) + push!(ent, mat.diagonal[i]) + push!(row, i) + push!(col, i) + end + for j in axes(mat.factor, 2) + for i in axes(mat.factor, 1) + push!(ent, mat.factor[i, j]) + push!(row, i) + push!(col, j) + end + end + else + coef = t.coefficient + if i != j + coef /= 2 + end + push!(ent, coef) + push!(row, i) + push!(col, j) end - push!(ent, coef) - push!(row, i) - push!(col, j) end _fill_until(model, length(model.blksz), entptr, type, length(ent)) @assert length(entptr) == length(model.blksz) @assert length(type) == length(model.blksz) end -function _add_constraint( +function MOI.add_constraint( model::Optimizer, func::MOI.ScalarAffineFunction{Cdouble}, set::MOI.EqualTo{Cdouble}, ) + reset_solution!(model) push!(model.Ainfo_entptr, Csize_t[]) push!(model.Ainfo_type, Cchar[]) _fill!( @@ -276,34 +313,7 @@ function _add_constraint( func, ) push!(model.b, MOI.constant(set) - MOI.constant(func)) - return length(model.b) -end - -function MOI.add_constraint( - model::Optimizer, - func::MOI.ScalarAffineFunction{Cdouble}, - set::MOI.EqualTo{Cdouble}, -) - reset_solution!(model) - index = _add_constraint(model, func, set) - return MOI.ConstraintIndex{typeof(func),typeof(set)}(index) -end - -function MOI.add_constraint( - model::Optimizer, - vi::MOI.VariableIndex, - set::MOI.EqualTo{Cdouble}, -) - - if !iszero(model.variable_constraint) - S = typeof(set) - throw(MOI.LowerBoundAlreadySet{S,S}(vi)) - end - reset_solution!(model) - func = convert(MOI.ScalarAffineFunction{Cdouble}, vi) - index = _add_constraint(model, func, set) - model.variable_constraint[vi.value] = index - return MOI.ConstraintIndex{typeof(vi),typeof(set)}(vi.value) + return MOI.ConstraintIndex{typeof(func),typeof(set)}(length(model.b)) end MOI.supports_incremental_interface(::Optimizer) = true @@ -399,8 +409,7 @@ end function MOI.empty!(optimizer::Optimizer) optimizer.objective_constant = 0.0 optimizer.objective_sign = 1 - empty!(optimizer.is_scalar_product) - empty!(optimizer.variable_constraint) + empty!(optimizer.dot_product) empty!(optimizer.blksz) empty!(optimizer.blktype) empty!(optimizer.varmap) @@ -531,15 +540,3 @@ function MOI.get( MOI.check_result_index_bounds(optimizer, attr) return optimizer.lambda[ci.value] end - -function MOI.get( - optimizer::Optimizer, - attr::MOI.ConstraintDual, - ci::MOI.ConstraintIndex{ - MOI.VariableIndex, - MOI.EqualTo{Cdouble}, - }, -) - MOI.check_result_index_bounds(optimizer, attr) - return optimizer.lambda[optimizer.variable_constraint[ci.value]] -end diff --git a/test/test_simple.jl b/test/test_simple.jl index 2b8870b..35fd493 100644 --- a/test/test_simple.jl +++ b/test/test_simple.jl @@ -54,12 +54,12 @@ function simple_lowrank_model() [MOI.TriangleVectorization(A)], ), ) - c = MOI.add_constraint(model, X[4], MOI.EqualTo(1.0)) + c = MOI.add_constraint(model, 1.0 * X[1], MOI.EqualTo(1.0)) MOI.set(model, MOI.ObjectiveSense(), MOI.MIN_SENSE) - obj = 1.0 * X[1] + 1.0 * X[3] + obj = 1.0 * X[2] + 1.0 * X[4] MOI.set(model, MOI.ObjectiveFunction{typeof(obj)}(), obj) @test MOI.get(model, MOI.TerminationStatus()) == MOI.OPTIMIZE_NOT_CALLED - return model, X[1:3], c + return model, X[2:4], c end function simple_test(model, X, c) @@ -76,14 +76,14 @@ function simple_test(model, X, c) @test σ ≈ sigma end -@testset "MOI wrapper for $f" for f in [simple_sparse_model] #, simple_lowrank_model] +@testset "MOI wrapper for $f" for f in [simple_sparse_model, simple_lowrank_model] model, X, c = f() MOI.optimize!(model) simple_test(model, X, c) end function _test_limit(attr, val, term) - model, _, _ = simple_model() + model, _, _ = simple_sparse_model() MOI.set(model, MOI.RawOptimizerAttribute(attr), val) MOI.optimize!(model) @test MOI.get(model, MOI.TerminationStatus()) == term @@ -111,7 +111,7 @@ end end @testset "continuity between solve" begin - model, X, c = simple_model() + model, X, c = simple_sparse_model() MOI.set(model, MOI.RawOptimizerAttribute("majiter"), SDPLR.MAX_MAJITER - 2) @test MOI.get(model, MOI.RawOptimizerAttribute("majiter")) == SDPLR.MAX_MAJITER - 2 From c19b883531d5fb451a17717f83fa63461b9a2f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Wed, 10 Jul 2024 20:08:57 +0200 Subject: [PATCH 3/8] Fix --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index be306d1..aa903c3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,4 @@ include("test_vibra.jl") include("test_simple.jl") include("bounds.jl") -include("MOI_wrapper.jl") +#include("MOI_wrapper.jl") From 6eacfe08f6ce6e4d62897105555437b13cfaed0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Wed, 10 Jul 2024 20:10:04 +0200 Subject: [PATCH 4/8] fix --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index aa903c3..be306d1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,4 @@ include("test_vibra.jl") include("test_simple.jl") include("bounds.jl") -#include("MOI_wrapper.jl") +include("MOI_wrapper.jl") From c7f3c55cac435e4ccfac755e6519036bf5b2816c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Wed, 10 Jul 2024 20:11:14 +0200 Subject: [PATCH 5/8] Fix format --- src/MOI_wrapper.jl | 11 +++++------ test/test_simple.jl | 16 +++++++++++----- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/MOI_wrapper.jl b/src/MOI_wrapper.jl index 05b0ca7..d5c720f 100644 --- a/src/MOI_wrapper.jl +++ b/src/MOI_wrapper.jl @@ -19,11 +19,8 @@ const _SetWithDotProd = MOI.SetWithDotProducts{ MOI.TriangleVectorization{MOI.LowRankMatrix{Cdouble}}, } -const SupportedSets = Union{ - MOI.Nonnegatives, - MOI.PositiveSemidefiniteConeTriangle, - _SetWithDotProd, -} +const SupportedSets = + Union{MOI.Nonnegatives,MOI.PositiveSemidefiniteConeTriangle,_SetWithDotProd} mutable struct Optimizer <: MOI.AbstractOptimizer objective_constant::Float64 @@ -263,7 +260,9 @@ function _fill!( blk, i, j = model.varmap[t.variable.value] _fill_until(model, abs(blk), entptr, type, length(ent)) if type[end] == Cchar('l') - error("Can either have one dot product variable or several normal variables in the same constraint") + error( + "Can either have one dot product variable or several normal variables in the same constraint", + ) end if blk < 0 type[end] = Cchar('l') diff --git a/test/test_simple.jl b/test/test_simple.jl index 35fd493..641c6cd 100644 --- a/test/test_simple.jl +++ b/test/test_simple.jl @@ -4,7 +4,10 @@ import Random import MathOptInterface as MOI # This is `test_conic_PositiveSemidefiniteConeTriangle_VectorOfVariables` -@testset "Solve simple with sdplrlib with $file" for file in ["simple_sparse.jl", "simple_lowrank.jl"] +@testset "Solve simple with sdplrlib with $file" for file in [ + "simple_sparse.jl", + "simple_lowrank.jl", +] include(file) # The `925` seed is taken from SDPLR's `main.c` Random.seed!(925) @@ -43,9 +46,11 @@ end function simple_lowrank_model() model = SDPLR.Optimizer() A = MOI.LowRankMatrix( - [-1/4, 1/4], - [-1.0 1.0 - 1.0 1.0] + [-1 / 4, 1 / 4], + [ + -1.0 1.0 + 1.0 1.0 + ], ) X, _ = MOI.add_constrained_variables( model, @@ -76,7 +81,8 @@ function simple_test(model, X, c) @test σ ≈ sigma end -@testset "MOI wrapper for $f" for f in [simple_sparse_model, simple_lowrank_model] +@testset "MOI wrapper for $f" for f in + [simple_sparse_model, simple_lowrank_model] model, X, c = f() MOI.optimize!(model) simple_test(model, X, c) From 5c78d6e59aa084226870b56d7902f52466ed82bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Wed, 10 Jul 2024 20:12:24 +0200 Subject: [PATCH 6/8] Checkout MOI --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beb8d28..40d274a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,13 @@ jobs: ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- + - name: MOI + shell: julia --project=@. {0} + run: | + using Pkg + Pkg.add([ + PackageSpec(name="MathOptInterface", rev="master"), + ]) - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 From ef2cf54ff3b9cd782905dfeb127a301a249dca23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Wed, 10 Jul 2024 20:18:51 +0200 Subject: [PATCH 7/8] Update .github/workflows/ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40d274a..e623086 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: run: | using Pkg Pkg.add([ - PackageSpec(name="MathOptInterface", rev="master"), + PackageSpec(name="MathOptInterface", rev="bl/lmi"), ]) - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 From 1f33f130c2f089c21a211ab0d1146eb56b7d77ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Legat?= Date: Thu, 11 Jul 2024 23:28:27 +0200 Subject: [PATCH 8/8] Fixes --- src/SDPLR.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/SDPLR.jl b/src/SDPLR.jl index 285f3a5..7d3f03b 100644 --- a/src/SDPLR.jl +++ b/src/SDPLR.jl @@ -129,14 +129,17 @@ function solve( k += 1 @assert CAinfo_entptr[k] <= CAinfo_entptr[k+1] for j in ((CAinfo_entptr[k]+1):CAinfo_entptr[k+1]) - @assert blktype[blk] == CAinfo_type[k] @assert 1 <= CArow[j] <= blksz[blk] @assert 1 <= CAcol[j] <= blksz[blk] if CAinfo_type[k] == Cchar('s') + @assert blktype[blk] == Cchar('s') @assert CArow[j] <= CAcol[j] - else - @assert CAinfo_type[k] == Cchar('d') + elseif CAinfo_type[k] == Cchar('d') + @assert blktype[blk] == Cchar('d') @assert CArow[j] == CAcol[j] + else + @assert CAinfo_type[k] == Cchar('l') + @assert blktype[blk] == Cchar('s') end end end