diff --git a/pyomo/common/unittest.py b/pyomo/common/unittest.py index e528df67fe6..513dc4772bb 100644 --- a/pyomo/common/unittest.py +++ b/pyomo/common/unittest.py @@ -876,10 +876,11 @@ def filter_fcn(self, line): # next 6 patterns ignore entries in pstats reports: 'function calls', 'List reduced', - '.py:', + '.py:', # timing/profiling output ' {built-in method', ' {method', ' {pyomo.core.expr.numvalue.as_numeric}', + ' {gurobipy.', ): if field in line: return True diff --git a/pyomo/gdp/plugins/multiple_bigm.py b/pyomo/gdp/plugins/multiple_bigm.py index 3362276246b..9ee5c9180ff 100644 --- a/pyomo/gdp/plugins/multiple_bigm.py +++ b/pyomo/gdp/plugins/multiple_bigm.py @@ -72,6 +72,14 @@ } +def Solver(val): + if isinstance(val, str): + return SolverFactory(val) + if not hasattr(val, 'solve'): + raise ValueError("Expected a string or solver object (with solve() method)") + return val + + @TransformationFactory.register( 'gdp.mbigm', doc="Relax disjunctive model using big-M terms specific to each disjunct", @@ -127,7 +135,8 @@ class MultipleBigMTransformation(GDP_to_MIP_Transformation, _BigM_MixIn): CONFIG.declare( 'solver', ConfigValue( - default=SolverFactory('gurobi'), + default='gurobi', + domain=Solver, description="A solver to use to solve the continuous subproblems for " "calculating the M values", ), diff --git a/pyomo/solvers/plugins/solvers/GUROBI.py b/pyomo/solvers/plugins/solvers/GUROBI.py index 3a3a4d52322..2a6ee8b676e 100644 --- a/pyomo/solvers/plugins/solvers/GUROBI.py +++ b/pyomo/solvers/plugins/solvers/GUROBI.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import io import os import sys import re @@ -18,9 +19,12 @@ from pyomo.common import Executable from pyomo.common.collections import Bunch +from pyomo.common.dependencies import attempt_import from pyomo.common.enums import maximize, minimize +from pyomo.common.errors import ApplicationError from pyomo.common.fileutils import this_file_dir -from pyomo.common.tee import capture_output +from pyomo.common.log import is_debug_set +from pyomo.common.tee import capture_output, TeeStream from pyomo.common.tempfiles import TempfileManager from pyomo.opt.base import ProblemFormat, ResultsFormat, OptSolver @@ -35,10 +39,11 @@ from pyomo.core.kernel.block import IBlock from pyomo.core import ConcreteModel, Var, Objective -from .gurobi_direct import gurobipy_available -from .ASL import ASL +from pyomo.solvers.plugins.solvers.gurobi_direct import gurobipy, gurobipy_available +from pyomo.solvers.plugins.solvers.ASL import ASL logger = logging.getLogger('pyomo.solvers') +GUROBI_RUN = attempt_import('pyomo.solvers.plugins.solvers.GUROBI_RUN')[0] @SolverFactory.register('gurobi', doc='The GUROBI LP/MIP solver') @@ -51,9 +56,15 @@ def __new__(cls, *args, **kwds): mode = 'lp' # if mode == 'lp': - return SolverFactory('_gurobi_shell', **kwds) + if gurobipy_available: + return SolverFactory('_gurobi_file', **kwds) + else: + return SolverFactory('_gurobi_shell', **kwds) if mode == 'mps': - opt = SolverFactory('_gurobi_shell', **kwds) + if gurobipy_available: + opt = SolverFactory('_gurobi_file', **kwds) + else: + opt = SolverFactory('_gurobi_shell', **kwds) opt.set_problem_format(ProblemFormat.mps) return opt if mode in ['python', 'direct']: @@ -192,8 +203,7 @@ def _warm_start(self, instance): # for each variable in the symbol_map, add a child to the # variables element. Both continuous and discrete are accepted - # (and required, depending on other options), according to the - # CPLEX manual. + # (and required, depending on other options). # # **Note**: This assumes that the symbol_map is "clean", i.e., # contains only references to the variables encountered in @@ -318,8 +328,6 @@ def _get_version(self): def create_command_line(self, executable, problem_files): # # Define log file - # The log file in CPLEX contains the solution trace, but the - # solver status can be found in the solution file. # if self._log_file is None: self._log_file = TempfileManager.create_tempfile(suffix='.gurobi.log') @@ -341,11 +349,10 @@ def create_command_line(self, executable, problem_files): warmstart_filename = self._warm_start_file_name # translate the options into a normal python dictionary, from a - # pyutilib SectionWrapper - the gurobi_run function doesn't know - # about pyomo, so the translation is necessary. - options_dict = {} - for key in self.options: - options_dict[key] = self.options[key] + # pyomo.common.collections.Bunch - the gurobi_run function + # doesn't know about pyomo, so the translation is necessary + # (`repr(options)` doesn't produce executable python code) + options_dict = dict(self.options) # NOTE: the gurobi shell is independent of Pyomo python # virtualized environment, so any imports - specifically @@ -354,21 +361,20 @@ def create_command_line(self, executable, problem_files): # NOTE: The gurobi plugin (GUROBI.py) and GUROBI_RUN.py live in # the same directory. script = "import sys\n" - script += "from gurobipy import *\n" script += "sys.path.append(%r)\n" % (this_file_dir(),) - script += "from GUROBI_RUN import *\n" - script += "gurobi_run(" + script += "import GUROBI_RUN\n" + script += "soln = GUROBI_RUN.gurobi_run(" mipgap = float(self.options.mipgap) if self.options.mipgap is not None else None for x in ( problem_filename, warmstart_filename, - solution_filename, None, options_dict, self._suffixes, ): script += "%r," % x script += ")\n" + script += "GUROBI_RUN.write_result(soln, %r)\n" % solution_filename script += "quit()\n" # dump the script and warm-start file names for the @@ -392,11 +398,10 @@ def create_command_line(self, executable, problem_files): return Bunch(cmd=cmd, script=script, log_file=self._log_file, env=None) def process_soln_file(self, results): - # the only suffixes that we extract from CPLEX are - # constraint duals, constraint slacks, and variable - # reduced-costs. scan through the solver suffix list - # and throw an exception if the user has specified - # any others. + # the only suffixes that we extract are constraint duals, + # constraint slacks, and variable reduced-costs. scan through + # the solver suffix list and throw an exception if the user has + # specified any others. extract_duals = False extract_slacks = False extract_rc = False @@ -588,3 +593,233 @@ def _postsolve(self): TempfileManager.pop(remove=not self._keepfiles) return results + + +@SolverFactory.register( + '_gurobi_file', doc='LP/MPS file-based direct interface to the GUROBI LP/MIP solver' +) +class GUROBIFILE(GUROBISHELL): + """Direct LP/MPS file-based interface to the GUROBI LP/MIP solver""" + + def available(self, exception_flag=False): + if not gurobipy_available: # this triggers the deferred import + if exception_flag: + raise ApplicationError("gurobipy module not importable") + return False + if getattr(self, '_available', None) is None: + self._check_license() + ans = self._available[0] + if exception_flag and not ans: + raise ApplicationError(msg % self.name) + return ans + + def license_is_valid(self): + return self.available(False) and self._available[1] + + def _check_license(self): + licensed = False + try: + # Gurobipy writes out license file information when creating + # the environment + with capture_output(capture_fd=True): + m = gurobipy.Model() + licensed = True + except gurobipy.GurobiError: + licensed = False + + self._available = (True, licensed) + + def _get_version(self): + return ( + gurobipy.GRB.VERSION_MAJOR, + gurobipy.GRB.VERSION_MINOR, + gurobipy.GRB.VERSION_TECHNICAL, + ) + + def _default_executable(self): + # Bogus, but not None (because the test infrastructure disables + # solvers where the executable() is None) + return "" + + def create_command_line(self, executable, problem_files): + # + # Define log file + # + if self._log_file is None: + self._log_file = TempfileManager.create_tempfile(suffix='.gurobi.log') + + # + # Define command line + # + return Bunch(cmd=[], script="", log_file=self._log_file, env=None) + + def _apply_solver(self): + # + # Execute the command + # + if is_debug_set(logger): + logger.debug("Running %s", self._command.cmd) + + problem_filename = self._problem_files[0] + warmstart_filename = self._warm_start_file_name + + # translate the options into a normal python dictionary, from a + # pyutilib SectionWrapper - because the gurobi_run function was + # originally designed to run in the Python environment + # distributed in the Gurobi installation (which doesn't know + # about pyomo) the translation is necessary. + options_dict = {} + for key in self.options: + options_dict[key] = self.options[key] + + # display the log/solver file names prior to execution. this is useful + # in case something crashes unexpectedly, which is not without precedent. + if self._keepfiles: + if self._log_file is not None: + print("Solver log file: '%s'" % self._log_file) + if self._problem_files != []: + print("Solver problem files: %s" % str(self._problem_files)) + + sys.stdout.flush() + ostreams = [io.StringIO()] + if self._tee: + ostreams.append(sys.stdout) + with TeeStream(*ostreams) as t: + with capture_output(output=t.STDOUT, capture_fd=False): + self._soln = GUROBI_RUN.gurobi_run( + problem_filename, + warmstart_filename, + None, + options_dict, + self._suffixes, + ) + self._log = ostreams[0].getvalue() + self._rc = 0 + sys.stdout.flush() + return Bunch(rc=self._rc, log=self._log) + + def process_soln_file(self, results): + # the only suffixes that we extract are constraint duals, + # constraint slacks, and variable reduced-costs. Scan through + # the solver suffix list and throw an exception if the user has + # specified any others. + extract_duals = False + extract_slacks = False + extract_rc = False + for suffix in self._suffixes: + flag = False + if re.match(suffix, "dual"): + extract_duals = True + flag = True + if re.match(suffix, "slack"): + extract_slacks = True + flag = True + if re.match(suffix, "rc"): + extract_rc = True + flag = True + if not flag: + raise RuntimeError( + "***The GUROBI solver plugin cannot extract solution suffix=" + + suffix + ) + + soln = Solution() + + # caching for efficiency + soln_variables = soln.variable + soln_constraints = soln.constraint + + num_variables_read = 0 + + # string compares are too expensive, so simply introduce some + # section IDs. + # 0 - unknown + # 1 - problem + # 2 - solution + # 3 - solver + + section = 0 # unknown + + solution_seen = False + + range_duals = {} + range_slacks = {} + + # Copy over the problem info + for key, val in self._soln['problem'].items(): + setattr(results.problem, key, val) + if results.problem.sense == 'minimize': + results.problem.sense = minimize + elif results.problem.sense == 'maximize': + results.problem.sense = maximize + + # Copy over the solver info + for key, val in self._soln['solver'].items(): + setattr(results.solver, key, val) + results.solver.status = getattr(SolverStatus, results.solver.status) + try: + results.solver.termination_condition = getattr( + TerminationCondition, results.solver.termination_condition + ) + except AttributeError: + results.solver.termination_condition = TerminationCondition.unknown + + # Copy over the solution information + sol = self._soln.get('solution', None) + if sol: + if 'status' in sol: + soln.status = sol['status'] + if 'gap' in sol: + soln.gap = sol['gap'] + obj = sol.get('objective', None) + if obj is not None: + soln.objective['__default_objective__'] = {'Value': obj} + if results.problem.sense == minimize: + results.problem.upper_bound = obj + else: + results.problem.lower_bound = obj + for name, val in sol.get('var', {}).items(): + if name == "ONE_VAR_CONSTANT": + continue + soln_variables[name] = {"Value": val} + num_variables_read += 1 + for name, val in sol.get('varrc', {}).items(): + if name == "ONE_VAR_CONSTANT": + continue + soln_variables[name]["Rc"] = val + for name, val in sol.get('constraintdual', {}).items(): + if name == "c_e_ONE_VAR_CONSTANT": + continue + if name.startswith('c_'): + soln_constraints.setdefault(name, {})["Dual"] = val + elif name.startswith('r_l_'): + range_duals.setdefault(name[4:], [0, 0])[0] = val + elif name.startswith('r_u_'): + range_duals.setdefault(name[4:], [0, 0])[1] = val + for name, val in sol.get('constraintslack', {}).items(): + if name == "c_e_ONE_VAR_CONSTANT": + continue + if name.startswith('c_'): + soln_constraints.setdefault(name, {})["Slack"] = val + elif name.startswith('r_l_'): + range_slacks.setdefault(name[4:], [0, 0])[0] = val + elif name.startswith('r_u_'): + range_slacks.setdefault(name[4:], [0, 0])[1] = val + + results.solution.insert(soln) + + # For the range constraints, supply only the dual with the largest + # magnitude (at least one should always be numerically zero) + for key, (ld, ud) in range_duals.items(): + if abs(ld) > abs(ud): + soln_constraints['r_l_' + key] = {"Dual": ld} + else: + # Use the same key + soln_constraints['r_l_' + key] = {"Dual": ud} + # slacks + for key, (ls, us) in range_slacks.items(): + if abs(ls) > abs(us): + soln_constraints.setdefault('r_l_' + key, {})["Slack"] = ls + else: + # Use the same key + soln_constraints.setdefault('r_l_' + key, {})["Slack"] = us diff --git a/pyomo/solvers/plugins/solvers/GUROBI_RUN.py b/pyomo/solvers/plugins/solvers/GUROBI_RUN.py index 88f953e18ae..0de1a61266e 100644 --- a/pyomo/solvers/plugins/solvers/GUROBI_RUN.py +++ b/pyomo/solvers/plugins/solvers/GUROBI_RUN.py @@ -13,9 +13,9 @@ import re -""" -This script is run using the Gurobi/system python. Do not assume any third party packages -are available! +"""This script is run using the Gurobi/system python. Do not assume any +third party packages are available! + """ from gurobipy import gurobi, read, GRB import sys @@ -40,7 +40,7 @@ def _is_numeric(x): return True -def gurobi_run(model_file, warmstart_file, soln_file, mipgap, options, suffixes): +def gurobi_run(model_file, warmstart_file, mipgap, options, suffixes): # figure out what suffixes we need to extract. extract_duals = False extract_slacks = False @@ -77,7 +77,7 @@ def gurobi_run(model_file, warmstart_file, soln_file, mipgap, options, suffixes) if model is None: print( - "***The GUROBI solver plugin failed to load the input LP file=" + soln_file + "***The GUROBI solver plugin failed to load the input LP file=" + model_file ) return @@ -107,6 +107,7 @@ def gurobi_run(model_file, warmstart_file, soln_file, mipgap, options, suffixes) # because the latter does not preserve the # Gurobi stack trace if not _is_numeric(value): + model.close() raise model.setParam(key, float(value)) @@ -239,13 +240,9 @@ def gurobi_run(model_file, warmstart_file, soln_file, mipgap, options, suffixes) # minimize obj_value = float('inf') - # write the solution file - solnfile = open(soln_file, "w+") - - # write the information required by results.problem - solnfile.write("section:problem\n") - name = model.getAttr(GRB.Attr.ModelName) - solnfile.write("name: " + name + '\n') + result = {} + problem = result['problem'] = {} + problem['name'] = model.getAttr(GRB.Attr.ModelName) # TODO: find out about bounds and fix this with error checking # this line fails for some reason so set the value to unknown @@ -258,97 +255,103 @@ def gurobi_run(model_file, warmstart_file, soln_file, mipgap, options, suffixes) bound = None if sense < 0: - solnfile.write("sense:maximize\n") + problem["sense"] = "maximize" if bound is None: - solnfile.write("upper_bound: %f\n" % float('inf')) - else: - solnfile.write("upper_bound: %s\n" % str(bound)) + bound = float('inf') + problem["upper_bound"] = bound else: - solnfile.write("sense:minimize\n") + problem["sense"] = "minimize" if bound is None: - solnfile.write("lower_bound: %f\n" % float('-inf')) - else: - solnfile.write("lower_bound: %s\n" % str(bound)) + bound = float('-inf') + problem["lower_bound"] = bound # TODO: Get the number of objective functions from GUROBI n_objs = 1 - solnfile.write("number_of_objectives: %d\n" % n_objs) + problem["number_of_objectives"] = n_objs cons = model.getConstrs() qcons = [] if GUROBI_VERSION[0] >= 5: qcons = model.getQConstrs() - solnfile.write( - "number_of_constraints: %d\n" % (len(cons) + len(qcons) + model.NumSOS,) - ) + problem["number_of_constraints"] = len(cons) + len(qcons) + model.NumSOS vars = model.getVars() - solnfile.write("number_of_variables: %d\n" % len(vars)) + problem["number_of_variables"] = len(vars) n_binvars = model.getAttr(GRB.Attr.NumBinVars) - solnfile.write("number_of_binary_variables: %d\n" % n_binvars) + problem["number_of_binary_variables"] = n_binvars n_intvars = model.getAttr(GRB.Attr.NumIntVars) - solnfile.write("number_of_integer_variables: %d\n" % n_intvars) - - solnfile.write("number_of_continuous_variables: %d\n" % (len(vars) - n_intvars,)) - - solnfile.write("number_of_nonzeros: %d\n" % model.getAttr(GRB.Attr.NumNZs)) + problem["number_of_integer_variables"] = n_intvars + problem["number_of_continuous_variables"] = len(vars) - n_intvars + problem["number_of_nonzeros"] = model.getAttr(GRB.Attr.NumNZs) # write out the information required by results.solver - solnfile.write("section:solver\n") + solver = result['solver'] = {} - solnfile.write('status: %s\n' % status) - solnfile.write('return_code: %s\n' % return_code) - solnfile.write('message: %s\n' % message) - solnfile.write('wall_time: %s\n' % str(wall_time)) - solnfile.write('termination_condition: %s\n' % term_cond) - solnfile.write('termination_message: %s\n' % message) + solver['status'] = status + solver['return_code'] = return_code + solver['message'] = message + solver['wall_time'] = wall_time + solver['termination_condition'] = term_cond + solver['termination_message'] = message is_discrete = False if model.getAttr(GRB.Attr.IsMIP): is_discrete = True if (term_cond == 'optimal') or (model.getAttr(GRB.Attr.SolCount) >= 1): - solnfile.write('section:solution\n') - solnfile.write('status: %s\n' % (solution_status)) - solnfile.write('message: %s\n' % message) - solnfile.write('objective: %s\n' % str(obj_value)) - solnfile.write('gap: 0.0\n') + solution = result['solution'] = {} + solution['status'] = solution_status + solution['message'] = message + solution['objective'] = obj_value + solution['gap'] = 0.0 vals = model.getAttr("X", vars) names = model.getAttr("VarName", vars) - for val, name in zip(vals, names): - solnfile.write('var: %s : %s\n' % (str(name), str(val))) + solution['var'] = {name: val for name, val in zip(names, vals)} - if (is_discrete is False) and (extract_reduced_costs is True): + if extract_reduced_costs and not is_discrete: vals = model.getAttr("Rc", vars) - for val, name in zip(vals, names): - solnfile.write('varrc: %s : %s\n' % (str(name), str(val))) + solution['varrc'] = {name: val for name, val in zip(names, vals)} if extract_duals or extract_slacks: con_names = model.getAttr("ConstrName", cons) if GUROBI_VERSION[0] >= 5: qcon_names = model.getAttr("QCName", qcons) - if (is_discrete is False) and (extract_duals is True): + if extract_duals and not is_discrete: + # Pi attributes in Gurobi are the constraint duals vals = model.getAttr("Pi", cons) - for val, name in zip(vals, con_names): - # Pi attributes in Gurobi are the constraint duals - solnfile.write("constraintdual: %s : %s\n" % (str(name), str(val))) + solution['constraintdual'] = { + name: val for name, val in zip(con_names, vals) + } if GUROBI_VERSION[0] >= 5: + # QCPI attributes in Gurobi are the constraint duals vals = model.getAttr("QCPi", qcons) - for val, name in zip(vals, qcon_names): - # QCPI attributes in Gurobi are the constraint duals - solnfile.write("constraintdual: %s : %s\n" % (str(name), str(val))) + solution['constraintdual'].update(zip(qcon_names, vals)) - if extract_slacks is True: + if extract_slacks: vals = model.getAttr("Slack", cons) - for val, name in zip(vals, con_names): - solnfile.write("constraintslack: %s : %s\n" % (str(name), str(val))) + solution['constraintslack'] = { + name: val for name, val in zip(con_names, vals) + } if GUROBI_VERSION[0] >= 5: vals = model.getAttr("QCSlack", qcons) - for val, name in zip(vals, qcon_names): - solnfile.write("constraintslack: %s : %s\n" % (str(name), str(val))) - - solnfile.close() + solution['constraintslack'].update(zip(qcon_names, vals)) + + model.close() + model = None + return result + + +def write_result(result, soln_file): + with open(soln_file, "w+") as FILE: + for section, data in result.items(): + FILE.write(f'section:{section}\n') + for key, val in data.items(): + if val.__class__ is dict: + for name, v in val.items(): + FILE.write(f'{key}:{name}:{v}\n') + else: + FILE.write(f'{key}:{val}\n') diff --git a/pyomo/solvers/plugins/solvers/gurobi_direct.py b/pyomo/solvers/plugins/solvers/gurobi_direct.py index 9cd81ba8a55..cdb04b63dec 100644 --- a/pyomo/solvers/plugins/solvers/gurobi_direct.py +++ b/pyomo/solvers/plugins/solvers/gurobi_direct.py @@ -70,6 +70,7 @@ def _parse_gurobi_version(gurobipy, avail): # exception! catch_exceptions=(Exception,), callback=_parse_gurobi_version, + defer_import=True, ) diff --git a/pyomo/solvers/tests/checks/test_gurobi.py b/pyomo/solvers/tests/checks/test_gurobi.py index e87685a046c..580f6f3b714 100644 --- a/pyomo/solvers/tests/checks/test_gurobi.py +++ b/pyomo/solvers/tests/checks/test_gurobi.py @@ -9,11 +9,12 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import io import pyomo.common.unittest as unittest from unittest.mock import patch, MagicMock try: - from pyomo.solvers.plugins.solvers.GUROBI_RUN import gurobi_run + from pyomo.solvers.plugins.solvers.GUROBI_RUN import gurobi_run, write_result from gurobipy import GRB gurobipy_available = True @@ -26,11 +27,8 @@ @unittest.skipIf(not gurobipy_available, "gurobipy is not available") class GurobiTest(unittest.TestCase): @unittest.skipIf(not has_worklimit, "gurobi < 9.5") - @patch("builtins.open") @patch("pyomo.solvers.plugins.solvers.GUROBI_RUN.read") - def test_work_limit(self, read: MagicMock, open: MagicMock): - file = MagicMock() - open.return_value = file + def test_work_limit(self, read: MagicMock): model = MagicMock() read.return_value = model @@ -49,8 +47,8 @@ def getAttr(attr): return None model.getAttr = getAttr - gurobi_run(None, None, None, None, {}, []) - self.assertTrue("WorkLimit" in file.write.call_args[0][0]) + result = gurobi_run(None, None, None, {}, []) + self.assertIn("WorkLimit", result['solver']['message']) if __name__ == '__main__':