Upload benchmark results

.gitignore

@@ -131,4 +131,4 @@ dmypy.json
 # Pycharm
 .idea/
 
-gdplib/*/benchmark_result/
+gdplib/*/benchmark_result/*.log

benchmark.py

@@ -5,9 +5,13 @@
 from datetime import datetime
 from importlib import import_module
 from pyomo.environ import *
+import sys
+from contextlib import redirect_stdout
 
 
-def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
+def benchmark(
+    model, strategy, timelimit, result_dir, subsolver="gams", solver_gams="baron"
+):
     """Benchmark the model using the given strategy and subsolver.
 
     The result files include the solver output and the JSON representation of the results.
@@ -22,27 +26,44 @@ def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
         the time limit for the solver
     result_dir : string
         the directory to store the benchmark results
+    subsolver : string
+        the subsolver used to solve the model
+    solver_gams : string
+        the solver used to solve the model in GAMS
 
     Returns
     -------
     None
     """
+    # We clone the model to avoid the solver starting from the optimal solution from previous solve.
     model = model.clone()
-    stdout = sys.stdout
+
+    # Direct the solver output to a file
     if strategy in ["gdp.bigm", "gdp.hull"]:
-        transformation_start_time = time.time()
-        TransformationFactory(strategy).apply_to(model)
-        transformation_end_time = time.time()
         with open(
-            result_dir + "/" + strategy + "_" + subsolver + ".log", "w"
-        ) as sys.stdout:
-            results = SolverFactory(subsolver).solve(
-                model, tee=True, timelimit=timelimit
-            )
-            results.solver.transformation_time = (
-                transformation_end_time - transformation_start_time
-            )
-            print(results)
+            result_dir + "/" + strategy + "_" + subsolver + "_" + solver_gams + ".log",
+            "w",
+        ) as f:
+            with redirect_stdout(f):
+                transformation_start_time = time.time()
+                TransformationFactory(strategy).apply_to(model)
+                transformation_end_time = time.time()
+                results = SolverFactory(subsolver).solve(
+                    model,
+                    tee=True,
+                    solver=solver_gams,
+                    add_options=[
+                        "option reslim=3600;option threads=1;option optcr=1e-2;"
+                    ],
+                    # keepfiles=True,
+                    # tmpdir=os.path.join(result_dir, strategy, "nlp"),
+                    # symbolic_solver_labels=True,
+                )
+                results.solver.strategy = strategy
+                results.solver.transformation_time = (
+                    transformation_end_time - transformation_start_time
+                )
+                print(results)
     elif strategy in [
         "gdpopt.enumerate",
         "gdpopt.loa",
@@ -51,60 +72,161 @@ def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
         "gdpopt.ric",
     ]:
         with open(
-            result_dir + "/" + strategy + "_" + subsolver + ".log", "w"
-        ) as sys.stdout:
-            results = SolverFactory(strategy).solve(
-                model,
-                tee=True,
-                nlp_solver=subsolver,
-                mip_solver=subsolver,
-                minlp_solver=subsolver,
-                local_minlp_solver=subsolver,
-                time_limit=timelimit,
-            )
-            print(results)
+            result_dir + "/" + strategy + "_" + subsolver + "_" + solver_gams + ".log",
+            "w",
+        ) as f:
+            with redirect_stdout(f):
+                results = SolverFactory(strategy).solve(
+                    model,
+                    tee=True,
+                    # bound_tolerance=1e-2,  # default is 1e-6
+                    nlp_solver=subsolver,
+                    nlp_solver_args=dict(
+                        solver=solver_gams,
+                        add_options=[
+                            "option threads=1;",
+                            '$onecho > baron.opt',
+                            'FirstLoc 1',
+                            # # 'nlpsol 9', #9: IPOPT, 6: GAMS NLP solver, default -1: Automatic NLP solver selection and switching strategy
+                            # 'optcr 1.e-2',
+                            # # 'optca 1.e-2',
+                            # #'maxiter 0', # force BARON to terminate after root node preprocessing
+                            # # 'maxiter 1', # termination after the solution of the root node
+                            # # 'maxiter 1e3',
+                            # #'numloc -1', # local searches in preprocessing from randomly generated starting points until global optimality is proved or MaxTime seconds have elapsed.
+                            # #'reslim 760',
+                            '$offecho',
+                            'GAMS_MODEL.optfile=1',
+                            "option optcr=1e-2;",
+                        ],
+                        # keepfiles=True,
+                        # tmpdir=os.path.join(result_dir, strategy, "nlp"),
+                        # symbolic_solver_labels=True,
+                        # logfile=result_dir + "/" + strategy + "_" + "nlp" + ".log",
+                        tee=True,
+                    ),
+                    mip_solver=subsolver,
+                    mip_solver_args=dict(add_options=["option threads=1"], tee=True),
+                    minlp_solver=subsolver,
+                    minlp_solver_args=dict(
+                        solver=solver_gams,
+                        add_options=[
+                            "option threads=1;",
+                            # '$onecho > baron.opt',
+                            # # 'FirstLoc 1',
+                            # # 'nlpsol 9', #9: IPOPT, 6: GAMS NLP solver, default -1: Automatic NLP solver selection and switching strategy
+                            # 'optcr 1.e-2',
+                            # # 'optca 1.e-2',
+                            # #'maxiter 0', # force BARON to terminate after root node preprocessing
+                            # # 'maxiter 1', # termination after the solution of the root node
+                            # # 'maxiter 1e3',
+                            # #'numloc -1', # local searches in preprocessing from randomly generated starting points until global optimality is proved or MaxTime seconds have elapsed.
+                            # #'reslim 760',
+                            # '$offecho',
+                            # 'GAMS_MODEL.optfile=1',
+                            "option optcr=1e-6;",
+                        ],
+                        tee=True,
+                        # keepfiles=True,
+                        # tmpdir=os.path.join(result_dir, strategy, "minlp"),
+                        # symbolic_solver_labels=True,
+                    ),
+                    local_minlp_solver=subsolver,
+                    local_minlp_solver_args=dict(
+                        solver=solver_gams,
+                        add_options=[
+                            "option threads=1;",
+                            '$onecho > baron.opt',
+                            'FirstLoc 1',
+                            # # 'nlpsol 9', #9: IPOPT, 6: GAMS NLP solver, default -1: Automatic NLP solver selection and switching strategy
+                            # 'optcr 1.e-2',
+                            # # 'optca 1.e-2',
+                            # #'maxiter 0', # force BARON to terminate after root node preprocessing
+                            # # 'maxiter 1', # termination after the solution of the root node
+                            # # 'maxiter 1e3',
+                            # #'numloc -1', # local searches in preprocessing from randomly generated starting points until global optimality is proved or MaxTime seconds have elapsed.
+                            # #'reslim 760',
+                            '$offecho',
+                            'GAMS_MODEL.optfile=1',
+                            "option optcr=1e-2;",
+                        ],
+                        tee=True,
+                        # keepfiles=True,
+                        # tmpdir=os.path.join(result_dir, strategy, "local_minlp"),
+                        # symbolic_solver_labels=True,
+                    ),
+                    time_limit=timelimit,
+                )
+                # results.solver.strategy = strategy
+                print(results)
 
-    sys.stdout = stdout
-    with open(result_dir + "/" + strategy + "_" + subsolver + ".json", "w") as f:
+    with open(
+        result_dir + "/" + strategy + "_" + subsolver + "_" + solver_gams + ".json", "w"
+    ) as f:
         json.dump(results.json_repn(), f)
     return None
 
 
 if __name__ == "__main__":
     instance_list = [
         # "batch_processing",
-        # "biofuel",
+        # "biofuel",  # enumeration got stuck
+        # "cstr",
         # "disease_model",
+        "ex1_linan_2023",
         # "gdp_col",
         # "hda",
-        "jobshop",
+        # "jobshop",
         # "kaibel",
-        # "positioning",
-        # "spectralog",
         # "med_term_purchasing",
         # "methanol",
         # "mod_hens",
         # "modprodnet",
+        # "positioning",
+        # "small_batch",
+        # "spectralog",
         # "stranded_gas",
-        # "syngas",
+        # "syngas"
     ]
     strategy_list = [
-        "gdp.bigm",
-        "gdp.hull",
+        # "gdp.bigm",
+        # "gdp.hull",
         "gdpopt.enumerate",
-        "gdpopt.loa",
-        "gdpopt.gloa",
-        "gdpopt.ric",
+        # "gdpopt.loa",
+        # "gdpopt.gloa",
+        # "gdpopt.ric",
+        # "gdpopt.lbb",
+    ]
+    solver_gams_list = [
+        'baron',
+        # 'scip'
     ]
     current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-    timelimit = 600
+    timelimit = 3600
 
     for instance in instance_list:
-        print("Benchmarking instance: " + instance)
-        result_dir = "gdplib/" + instance + "/benchmark_result/"
+        result_dir = "gdplib/" + instance + "/benchmark_result/" + current_time
         os.makedirs(result_dir, exist_ok=True)
 
+        print("Benchmarking instance: ", instance)
         model = import_module("gdplib." + instance).build_model()
 
         for strategy in strategy_list:
-            benchmark(model, strategy, timelimit, result_dir)
+            for solver_gams in solver_gams_list:
+                if os.path.exists(
+                    result_dir
+                    + "/"
+                    + strategy
+                    + "_"
+                    + "gams"
+                    + "_"
+                    + solver_gams
+                    + ".json"
+                ):
+                    continue
+                try:
+                    benchmark(
+                        model, strategy, timelimit, result_dir, "gams", solver_gams
+                    )
+                except:
+                    pass

gdplib/batch_processing/benchmark_result/gdp.bigm_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "/tmp/tmpwrtwe4_v/model.gms", "Lower bound": 679365.3232855867, "Upper bound": 679365.3232855867, "Number of objectives": 1, "Number of constraints": 791.0, "Number of variables": 288.0, "Number of binary variables": null, "Number of integer variables": 138.0, "Number of continuous variables": 150.0, "Number of nonzeros": 2528.0, "Sense": "minimize"}], "Solver": [{"Name": "GAMS (44, 3, 0, 0)", "Status": "ok", "Return code": 0, "Message": null, "User time": 42.391000455245376, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.012274026870727539}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}

gdplib/batch_processing/benchmark_result/gdp.hull_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "/tmp/tmpx01tl19l/model.gms", "Lower bound": 679365.3345204662, "Upper bound": 679365.3345204662, "Number of objectives": 1, "Number of constraints": 1286.0, "Number of variables": 666.0, "Number of binary variables": null, "Number of integer variables": 138.0, "Number of continuous variables": 528.0, "Number of nonzeros": 3527.0, "Sense": "minimize"}], "Solver": [{"Name": "GAMS (44, 3, 0, 0)", "Status": "ok", "Return code": 0, "Message": null, "User time": 48.421000060625374, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.10417389869689941}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}

gdplib/batch_processing/benchmark_result/gdpopt.enumerate_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.3345666251, "Upper bound": 679365.3345666251, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - enumerate", "Status": "ok", "User time": 867.9352501640096, "Wallclock time": 867.9352501640096, "Termination condition": "optimal", "Iterations": 512, "Timing": {"main_timer_start_time": 784170.502792807, "nlp": 867.8575911587104, "total": 867.9352501640096}}]}

gdplib/batch_processing/benchmark_result/gdpopt.lbb_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.3345090421, "Upper bound": 679365.3345090421, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - LBB", "Status": "ok", "User time": 203.50265267002396, "Wallclock time": 203.50265267002396, "Termination condition": "optimal", "Iterations": 19, "Timing": {"main_timer_start_time": 793897.053697714, "total": 203.50265267002396}}]}

gdplib/batch_processing/benchmark_result/gdpopt.loa_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 583342.938217141, "Upper bound": 679365.3345166193, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - LOA", "Status": "ok", "User time": 3600.005251016002, "Wallclock time": 3600.005251016002, "Termination condition": "maxTimeLimit", "Iterations": 107, "Timing": {"main_timer_start_time": 785038.463734412, "mip": 3389.1861617467366, "nlp": 210.6244352083886, "integer cut generation": 0.044262996525503695, "total": 3600.005251016002}}]}

gdplib/batch_processing/benchmark_result/gdpopt.ric_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.3345166193, "Upper bound": 679365.3345166193, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - RIC", "Status": "ok", "User time": 1449.0040160750505, "Wallclock time": 1449.0040160750505, "Termination condition": "optimal", "Iterations": 513, "Timing": {"main_timer_start_time": 792448.022091002, "mip": 355.0396141371457, "nlp": 1093.48349528166, "integer cut generation": 0.26000761275645345, "total": 1449.0040160750505}}]}

gdplib/batch_processing/benchmark_result/summary/gdp.bigm_gams_baron.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.3348692344, "Upper bound": 679365.3348692344, "Number of objectives": 1, "Number of constraints": 791.0, "Number of variables": 288.0, "Number of binary variables": null, "Number of integer variables": 138.0, "Number of continuous variables": 150.0, "Number of nonzeros": 2528.0, "Sense": "minimize"}], "Solver": [{"Name": "GAMS (47, 6, 0, 0) BM", "Status": "ok", "Return code": 0, "Message": null, "User time": 37.760000000707805, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.014738798141479492}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}

gdplib/batch_processing/benchmark_result/summary/gdp.hull_gams_baron.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.3348692352, "Upper bound": 679365.3348692352, "Number of objectives": 1, "Number of constraints": 1286.0, "Number of variables": 666.0, "Number of binary variables": null, "Number of integer variables": 138.0, "Number of continuous variables": 528.0, "Number of nonzeros": 3527.0, "Sense": "minimize"}], "Solver": [{"Name": "GAMS (47, 6, 0, 0) H", "Status": "ok", "Return code": 0, "Message": null, "User time": 98.6950000282377, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.09286046028137207}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}

gdplib/batch_processing/benchmark_result/summary/gdpopt.enumerate_gams_baron.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.3348692338, "Upper bound": 679365.3348692338, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - enumerate", "Status": "ok", "User time": 3123.1129144933075, "Wallclock time": 3123.1129144933075, "Termination condition": "optimal", "Iterations": 512, "Timing": {"main_timer_start_time": 5703142.926052286, "nlp": 3123.048896836117, "total": 3123.1129144933075}}]}

gdplib/batch_processing/benchmark_result/summary/gdpopt.lbb_gams_baron.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.334860539, "Upper bound": 679365.334860539, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - LBB", "Status": "ok", "User time": 610.0009056953713, "Wallclock time": 610.0009056953713, "Termination condition": "optimal", "Iterations": 19, "Timing": {"main_timer_start_time": 5744858.435808686, "total": 610.0009056953713}}]}

gdplib/batch_processing/benchmark_result/summary/gdpopt.loa_gams_baron.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 90097.17657001011, "Upper bound": 679365.3348692361, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - LOA", "Status": "ok", "User time": 3600.142469989136, "Wallclock time": 3600.142469989136, "Termination condition": "maxTimeLimit", "Iterations": 428, "Timing": {"main_timer_start_time": 5737645.657170686, "mip": 205.6276945071295, "nlp": 3394.1985993934795, "integer cut generation": 0.142986205406487, "total": 3600.142469989136}}]}