Upload benchmark results

.gitignore

@@ -131,4 +131,4 @@ dmypy.json
 # Pycharm
 .idea/
 
-gdplib/*/benchmark_result/
+gdplib/*/benchmark_result/*.log

benchmark.py

@@ -5,6 +5,8 @@
 from datetime import datetime
 from importlib import import_module
 from pyomo.environ import *
+import sys
+from contextlib import redirect_stdout
 
 
 def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
@@ -27,44 +29,49 @@ def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
     -------
     None
     """
+    # We clone the model to avoid the solver starting from the optimal solution from previous solve.
     model = model.clone()
-    stdout = sys.stdout
+
+    # Direct the solver output to a file
     if strategy in ["gdp.bigm", "gdp.hull"]:
-        transformation_start_time = time.time()
-        TransformationFactory(strategy).apply_to(model)
-        transformation_end_time = time.time()
-        with open(
-            result_dir + "/" + strategy + "_" + subsolver + ".log", "w"
-        ) as sys.stdout:
-            results = SolverFactory(subsolver).solve(
-                model, tee=True, timelimit=timelimit
-            )
-            results.solver.transformation_time = (
-                transformation_end_time - transformation_start_time
-            )
-            print(results)
+        with open(result_dir + "/" + strategy + "_" + subsolver + ".log", "w") as f:
+            with redirect_stdout(f):
+                transformation_start_time = time.time()
+                TransformationFactory(strategy).apply_to(model)
+                transformation_end_time = time.time()
+                results = SolverFactory(subsolver).solve(
+                    model,
+                    tee=True,
+                    add_options=["option reslim=3600;option threads=1;"],
+                )
+                results.solver.transformation_time = (
+                    transformation_end_time - transformation_start_time
+                )
+                print(results)
     elif strategy in [
         "gdpopt.enumerate",
         "gdpopt.loa",
         "gdpopt.gloa",
         "gdpopt.lbb",
         "gdpopt.ric",
     ]:
-        with open(
-            result_dir + "/" + strategy + "_" + subsolver + ".log", "w"
-        ) as sys.stdout:
-            results = SolverFactory(strategy).solve(
-                model,
-                tee=True,
-                nlp_solver=subsolver,
-                mip_solver=subsolver,
-                minlp_solver=subsolver,
-                local_minlp_solver=subsolver,
-                time_limit=timelimit,
-            )
-            print(results)
+        with open(result_dir + "/" + strategy + "_" + subsolver + ".log", "w") as f:
+            with redirect_stdout(f):
+                results = SolverFactory(strategy).solve(
+                    model,
+                    tee=True,
+                    nlp_solver=subsolver,
+                    nlp_solver_args=dict(add_options=["option threads=1;"]),
+                    mip_solver=subsolver,
+                    mip_solver_args=dict(add_options=["option threads=1;"]),
+                    minlp_solver=subsolver,
+                    minlp_solver_args=dict(add_options=["option threads=1;"]),
+                    local_minlp_solver=subsolver,
+                    local_minlp_solver_args=dict(add_options=["option threads=1;"]),
+                    time_limit=timelimit,
+                )
+                print(results)
 
-    sys.stdout = stdout
     with open(result_dir + "/" + strategy + "_" + subsolver + ".json", "w") as f:
         json.dump(results.json_repn(), f)
     return None
@@ -73,20 +80,23 @@ def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
 if __name__ == "__main__":
     instance_list = [
         # "batch_processing",
-        # "biofuel",
-        # "disease_model",
-        # "gdp_col",
-        # "hda",
+        # "biofuel", # enumeration got stuck
+        # "cstr",
+        "disease_model",
+        "ex1_linan_2023",
+        "gdp_col",
+        "hda",
         "jobshop",
         # "kaibel",
-        # "positioning",
-        # "spectralog",
-        # "med_term_purchasing",
-        # "methanol",
-        # "mod_hens",
-        # "modprodnet",
-        # "stranded_gas",
-        # "syngas",
+        "med_term_purchasing",
+        "methanol",
+        "mod_hens",
+        "modprodnet",
+        "positioning",
+        "small_batch",
+        "spectralog",
+        "stranded_gas",
+        "syngas",
     ]
     strategy_list = [
         "gdp.bigm",
@@ -95,16 +105,22 @@ def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
         "gdpopt.loa",
         "gdpopt.gloa",
         "gdpopt.ric",
+        "gdpopt.lbb",
     ]
     current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-    timelimit = 600
+    timelimit = 3600
 
     for instance in instance_list:
-        print("Benchmarking instance: " + instance)
         result_dir = "gdplib/" + instance + "/benchmark_result/"
         os.makedirs(result_dir, exist_ok=True)
 
+        print("Benchmarking instance: ", instance)
         model = import_module("gdplib." + instance).build_model()
 
         for strategy in strategy_list:
-            benchmark(model, strategy, timelimit, result_dir)
+            if os.path.exists(result_dir + "/" + strategy + "_" + "gams" + ".json"):
+                continue
+            try:
+                benchmark(model, strategy, timelimit, result_dir, "gams")
+            except:
+                pass

gdplib/batch_processing/benchmark_result/gdp.bigm_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "/tmp/tmpwrtwe4_v/model.gms", "Lower bound": 679365.3232855867, "Upper bound": 679365.3232855867, "Number of objectives": 1, "Number of constraints": 791.0, "Number of variables": 288.0, "Number of binary variables": null, "Number of integer variables": 138.0, "Number of continuous variables": 150.0, "Number of nonzeros": 2528.0, "Sense": "minimize"}], "Solver": [{"Name": "GAMS (44, 3, 0, 0)", "Status": "ok", "Return code": 0, "Message": null, "User time": 42.391000455245376, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.012274026870727539}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}

gdplib/batch_processing/benchmark_result/gdp.hull_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "/tmp/tmpx01tl19l/model.gms", "Lower bound": 679365.3345204662, "Upper bound": 679365.3345204662, "Number of objectives": 1, "Number of constraints": 1286.0, "Number of variables": 666.0, "Number of binary variables": null, "Number of integer variables": 138.0, "Number of continuous variables": 528.0, "Number of nonzeros": 3527.0, "Sense": "minimize"}], "Solver": [{"Name": "GAMS (44, 3, 0, 0)", "Status": "ok", "Return code": 0, "Message": null, "User time": 48.421000060625374, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.10417389869689941}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}

gdplib/batch_processing/benchmark_result/gdpopt.enumerate_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.3345666251, "Upper bound": 679365.3345666251, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - enumerate", "Status": "ok", "User time": 867.9352501640096, "Wallclock time": 867.9352501640096, "Termination condition": "optimal", "Iterations": 512, "Timing": {"main_timer_start_time": 784170.502792807, "nlp": 867.8575911587104, "total": 867.9352501640096}}]}

gdplib/batch_processing/benchmark_result/gdpopt.lbb_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.3345090421, "Upper bound": 679365.3345090421, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - LBB", "Status": "ok", "User time": 203.50265267002396, "Wallclock time": 203.50265267002396, "Termination condition": "optimal", "Iterations": 19, "Timing": {"main_timer_start_time": 793897.053697714, "total": 203.50265267002396}}]}

gdplib/batch_processing/benchmark_result/gdpopt.loa_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 583342.938217141, "Upper bound": 679365.3345166193, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - LOA", "Status": "ok", "User time": 3600.005251016002, "Wallclock time": 3600.005251016002, "Termination condition": "maxTimeLimit", "Iterations": 107, "Timing": {"main_timer_start_time": 785038.463734412, "mip": 3389.1861617467366, "nlp": 210.6244352083886, "integer cut generation": 0.044262996525503695, "total": 3600.005251016002}}]}

gdplib/batch_processing/benchmark_result/gdpopt.ric_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "Batch Processing Optimization Problem", "Lower bound": 679365.3345166193, "Upper bound": 679365.3345166193, "Number of objectives": 1, "Number of constraints": 601, "Number of variables": 287, "Number of binary variables": 138, "Number of integer variables": 0, "Number of continuous variables": 149, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 9}], "Solver": [{"Name": "GDPopt (22, 5, 13) - RIC", "Status": "ok", "User time": 1449.0040160750505, "Wallclock time": 1449.0040160750505, "Termination condition": "optimal", "Iterations": 513, "Timing": {"main_timer_start_time": 792448.022091002, "mip": 355.0396141371457, "nlp": 1093.48349528166, "integer cut generation": 0.26000761275645345, "total": 1449.0040160750505}}]}

gdplib/disease_model/benchmark_result/gdp.bigm_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "/tmp/tmpi1k8tzo4/model.gms", "Lower bound": 304.3945184893451, "Upper bound": 304.4162107198233, "Number of objectives": 1, "Number of constraints": 910.0, "Number of variables": 1251.0, "Number of binary variables": null, "Number of integer variables": 52.0, "Number of continuous variables": 1199.0, "Number of nonzeros": 3482.0, "Sense": "minimize"}], "Solver": [{"Name": "GAMS (44, 3, 0, 0)", "Status": "ok", "Return code": 0, "Message": null, "User time": 1.685000164434314, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.010722160339355469}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}

gdplib/disease_model/benchmark_result/gdp.hull_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "/tmp/tmpjs8wc6fe/model.gms", "Lower bound": 304.39411746917403, "Upper bound": 304.4162107198197, "Number of objectives": 1, "Number of constraints": 1248.0, "Number of variables": 1407.0, "Number of binary variables": null, "Number of integer variables": 52.0, "Number of continuous variables": 1355.0, "Number of nonzeros": 4132.0, "Sense": "minimize"}], "Solver": [{"Name": "GAMS (44, 3, 0, 0)", "Status": "ok", "Return code": 0, "Message": null, "User time": 2.265999955125153, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.04540753364562988}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}

gdplib/disease_model/benchmark_result/gdpopt.enumerate_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "SIR Disease Model", "Lower bound": -Infinity, "Upper bound": 335.3755252019032, "Number of objectives": 1, "Number of constraints": 831, "Number of variables": 1250, "Number of binary variables": 52, "Number of integer variables": 0, "Number of continuous variables": 1198, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 26}], "Solver": [{"Name": "GDPopt (22, 5, 13) - enumerate", "Status": "ok", "User time": 3621.4882010290166, "Wallclock time": 3621.4882010290166, "Termination condition": "maxTimeLimit", "Iterations": 6452, "Timing": {"main_timer_start_time": 875323.264967119, "nlp": 3010.0583160168026, "total": 3621.4882010290166}}]}

gdplib/disease_model/benchmark_result/gdpopt.gloa_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "SIR Disease Model", "Lower bound": 304.41621103465036, "Upper bound": 304.41621103465036, "Number of objectives": 1, "Number of constraints": 831, "Number of variables": 1250, "Number of binary variables": 52, "Number of integer variables": 0, "Number of continuous variables": 1198, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 26}], "Solver": [{"Name": "GDPopt (22, 5, 13) - GLOA", "Status": "ok", "User time": 2.5978779919678345, "Wallclock time": 2.5978779919678345, "Termination condition": "optimal", "Iterations": 1, "Timing": {"main_timer_start_time": 878723.782462487, "mip": 1.9891705990303308, "nlp": 0.43279405997600406, "integer cut generation": 0.000803429982624948, "total": 2.5978779919678345}}]}

gdplib/disease_model/benchmark_result/gdpopt.lbb_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "SIR Disease Model", "Lower bound": NaN, "Upper bound": 329.65823101688966, "Number of objectives": 1, "Number of constraints": 831, "Number of variables": 1250, "Number of binary variables": 52, "Number of integer variables": 0, "Number of continuous variables": 1198, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 26}], "Solver": [{"Name": "GDPopt (22, 5, 13) - LBB", "Status": "ok", "User time": 422.74936116405297, "Wallclock time": 422.74936116405297, "Termination condition": "optimal", "Iterations": 778, "Timing": {"main_timer_start_time": 878944.786042594, "total": 422.74936116405297}}]}

gdplib/disease_model/benchmark_result/gdpopt.loa_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "SIR Disease Model", "Lower bound": 304.41621103465036, "Upper bound": 304.41621103465036, "Number of objectives": 1, "Number of constraints": 831, "Number of variables": 1250, "Number of binary variables": 52, "Number of integer variables": 0, "Number of continuous variables": 1198, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 26}], "Solver": [{"Name": "GDPopt (22, 5, 13) - LOA", "Status": "ok", "User time": 2.7232641809387133, "Wallclock time": 2.7232641809387133, "Termination condition": "optimal", "Iterations": 1, "Timing": {"main_timer_start_time": 878721.023115881, "mip": 2.0664316890761256, "nlp": 0.477121229050681, "integer cut generation": 0.000876300036907196, "total": 2.7232641809387133}}]}

gdplib/disease_model/benchmark_result/gdpopt.ric_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "SIR Disease Model", "Lower bound": 304.41621103465036, "Upper bound": 304.41621103465036, "Number of objectives": 1, "Number of constraints": 831, "Number of variables": 1250, "Number of binary variables": 52, "Number of integer variables": 0, "Number of continuous variables": 1198, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 26}], "Solver": [{"Name": "GDPopt (22, 5, 13) - RIC", "Status": "ok", "User time": 2.662617118912749, "Wallclock time": 2.662617118912749, "Termination condition": "optimal", "Iterations": 1, "Timing": {"main_timer_start_time": 878726.414061818, "mip": 1.9992133700288832, "nlp": 0.483937795041129, "integer cut generation": 0.000771639053709805, "total": 2.662617118912749}}]}

gdplib/gdp_col/benchmark_result/gdp.bigm_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "/tmp/tmph9d34omk/model.gms", "Lower bound": 20916.392212763814, "Upper bound": 20916.392212763814, "Number of objectives": 1, "Number of constraints": 1086.0, "Number of variables": 434.0, "Number of binary variables": null, "Number of integer variables": 28.0, "Number of continuous variables": 406.0, "Number of nonzeros": 3897.0, "Sense": "minimize"}], "Solver": [{"Name": "GAMS (44, 3, 0, 0)", "Status": "ok", "Return code": 0, "Message": null, "User time": 2.372000436298549, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.022061586380004883}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}

gdplib/gdp_col/benchmark_result/gdpopt.enumerate_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "benzene-toluene column", "Lower bound": 9277.598680606905, "Upper bound": 9277.598680606905, "Number of objectives": 1, "Number of constraints": 603, "Number of variables": 433, "Number of binary variables": 28, "Number of integer variables": 0, "Number of continuous variables": 405, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 15}], "Solver": [{"Name": "GDPopt (22, 5, 13) - enumerate", "Status": "ok", "User time": 1555.0285036379937, "Wallclock time": 1555.0285036379937, "Termination condition": "optimal", "Iterations": 32768, "Timing": {"main_timer_start_time": 879371.769702976, "nlp": 1554.1486481248867, "total": 1555.0285036379937}}]}

gdplib/gdp_col/benchmark_result/gdpopt.gloa_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "benzene-toluene column", "Lower bound": 19346.124800886784, "Upper bound": 19346.124800886784, "Number of objectives": 1, "Number of constraints": 603, "Number of variables": 433, "Number of binary variables": 28, "Number of integer variables": 0, "Number of continuous variables": 405, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 15}], "Solver": [{"Name": "GDPopt (22, 5, 13) - GLOA", "Status": "ok", "User time": 26.470644732005894, "Wallclock time": 26.470644732005894, "Termination condition": "optimal", "Iterations": 14, "Timing": {"main_timer_start_time": 880807.601197363, "mip": 17.16823974798899, "nlp": 7.965824775281362, "integer cut generation": 0.006472834968008101, "total": 26.470644732005894}}]}

gdplib/gdp_col/benchmark_result/gdpopt.loa_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "benzene-toluene column", "Lower bound": 19346.124800886784, "Upper bound": 19346.124800886784, "Number of objectives": 1, "Number of constraints": 603, "Number of variables": 433, "Number of binary variables": 28, "Number of integer variables": 0, "Number of continuous variables": 405, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 15}], "Solver": [{"Name": "GDPopt (22, 5, 13) - LOA", "Status": "ok", "User time": 72.49163393408526, "Wallclock time": 72.49163393408526, "Termination condition": "optimal", "Iterations": 35, "Timing": {"main_timer_start_time": 880735.070192499, "mip": 49.99203102546744, "nlp": 20.802424076013267, "integer cut generation": 0.01504453516099602, "total": 72.49163393408526}}]}

gdplib/gdp_col/benchmark_result/gdpopt.ric_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "benzene-toluene column", "Lower bound": 19346.124800884547, "Upper bound": 19346.124800884547, "Number of objectives": 1, "Number of constraints": 603, "Number of variables": 433, "Number of binary variables": 28, "Number of integer variables": 0, "Number of continuous variables": 405, "Number of nonzeros": null, "Sense": "minimize", "Number of disjunctions": 15}], "Solver": [{"Name": "GDPopt (22, 5, 13) - RIC", "Status": "ok", "User time": 30.866129517904483, "Wallclock time": 30.866129517904483, "Termination condition": "optimal", "Iterations": 35, "Timing": {"main_timer_start_time": 880844.913717317, "mip": 12.74963744287379, "nlp": 17.02741907502059, "integer cut generation": 0.01807352132163942, "total": 30.866129517904483}}]}

gdplib/hda/benchmark_result/gdp.bigm_gams.json

@@ -0,0 +1 @@
+{"Problem": [{"Name": "/tmp/tmprrxwit7o/model.gms", "Lower bound": 5714.555997088593, "Upper bound": 3528.950792100343, "Number of objectives": 1, "Number of constraints": 1097.0, "Number of variables": 722.0, "Number of binary variables": null, "Number of integer variables": 12.0, "Number of continuous variables": 710.0, "Number of nonzeros": 3722.0, "Sense": "maximize"}], "Solver": [{"Name": "GAMS (44, 3, 0, 0)", "Status": "ok", "Return code": 0, "Message": null, "User time": 3600.103999930434, "System time": null, "Wallclock time": null, "Termination condition": "optimal", "Termination message": null, "Transformation time": 0.033936262130737305}], "Solution": [{"number of solutions": 0, "number of solutions displayed": 0}]}