Run STBO for Reizman benchmark

sustainable-processes · Jun 10, 2021 · 60ec0e2 · 60ec0e2
1 parent f1f3a0b
commit 60ec0e2
Show file tree

Hide file tree

Showing 9 changed files with 766 additions and 490 deletions.
diff --git a/dvc.lock b/dvc.lock
@@ -55,102 +55,224 @@ stages:
       size: 585221
       nfiles: 8
   baumgartner_suzuki_benchmark_training:
-    cmd: python multitask/suzuki_benchmark_optimization.py train-benchmark data/baumgartner_suzuki/ord/baumgartner_suzuki.pb
-      data/baumgartner_suzuki/emulator figures/ --no-split-catalyst --max-epochs 1000
+    cmd: python multitask/suzuki_benchmark_training.py data/baumgartner_suzuki/ord/baumgartner_suzuki.pb
+      data/baumgartner_suzuki/emulator figures/ --no-split-catalyst --max-epochs 1
       --cv-folds 5 --verbose 0 --no-include-reactant-concentrations --no-print-warnings
     deps:
     - path: data/baumgartner_suzuki/ord/
       md5: 82ab5143f2c69df002c24d12f90ab832.dir
       size: 446426
       nfiles: 2
-    - path: multitask/suzuki_benchmark_optimization.py
-      md5: 31f2741371f8c21815c1c6d2281f4a4e
-      size: 15243
+    - path: multitask/suzuki_benchmark_training.py
+      md5: 2322347bbf69a3cf1d9f382d0c5af0ef
+      size: 5233
+    - path: multitask/suzuki_data_utils.py
+      md5: 4a455509eb2a5595fc1df69c27bea77b
+      size: 5698
+    - path: multitask/suzuki_emulator.py
+      md5: 7e823e1e9e8e7ddd620c2abc6a4296fc
+      size: 2245
     outs:
     - path: data/baumgartner_suzuki/emulator/
-      md5: c0dc4ce92c2dce500e1aa1124b74eeb8.dir
-      size: 156810
+      md5: fd574ead62d496661219bd933a60dc8e.dir
+      size: 156815
       nfiles: 6
     - path: figures/baumgartner_suzuki_parity_plot.png
-      md5: 49e1ace188cb7b115a43cd7ba585c2bb
-      size: 245100
+      md5: 5129c6847773547fed80c8a6c2020779
+      size: 179488
   reizman_suzuki_benchmark_training@2:
-    cmd: python multitask/suzuki_benchmark_optimization.py train-benchmark data/reizman_suzuki/ord/reizman_suzuki_case_2.pb
+    cmd: python multitask/suzuki_benchmark_training.py data/reizman_suzuki/ord/reizman_suzuki_case_2.pb
       data/reizman_suzuki/emulator_case_2/ figures/ --no-split-catalyst --max-epochs
-      1000 --cv-folds 5 --verbose 0 --no-print-warnings
+      1 --cv-folds 5 --verbose 0 --no-print-warnings
     deps:
     - path: data/reizman_suzuki/ord
       md5: e4d8a72cf6ed315163bc7ec442299418.dir
       size: 585221
       nfiles: 8
-    - path: multitask/suzuki_benchmark_optimization.py
-      md5: 31f2741371f8c21815c1c6d2281f4a4e
-      size: 15243
+    - path: multitask/suzuki_benchmark_training.py
+      md5: 2322347bbf69a3cf1d9f382d0c5af0ef
+      size: 5233
+    - path: multitask/suzuki_data_utils.py
+      md5: 4a455509eb2a5595fc1df69c27bea77b
+      size: 5698
+    - path: multitask/suzuki_emulator.py
+      md5: 7e823e1e9e8e7ddd620c2abc6a4296fc
+      size: 2245
     outs:
     - path: data/reizman_suzuki/emulator_case_2/
-      md5: 2a097940ec2a974ec75112609a03b02e.dir
-      size: 156812
+      md5: 3afe30ff21efd053753752b364422ad0.dir
+      size: 156802
       nfiles: 6
     - path: figures/reizman_suzuki_case_2_parity_plot.png
-      md5: 9946990b3777efeab4dda82fac8b8b4b
-      size: 189704
+      md5: 15c77c1162f10cac578f4179879336a1
+      size: 153119
   reizman_suzuki_benchmark_training@1:
-    cmd: python multitask/suzuki_benchmark_optimization.py train-benchmark data/reizman_suzuki/ord/reizman_suzuki_case_1.pb
+    cmd: python multitask/suzuki_benchmark_training.py data/reizman_suzuki/ord/reizman_suzuki_case_1.pb
       data/reizman_suzuki/emulator_case_1/ figures/ --no-split-catalyst --max-epochs
-      1000 --cv-folds 5 --verbose 0 --no-print-warnings
+      1 --cv-folds 5 --verbose 0 --no-print-warnings
     deps:
     - path: data/reizman_suzuki/ord
       md5: e4d8a72cf6ed315163bc7ec442299418.dir
       size: 585221
       nfiles: 8
-    - path: multitask/suzuki_benchmark_optimization.py
-      md5: 31f2741371f8c21815c1c6d2281f4a4e
-      size: 15243
+    - path: multitask/suzuki_benchmark_training.py
+      md5: 2322347bbf69a3cf1d9f382d0c5af0ef
+      size: 5233
+    - path: multitask/suzuki_data_utils.py
+      md5: 4a455509eb2a5595fc1df69c27bea77b
+      size: 5698
+    - path: multitask/suzuki_emulator.py
+      md5: 7e823e1e9e8e7ddd620c2abc6a4296fc
+      size: 2245
     outs:
     - path: data/reizman_suzuki/emulator_case_1/
-      md5: 76982e005d3f1052d46565a051181d28.dir
-      size: 156799
+      md5: 7b00debf5cad42059b6b6b014064da00.dir
+      size: 156788
       nfiles: 6
     - path: figures/reizman_suzuki_case_1_parity_plot.png
-      md5: f78cf37f051fa510a12b8b24b88b055b
-      size: 205906
+      md5: 154ff2638091318cf98871f3a0b8a0e6
+      size: 154327
   reizman_suzuki_benchmark_training@3:
-    cmd: python multitask/suzuki_benchmark_optimization.py train-benchmark data/reizman_suzuki/ord/reizman_suzuki_case_3.pb
+    cmd: python multitask/suzuki_benchmark_training.py data/reizman_suzuki/ord/reizman_suzuki_case_3.pb
       data/reizman_suzuki/emulator_case_3/ figures/ --no-split-catalyst --max-epochs
-      1000 --cv-folds 5 --verbose 0 --no-print-warnings
+      1 --cv-folds 5 --verbose 0 --no-print-warnings
     deps:
     - path: data/reizman_suzuki/ord
       md5: e4d8a72cf6ed315163bc7ec442299418.dir
       size: 585221
       nfiles: 8
-    - path: multitask/suzuki_benchmark_optimization.py
-      md5: 31f2741371f8c21815c1c6d2281f4a4e
-      size: 15243
+    - path: multitask/suzuki_benchmark_training.py
+      md5: 2322347bbf69a3cf1d9f382d0c5af0ef
+      size: 5233
+    - path: multitask/suzuki_data_utils.py
+      md5: 4a455509eb2a5595fc1df69c27bea77b
+      size: 5698
+    - path: multitask/suzuki_emulator.py
+      md5: 7e823e1e9e8e7ddd620c2abc6a4296fc
+      size: 2245
     outs:
     - path: data/reizman_suzuki/emulator_case_3/
-      md5: f77ec898372ea7fadf5f9b6c9a84df85.dir
+      md5: 0869d0257c0319e9c2d8dcfe738b8be7.dir
       size: 156802
       nfiles: 6
     - path: figures/reizman_suzuki_case_3_parity_plot.png
-      md5: 9ee3242e23b78cd422bef42e3b6ec7b4
-      size: 197209
+      md5: 81d3f5ace8d78efbcc24c8cec2933f98
+      size: 154362
   reizman_suzuki_benchmark_training@4:
-    cmd: python multitask/suzuki_benchmark_optimization.py train-benchmark data/reizman_suzuki/ord/reizman_suzuki_case_4.pb
+    cmd: python multitask/suzuki_benchmark_training.py data/reizman_suzuki/ord/reizman_suzuki_case_4.pb
       data/reizman_suzuki/emulator_case_4/ figures/ --no-split-catalyst --max-epochs
-      1000 --cv-folds 5 --verbose 0 --no-print-warnings
+      1 --cv-folds 5 --verbose 0 --no-print-warnings
     deps:
     - path: data/reizman_suzuki/ord
       md5: e4d8a72cf6ed315163bc7ec442299418.dir
       size: 585221
       nfiles: 8
-    - path: multitask/suzuki_benchmark_optimization.py
-      md5: 31f2741371f8c21815c1c6d2281f4a4e
-      size: 15243
+    - path: multitask/suzuki_benchmark_training.py
+      md5: 2322347bbf69a3cf1d9f382d0c5af0ef
+      size: 5233
+    - path: multitask/suzuki_data_utils.py
+      md5: 4a455509eb2a5595fc1df69c27bea77b
+      size: 5698
+    - path: multitask/suzuki_emulator.py
+      md5: 7e823e1e9e8e7ddd620c2abc6a4296fc
+      size: 2245
     outs:
     - path: data/reizman_suzuki/emulator_case_4/
-      md5: 055ae78a3b46a02a27feb4c8bfcaa8bb.dir
-      size: 156801
+      md5: 639c13b7819dfde2fb1fe20b7909de87.dir
+      size: 156796
       nfiles: 6
     - path: figures/reizman_suzuki_case_4_parity_plot.png
-      md5: c6c83ca9d5c3edcfcf2cd3681f8a1ea6
-      size: 206122
+      md5: c8dbc91c7ba82f70a2b3abaf06f9e989
+      size: 165254
+  suzuki_stbo_reizman@2:
+    cmd: python multitask/suzuki_optimization.py stbo reizman_suzuki_case_2 data/reizman_suzuki/emulator_case_2
+      data/reizman_suzuki/results_stbo_case_2/ --max-experiments 20 --batch-size 1
+      --repeats 20
+    deps:
+    - path: data/reizman_suzuki/emulator_case_2
+      md5: 3afe30ff21efd053753752b364422ad0.dir
+      size: 156802
+      nfiles: 6
+    - path: multitask/suzuki_data_utils.py
+      md5: 4a455509eb2a5595fc1df69c27bea77b
+      size: 5698
+    - path: multitask/suzuki_emulator.py
+      md5: 7e823e1e9e8e7ddd620c2abc6a4296fc
+      size: 2245
+    - path: multitask/suzuki_optimization.py
+      md5: e4c1ca7dcc1b475c3175d4d8d2215d43
+      size: 3578
+    outs:
+    - path: data/reizman_suzuki/results_stbo_case_2/
+      md5: 1c984f62712caa1d3b32e31919aa8e30.dir
+      size: 251993
+      nfiles: 20
+  suzuki_stbo_reizman@4:
+    cmd: python multitask/suzuki_optimization.py stbo reizman_suzuki_case_4 data/reizman_suzuki/emulator_case_4
+      data/reizman_suzuki/results_stbo_case_4/ --max-experiments 20 --batch-size 1
+      --repeats 20
+    deps:
+    - path: data/reizman_suzuki/emulator_case_4
+      md5: 639c13b7819dfde2fb1fe20b7909de87.dir
+      size: 156796
+      nfiles: 6
+    - path: multitask/suzuki_data_utils.py
+      md5: 4a455509eb2a5595fc1df69c27bea77b
+      size: 5698
+    - path: multitask/suzuki_emulator.py
+      md5: 7e823e1e9e8e7ddd620c2abc6a4296fc
+      size: 2245
+    - path: multitask/suzuki_optimization.py
+      md5: e4c1ca7dcc1b475c3175d4d8d2215d43
+      size: 3578
+    outs:
+    - path: data/reizman_suzuki/results_stbo_case_4/
+      md5: 87a07e9fe6797c7fc0f1f3945c597fb7.dir
+      size: 250718
+      nfiles: 20
+  suzuki_stbo_reizman@3:
+    cmd: python multitask/suzuki_optimization.py stbo reizman_suzuki_case_3 data/reizman_suzuki/emulator_case_3
+      data/reizman_suzuki/results_stbo_case_3/ --max-experiments 20 --batch-size 1
+      --repeats 20
+    deps:
+    - path: data/reizman_suzuki/emulator_case_3
+      md5: 0869d0257c0319e9c2d8dcfe738b8be7.dir
+      size: 156802
+      nfiles: 6
+    - path: multitask/suzuki_data_utils.py
+      md5: 4a455509eb2a5595fc1df69c27bea77b
+      size: 5698
+    - path: multitask/suzuki_emulator.py
+      md5: 7e823e1e9e8e7ddd620c2abc6a4296fc
+      size: 2245
+    - path: multitask/suzuki_optimization.py
+      md5: e4c1ca7dcc1b475c3175d4d8d2215d43
+      size: 3578
+    outs:
+    - path: data/reizman_suzuki/results_stbo_case_3/
+      md5: 89b19c16b8918a83c4fa4b4db2c996f0.dir
+      size: 250246
+      nfiles: 20
+  suzuki_stbo_reizman@1:
+    cmd: python multitask/suzuki_optimization.py stbo reizman_suzuki_case_1 data/reizman_suzuki/emulator_case_1
+      data/reizman_suzuki/results_stbo_case_1/ --max-experiments 20 --batch-size 1
+      --repeats 20
+    deps:
+    - path: data/reizman_suzuki/emulator_case_1
+      md5: 7b00debf5cad42059b6b6b014064da00.dir
+      size: 156788
+      nfiles: 6
+    - path: multitask/suzuki_data_utils.py
+      md5: 4a455509eb2a5595fc1df69c27bea77b
+      size: 5698
+    - path: multitask/suzuki_emulator.py
+      md5: 7e823e1e9e8e7ddd620c2abc6a4296fc
+      size: 2245
+    - path: multitask/suzuki_optimization.py
+      md5: e4c1ca7dcc1b475c3175d4d8d2215d43
+      size: 3578
+    outs:
+    - path: data/reizman_suzuki/results_stbo_case_1/
+      md5: ade9935db95e591ec9a34878e78dd416.dir
+      size: 250414
+      nfiles: 20
diff --git a/dvc.yaml b/dvc.yaml
@@ -24,7 +24,7 @@ stages:
   baumgartner_suzuki_benchmark_training:
     desc: "Train a benchmark based on data in the ORD schema format"
     cmd: >-
-      python multitask/suzuki_benchmark_optimization.py train-benchmark
+      python multitask/suzuki_benchmark_training.py
       data/baumgartner_suzuki/ord/baumgartner_suzuki.pb
       data/baumgartner_suzuki/emulator
       figures/
@@ -35,7 +35,9 @@ stages:
       --${suzuki_benchmark_training.include-reactant-concentrations}
       --no-print-warnings
     deps:
-      - multitask/suzuki_benchmark_optimization.py
+      - multitask/suzuki_benchmark_training.py
+      - multitask/suzuki_emulator.py
+      - multitask/suzuki_data_utils.py
       - data/baumgartner_suzuki/ord/
     outs:
     - data/baumgartner_suzuki/emulator/
@@ -50,7 +52,7 @@ stages:
     do:
       desc: "Train a benchmark based on data in the ORD schema format"
       cmd: >-
-        python multitask/suzuki_benchmark_optimization.py train-benchmark
+        python multitask/suzuki_benchmark_training.py
         data/reizman_suzuki/ord/reizman_suzuki_case_${item}.pb
         data/reizman_suzuki/emulator_case_${item}/
         figures/
@@ -60,8 +62,73 @@ stages:
         --verbose ${suzuki_benchmark_training.verbose}
         --no-print-warnings
       deps:
-        - multitask/suzuki_benchmark_optimization.py
+        - multitask/suzuki_benchmark_training.py
+        - multitask/suzuki_emulator.py
+        - multitask/suzuki_data_utils.py
         - data/reizman_suzuki/ord
       outs:
         - data/reizman_suzuki/emulator_case_${item}/
-        - figures/reizman_suzuki_case_${item}_parity_plot.png
+        - figures/reizman_suzuki_case_${item}_parity_plot.png
+
+  suzuki_stbo_reizman:
+    foreach: # cases
+      - 1
+      - 2
+      - 3
+      - 4
+    do:
+      desc: "Optimization of Reizman using Single Task Bayesian Optimization"
+      cmd: >-
+        python multitask/suzuki_optimization.py stbo
+        reizman_suzuki_case_${item}
+        data/reizman_suzuki/emulator_case_${item}
+        data/reizman_suzuki/results_stbo_case_${item}/
+        --max-experiments ${suzuki_optimization.max-experiments}
+        --batch-size ${suzuki_optimization.batch-size}
+        --repeats ${suzuki_optimization.repeats}
+      deps:
+        - multitask/suzuki_optimization.py
+        - multitask/suzuki_emulator.py
+        - multitask/suzuki_data_utils.py
+        - data/reizman_suzuki/emulator_case_${item}
+      outs:
+        - data/reizman_suzuki/results_stbo_case_${item}/
+
+#  suzuki_mtbo:
+#    foreach:
+#      - benchmark: data/baumgartner_suzuki/emulator
+#        dataset_1: data/reizman_suzuki/ord/reizman_suzuki_case_1.pb
+#        dataset_2: data/reizman_suzuki/ord/reizman_suzuki_case_2.pb
+#        dataset_3: data/reizman_suzuki/ord/reizman_suzuki_case_3.pb
+#        dataset_4: data/reizman_suzuki/ord/reizman_suzuki_case_4.pb
+#      - benchmark: data/reizman_suzuki/emulator_case_1
+#        dataset_1: data/baumgartner_suzuki/ord/baumgartner_suzuki.pb
+#        dataset_2: data/reizman_suzuki/ord/reizman_suzuki_case_2.pb
+#        dataset_3: data/reizman_suzuki/ord/reizman_suzuki_case_3.pb
+#        dataset_4: data/reizman_suzuki/ord/reizman_suzuki_case_4.pb
+#      - benchmark: data/reizman_suzuki/emulator_case_2
+#        dataset_1: data/baumgartner_suzuki/ord/baumgartner_suzuki.pb
+#        dataset_2: data/reizman_suzuki/ord/reizman_suzuki_case_1.pb
+#        dataset_3: data/reizman_suzuki/ord/reizman_suzuki_case_3.pb
+#        dataset_4: data/reizman_suzuki/ord/reizman_suzuki_case_4.pb
+#      - benchmark: data/reizman_suzuki/emulator_case_3
+#        dataset_1: data/baumgartner_suzuki/ord/baumgartner_suzuki.pb
+#        dataset_2: data/reizman_suzuki/ord/reizman_suzuki_case_1.pb
+#        dataset_3: data/reizman_suzuki/ord/reizman_suzuki_case_2.pb
+#        dataset_4: data/reizman_suzuki/ord/reizman_suzuki_case_4.pb
+#      - benchmark: data/reizman_suzuki/emulator_case_4
+#        dataset_1: data/baumgartner_suzuki/ord/baumgartner_suzuki.pb
+#        dataset_2: data/reizman_suzuki/ord/reizman_suzuki_case_1.pb
+#        dataset_3: data/reizman_suzuki/ord/reizman_suzuki_case_2.pb
+#        dataset_4: data/reizman_suzuki/ord/reizman_suzuki_case_3.pb
+#    do:
+#      desc: "Optimization using Multitask Bayesian Optimization"
+#      cmd: >-
+#        python multitask/suzuki_benchmark_optimization.py mtbo-optimization
+#        ${item.benchmark}
+#        ${item.dataset_1} ${item.dataset_2} ${item.dataset_3} ${item.dataset_4}
+#        ${key}
+#        --max-experiments ${suzuki_mtbo.max-experiments}
+#        --batch-size ${suzuki_mtbo.batch-szie}
+#        --repeats ${suzuki_mtbo.repeats}
+#        --no-print-warnings
diff --git a/multitask/__init__.py b/multitask/__init__.py
@@ -1,2 +1,2 @@
 from .utils import *
-from .suzuki_benchmark_optimization import *
+from .suzuki_benchmark_training import *