Add updates for lgtm CI security site (#357)

* Update code to remove LGTM Errors and Warnings and implement Recommendations. * Change BaseException to Exception. * Add lgtm config file. * Changes for flake8. * Add TypeError yo yam read. * Add TypeError to yaml read. * Just return when successful on the yaml read. * Fix typo. * Add merlin/examples to lgtm exclude list as well. * Add ssl comment. * Fix typo.
LLNL · Mar 7, 2022 · 681d175 · 681d175
1 parent abc4c66
commit 681d175
Show file tree

Hide file tree

Showing 33 changed files with 102 additions and 93 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- Code updates to satisfy lgtm CI security checker
+
+### Fixed
+- A bug in the ssl config was not returning the proper values
+
 ## [1.8.4]
 ### Added
 - Auto-release of pypi packages

diff --git a/lgtm.yml b/lgtm.yml
@@ -0,0 +1,25 @@
+##########################################################################################
+# Customize file classifications.                                                        #
+# Results from files under any classifier will be excluded from LGTM                     #
+# statistics.                                                                            #
+##########################################################################################
+
+##########################################################################################
+# Use the `path_classifiers` block to define changes to the default classification of    #
+# files.                                                                                 #
+##########################################################################################
+
+path_classifiers:
+  test:
+    # Classify all files in the top-level directories tests/ as test code.
+    - exclude: 
+    - tests
+    - merlin/examples
+
+#########################################################################################
+# Use the `queries` block to change the default display of query results.               #
+#########################################################################################
+
+queries:
+  # Specifically hide the results of clear-text-logging-sensitive-data
+  - exclude: py/clear-text-logging-sensitive-data
diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py
@@ -90,7 +90,7 @@ def scale_samples(samples_norm, limits, limits_norm=(0, 1), do_log=False):
     if not hasattr(do_log, "__iter__"):
         do_log = ndims * [do_log]
     logs = np.asarray(do_log)
-    lims_norm = np.array([limits_norm for i in logs])
+    lims_norm = np.array([limits_norm] * len(logs))
     _lims = []
     for limit, log in zip(limits, logs):
         if log:

diff --git a/merlin/config/broker.py b/merlin/config/broker.py
@@ -219,7 +219,6 @@ def get_connection_string(include_password=True):
         raise ValueError(f"Error: {broker} is not a supported broker.")
     else:
         return _sort_valid_broker(broker, config_path, include_password)
-    return None
 
 
 def _sort_valid_broker(broker, config_path, include_password):

diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py
@@ -298,7 +298,8 @@ def merge_sslmap(server_ssl: Dict[str, Union[str, ssl.VerifyMode]], ssl_map: Dic
             new_server_ssl[ssl_map[k]] = server_ssl[k]
         else:
             new_server_ssl[k] = server_ssl[k]
-    server_ssl = new_server_ssl
+
+    return new_server_ssl
 
 
 app_config: Dict = get_config(None)

diff --git a/merlin/examples/workflows/feature_demo/scripts/hello_world.py b/merlin/examples/workflows/feature_demo/scripts/hello_world.py
@@ -1,6 +1,5 @@
 import argparse
 import json
-import sys
 
 
 def process_args(args):
@@ -32,4 +31,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/flux/scripts/flux_info.py b/merlin/examples/workflows/flux/scripts/flux_info.py
@@ -39,34 +39,32 @@
     kvs.get(f, "lwj")
 
     for d in kvs.walk("lwj", flux_handle=f):
-        try:
-            # print(type(d))
-            fdir = "lwj.{0}".format(d[0])
+        # print(type(d))
+        fdir = "lwj.{0}".format(d[0])
 
-            qcreate = "{0}.create-time".format(fdir)
-            create_time = kvs.get(f, qcreate)
+        qcreate = "{0}.create-time".format(fdir)
+        create_time = kvs.get(f, qcreate)
 
-            qstart = "{0}.starting-time".format(fdir)
-            start_time = kvs.get(f, qstart)
+        qstart = "{0}.starting-time".format(fdir)
+        start_time = kvs.get(f, qstart)
 
-            qrun = "{0}.running-time".format(fdir)
-            start_time = kvs.get(f, qrun)
+        qrun = "{0}.running-time".format(fdir)
+        start_time = kvs.get(f, qrun)
 
-            qcomplete = "{0}.complete-time".format(fdir)
-            complete_time = kvs.get(f, qcomplete)
+        qcomplete = "{0}.complete-time".format(fdir)
+        complete_time = kvs.get(f, qcomplete)
 
-            qcompleting = "{0}.completing-time".format(fdir)
-            completing_time = kvs.get(f, qcompleting)
+        qcompleting = "{0}.completing-time".format(fdir)
+        completing_time = kvs.get(f, qcompleting)
 
-            qwall = "{0}.walltime".format(fdir)
-            wall_time = kvs.get(f, qwall)
+        qwall = "{0}.walltime".format(fdir)
+        wall_time = kvs.get(f, qwall)
 
-            print(
-                f"Job {d[0]}: create: {create_time} start {start_time} run {start_time} completing {completing_time} complete {complete_time} wall {wall_time}"
-            )
-        except BaseException:
-            pass
-except BaseException:
+        print(
+            f"Job {d[0]}: create: {create_time} start {start_time} run {start_time} completing {completing_time} complete {complete_time} wall {wall_time}"
+        )
+
+except KeyError:
     top_dir = "job"
 
     def get_data_dict(key: str) -> Dict:

diff --git a/merlin/examples/workflows/flux/scripts/make_samples.py b/merlin/examples/workflows/flux/scripts/make_samples.py
@@ -1,6 +1,5 @@
 import argparse
 import ast
-import sys
 
 import numpy as np
 
@@ -58,4 +57,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py
@@ -1,6 +1,5 @@
 import argparse
 import os
-import sys
 from concurrent.futures import ProcessPoolExecutor
 
 import matplotlib.pyplot as plt
@@ -98,4 +97,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/hpc_demo/faker_sample.py b/merlin/examples/workflows/hpc_demo/faker_sample.py
@@ -1,5 +1,4 @@
 import argparse
-import sys
 
 from faker import Faker
 
@@ -38,4 +37,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/hpc_demo/sample_collector.py b/merlin/examples/workflows/hpc_demo/sample_collector.py
@@ -1,6 +1,5 @@
 import argparse
 import os
-import sys
 from concurrent.futures import ProcessPoolExecutor
 
 
@@ -46,4 +45,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/hpc_demo/sample_processor.py b/merlin/examples/workflows/hpc_demo/sample_processor.py
@@ -1,7 +1,6 @@
 import argparse
 import os
 import pathlib
-import sys
 
 import pandas as pd
 
@@ -51,4 +50,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py
@@ -1,6 +1,5 @@
 import argparse
 import os
-import sys
 from concurrent.futures import ProcessPoolExecutor
 
 import matplotlib.pyplot as plt
@@ -98,4 +97,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/iterative_demo/faker_sample.py b/merlin/examples/workflows/iterative_demo/faker_sample.py
@@ -1,5 +1,4 @@
 import argparse
-import sys
 
 from faker import Faker
 
@@ -38,4 +37,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/iterative_demo/sample_collector.py b/merlin/examples/workflows/iterative_demo/sample_collector.py
@@ -1,6 +1,5 @@
 import argparse
 import os
-import sys
 from concurrent.futures import ProcessPoolExecutor
 
 
@@ -46,4 +45,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/iterative_demo/sample_processor.py b/merlin/examples/workflows/iterative_demo/sample_processor.py
@@ -1,7 +1,6 @@
 import argparse
 import os
 import pathlib
-import sys
 
 import pandas as pd
 
@@ -51,4 +50,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/lsf/scripts/make_samples.py b/merlin/examples/workflows/lsf/scripts/make_samples.py
@@ -1,6 +1,5 @@
 import argparse
 import ast
-import sys
 
 import numpy as np
 
@@ -58,4 +57,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    main()
diff --git a/merlin/examples/workflows/null_spec/scripts/read_output.py b/merlin/examples/workflows/null_spec/scripts/read_output.py
@@ -41,7 +41,8 @@ def single_task_times():
                     match = matches.group(0)
                     match = float(match.strip("s:"))
                     task_durations.append(match)
-        except BaseException:
+        except Exception as e:
+            print(f"single_task_times Exception= {e}\n")
             continue
 
     print(str(task_durations))
@@ -58,9 +59,9 @@ def merlin_run_time():
         total += result
     try:
         print(f"c{args.c}_s{args.s} merlin run : " + str(result))
-    except BaseException:
+    except Exception as e:
         result = None
-        print(f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}")
+        print(f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}\n{e}")
 
 
 def start_verify_time():
@@ -74,12 +75,12 @@ def start_verify_time():
             element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f")
             timestamp = datetime.datetime.timestamp(element)
             all_timestamps.append(timestamp)
-        except BaseException:
+        except Exception as e:
             continue
         try:
             print(f"c{args.c}_s{args.s} start verify : " + str(all_timestamps[0]))
-        except BaseException:
-            print(f"c{args.c}_s{args.s} start verify : ERROR")
+        except Exception as e:
+            print(f"c{args.c}_s{args.s} start verify : ERROR\n{e}")
 
 
 def start_run_workers_time():
@@ -93,7 +94,7 @@ def start_run_workers_time():
             element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f")
             timestamp = datetime.datetime.timestamp(element)
             all_timestamps.append(timestamp)
-        except BaseException:
+        except Exception:
             continue
     earliest = min(all_timestamps)
     print(f"c{args.c}_s{args.s} start run-workers : " + str(earliest))
@@ -110,7 +111,7 @@ def start_sample1_time():
             element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f")
             timestamp = datetime.datetime.timestamp(element)
             all_timestamps.append(timestamp)
-        except BaseException:
+        except Exception:
             continue
     earliest = min(all_timestamps)
     print(f"c{args.c}_s{args.s} start samp1 : " + str(earliest))

diff --git a/merlin/examples/workflows/null_spec/scripts/read_output_chain.py b/merlin/examples/workflows/null_spec/scripts/read_output_chain.py
@@ -53,7 +53,7 @@ def single_task_times():
                     match = matches.group(0)
                     match = float(match.strip("s:"))
                     task_durations.append(match)
-        except BaseException:
+        except Exception:
             print(f"c{filled_c}_s{k} task times : ERROR")
             continue
 
@@ -71,7 +71,7 @@ def merlin_run_time():
         total += result
     try:
         print(f"c{filled_c} merlin run : " + str(result))
-    except BaseException:
+    except Exception:
         result = None
         print(f"c{filled_c} merlin run : ERROR -- result={result}, args.errfile={args.errfile}")
 
@@ -87,12 +87,12 @@ def start_verify_time():
             element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f")
             timestamp = datetime.datetime.timestamp(element)
             all_timestamps.append(timestamp)
-        except BaseException:
+        except Exception:
             print(f"c{filled_c}_s{k} start verify : ERROR")
             continue
         try:
             print(f"c{filled_c}_s{k} start verify : " + str(all_timestamps[0]))
-        except BaseException:
+        except Exception:
             print(f"c{filled_c}_s{k} start verify : ERROR")
 
 
@@ -107,7 +107,7 @@ def start_run_workers_time():
             element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f")
             timestamp = datetime.datetime.timestamp(element)
             all_timestamps.append(timestamp)
-        except BaseException:
+        except Exception:
             continue
         earliest = min(all_timestamps)
         print(f"c{filled_c}_s{k} start run-workers : " + str(earliest))
@@ -124,7 +124,7 @@ def start_sample1_time():
             element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f")
             timestamp = datetime.datetime.timestamp(element)
             all_timestamps.append(timestamp)
-        except BaseException:
+        except Exception:
             print(f"c{filled_c}_s{k} start samp1 : ERROR")
             continue
         earliest = min(all_timestamps)

diff --git a/merlin/examples/workflows/openfoam_wf/scripts/learn.py b/merlin/examples/workflows/openfoam_wf/scripts/learn.py
@@ -103,8 +103,6 @@
 ax[0][1].set_ylim([y_min, y_max])
 
 
-y_pred_all = regr.predict(X)
-input_enstrophy = ax[1][1].scatter(X[:, 0], 10 ** y[:, 1], s=100, edgecolors="black")
 ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize)
 ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize)
 ax[1][1].set_title("Average Energy Variation with Lidspeed")

diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py
@@ -103,8 +103,6 @@
 ax[0][1].set_ylim([y_min, y_max])
 
 
-y_pred_all = regr.predict(X)
-input_enstrophy = ax[1][1].scatter(X[:, 0], 10 ** y[:, 1], s=100, edgecolors="black")
 ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize)
 ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize)
 ax[1][1].set_title("Average Energy Variation with Lidspeed")

diff --git a/merlin/examples/workflows/optimization/scripts/visualizer.py b/merlin/examples/workflows/optimization/scripts/visualizer.py
@@ -9,7 +9,6 @@
 
 import matplotlib.pyplot as plt
 import numpy as np
-from mpl_toolkits.mplot3d import Axes3D
 
 
 plt.style.use("seaborn-white")