diff --git a/CHANGELOG.md b/CHANGELOG.md index b44333f52..d7b0d980d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Added +- Code updates to satisfy lgtm CI security checker + +### Fixed +- A bug in the ssl config was not returning the proper values + ## [1.8.4] ### Added - Auto-release of pypi packages diff --git a/lgtm.yml b/lgtm.yml new file mode 100644 index 000000000..e3f53c87d --- /dev/null +++ b/lgtm.yml @@ -0,0 +1,25 @@ +########################################################################################## +# Customize file classifications. # +# Results from files under any classifier will be excluded from LGTM # +# statistics. # +########################################################################################## + +########################################################################################## +# Use the `path_classifiers` block to define changes to the default classification of # +# files. # +########################################################################################## + +path_classifiers: + test: + # Classify all files in the top-level directories tests/ as test code. + - exclude: + - tests + - merlin/examples + +######################################################################################### +# Use the `queries` block to change the default display of query results. # +######################################################################################### + +queries: + # Specifically hide the results of clear-text-logging-sensitive-data + - exclude: py/clear-text-logging-sensitive-data diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index e11721cae..638795e3b 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -90,7 +90,7 @@ def scale_samples(samples_norm, limits, limits_norm=(0, 1), do_log=False): if not hasattr(do_log, "__iter__"): do_log = ndims * [do_log] logs = np.asarray(do_log) - lims_norm = np.array([limits_norm for i in logs]) + lims_norm = np.array([limits_norm] * len(logs)) _lims = [] for limit, log in zip(limits, logs): if log: diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 5a0430dc4..8cb5312f0 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -219,7 +219,6 @@ def get_connection_string(include_password=True): raise ValueError(f"Error: {broker} is not a supported broker.") else: return _sort_valid_broker(broker, config_path, include_password) - return None def _sort_valid_broker(broker, config_path, include_password): diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index e237d741a..f7430ac68 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -298,7 +298,8 @@ def merge_sslmap(server_ssl: Dict[str, Union[str, ssl.VerifyMode]], ssl_map: Dic new_server_ssl[ssl_map[k]] = server_ssl[k] else: new_server_ssl[k] = server_ssl[k] - server_ssl = new_server_ssl + + return new_server_ssl app_config: Dict = get_config(None) diff --git a/merlin/examples/workflows/feature_demo/scripts/hello_world.py b/merlin/examples/workflows/feature_demo/scripts/hello_world.py index 9d029bbf0..ab14bedf4 100644 --- a/merlin/examples/workflows/feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/feature_demo/scripts/hello_world.py @@ -1,6 +1,5 @@ import argparse import json -import sys def process_args(args): @@ -32,4 +31,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/flux/scripts/flux_info.py b/merlin/examples/workflows/flux/scripts/flux_info.py index cc3114f04..d4599c652 100755 --- a/merlin/examples/workflows/flux/scripts/flux_info.py +++ b/merlin/examples/workflows/flux/scripts/flux_info.py @@ -39,34 +39,32 @@ kvs.get(f, "lwj") for d in kvs.walk("lwj", flux_handle=f): - try: - # print(type(d)) - fdir = "lwj.{0}".format(d[0]) + # print(type(d)) + fdir = "lwj.{0}".format(d[0]) - qcreate = "{0}.create-time".format(fdir) - create_time = kvs.get(f, qcreate) + qcreate = "{0}.create-time".format(fdir) + create_time = kvs.get(f, qcreate) - qstart = "{0}.starting-time".format(fdir) - start_time = kvs.get(f, qstart) + qstart = "{0}.starting-time".format(fdir) + start_time = kvs.get(f, qstart) - qrun = "{0}.running-time".format(fdir) - start_time = kvs.get(f, qrun) + qrun = "{0}.running-time".format(fdir) + start_time = kvs.get(f, qrun) - qcomplete = "{0}.complete-time".format(fdir) - complete_time = kvs.get(f, qcomplete) + qcomplete = "{0}.complete-time".format(fdir) + complete_time = kvs.get(f, qcomplete) - qcompleting = "{0}.completing-time".format(fdir) - completing_time = kvs.get(f, qcompleting) + qcompleting = "{0}.completing-time".format(fdir) + completing_time = kvs.get(f, qcompleting) - qwall = "{0}.walltime".format(fdir) - wall_time = kvs.get(f, qwall) + qwall = "{0}.walltime".format(fdir) + wall_time = kvs.get(f, qwall) - print( - f"Job {d[0]}: create: {create_time} start {start_time} run {start_time} completing {completing_time} complete {complete_time} wall {wall_time}" - ) - except BaseException: - pass -except BaseException: + print( + f"Job {d[0]}: create: {create_time} start {start_time} run {start_time} completing {completing_time} complete {complete_time} wall {wall_time}" + ) + +except KeyError: top_dir = "job" def get_data_dict(key: str) -> Dict: diff --git a/merlin/examples/workflows/flux/scripts/make_samples.py b/merlin/examples/workflows/flux/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/flux/scripts/make_samples.py +++ b/merlin/examples/workflows/flux/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py index ea84f7d6f..7d06ab594 100644 --- a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -98,4 +97,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/faker_sample.py b/merlin/examples/workflows/hpc_demo/faker_sample.py index d6f7020e8..ee8bf2f5c 100644 --- a/merlin/examples/workflows/hpc_demo/faker_sample.py +++ b/merlin/examples/workflows/hpc_demo/faker_sample.py @@ -1,5 +1,4 @@ import argparse -import sys from faker import Faker @@ -38,4 +37,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/sample_collector.py b/merlin/examples/workflows/hpc_demo/sample_collector.py index cfaac6e17..f62111e8e 100644 --- a/merlin/examples/workflows/hpc_demo/sample_collector.py +++ b/merlin/examples/workflows/hpc_demo/sample_collector.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor @@ -46,4 +45,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/hpc_demo/sample_processor.py b/merlin/examples/workflows/hpc_demo/sample_processor.py index ed4dd596b..9ec0951e9 100644 --- a/merlin/examples/workflows/hpc_demo/sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/sample_processor.py @@ -1,7 +1,6 @@ import argparse import os import pathlib -import sys import pandas as pd @@ -51,4 +50,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py index ea84f7d6f..7d06ab594 100644 --- a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor import matplotlib.pyplot as plt @@ -98,4 +97,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/faker_sample.py b/merlin/examples/workflows/iterative_demo/faker_sample.py index d6f7020e8..ee8bf2f5c 100644 --- a/merlin/examples/workflows/iterative_demo/faker_sample.py +++ b/merlin/examples/workflows/iterative_demo/faker_sample.py @@ -1,5 +1,4 @@ import argparse -import sys from faker import Faker @@ -38,4 +37,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/sample_collector.py b/merlin/examples/workflows/iterative_demo/sample_collector.py index cfaac6e17..f62111e8e 100644 --- a/merlin/examples/workflows/iterative_demo/sample_collector.py +++ b/merlin/examples/workflows/iterative_demo/sample_collector.py @@ -1,6 +1,5 @@ import argparse import os -import sys from concurrent.futures import ProcessPoolExecutor @@ -46,4 +45,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/iterative_demo/sample_processor.py b/merlin/examples/workflows/iterative_demo/sample_processor.py index ed4dd596b..9ec0951e9 100644 --- a/merlin/examples/workflows/iterative_demo/sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/sample_processor.py @@ -1,7 +1,6 @@ import argparse import os import pathlib -import sys import pandas as pd @@ -51,4 +50,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/lsf/scripts/make_samples.py b/merlin/examples/workflows/lsf/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/lsf/scripts/make_samples.py +++ b/merlin/examples/workflows/lsf/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/null_spec/scripts/read_output.py b/merlin/examples/workflows/null_spec/scripts/read_output.py index bc238d9d2..7c0f8017e 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output.py @@ -41,7 +41,8 @@ def single_task_times(): match = matches.group(0) match = float(match.strip("s:")) task_durations.append(match) - except BaseException: + except Exception as e: + print(f"single_task_times Exception= {e}\n") continue print(str(task_durations)) @@ -58,9 +59,9 @@ def merlin_run_time(): total += result try: print(f"c{args.c}_s{args.s} merlin run : " + str(result)) - except BaseException: + except Exception as e: result = None - print(f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}") + print(f"c{args.c}_s{args.s} merlin run : ERROR -- result={result}, args.errfile={args.errfile}\n{e}") def start_verify_time(): @@ -74,12 +75,12 @@ def start_verify_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception as e: continue try: print(f"c{args.c}_s{args.s} start verify : " + str(all_timestamps[0])) - except BaseException: - print(f"c{args.c}_s{args.s} start verify : ERROR") + except Exception as e: + print(f"c{args.c}_s{args.s} start verify : ERROR\n{e}") def start_run_workers_time(): @@ -93,7 +94,7 @@ def start_run_workers_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: continue earliest = min(all_timestamps) print(f"c{args.c}_s{args.s} start run-workers : " + str(earliest)) @@ -110,7 +111,7 @@ def start_sample1_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: continue earliest = min(all_timestamps) print(f"c{args.c}_s{args.s} start samp1 : " + str(earliest)) diff --git a/merlin/examples/workflows/null_spec/scripts/read_output_chain.py b/merlin/examples/workflows/null_spec/scripts/read_output_chain.py index f9fdfd521..ae9fb11a5 100644 --- a/merlin/examples/workflows/null_spec/scripts/read_output_chain.py +++ b/merlin/examples/workflows/null_spec/scripts/read_output_chain.py @@ -53,7 +53,7 @@ def single_task_times(): match = matches.group(0) match = float(match.strip("s:")) task_durations.append(match) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} task times : ERROR") continue @@ -71,7 +71,7 @@ def merlin_run_time(): total += result try: print(f"c{filled_c} merlin run : " + str(result)) - except BaseException: + except Exception: result = None print(f"c{filled_c} merlin run : ERROR -- result={result}, args.errfile={args.errfile}") @@ -87,12 +87,12 @@ def start_verify_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} start verify : ERROR") continue try: print(f"c{filled_c}_s{k} start verify : " + str(all_timestamps[0])) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} start verify : ERROR") @@ -107,7 +107,7 @@ def start_run_workers_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: continue earliest = min(all_timestamps) print(f"c{filled_c}_s{k} start run-workers : " + str(earliest)) @@ -124,7 +124,7 @@ def start_sample1_time(): element = datetime.datetime.strptime(match, "%Y-%m-%d %H:%M:%S,%f") timestamp = datetime.datetime.timestamp(element) all_timestamps.append(timestamp) - except BaseException: + except Exception: print(f"c{filled_c}_s{k} start samp1 : ERROR") continue earliest = min(all_timestamps) diff --git a/merlin/examples/workflows/openfoam_wf/scripts/learn.py b/merlin/examples/workflows/openfoam_wf/scripts/learn.py index b7c40f6d3..96124c46f 100644 --- a/merlin/examples/workflows/openfoam_wf/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf/scripts/learn.py @@ -103,8 +103,6 @@ ax[0][1].set_ylim([y_min, y_max]) -y_pred_all = regr.predict(X) -input_enstrophy = ax[1][1].scatter(X[:, 0], 10 ** y[:, 1], s=100, edgecolors="black") ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize) ax[1][1].set_title("Average Energy Variation with Lidspeed") diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py index b7c40f6d3..96124c46f 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py @@ -103,8 +103,6 @@ ax[0][1].set_ylim([y_min, y_max]) -y_pred_all = regr.predict(X) -input_enstrophy = ax[1][1].scatter(X[:, 0], 10 ** y[:, 1], s=100, edgecolors="black") ax[1][1].set_xlabel(r"Lidspeed ($\frac{m}{s}$)", fontsize=fontsize) ax[1][1].set_ylabel(r"$Energy$", fontsize=fontsize) ax[1][1].set_title("Average Energy Variation with Lidspeed") diff --git a/merlin/examples/workflows/optimization/scripts/visualizer.py b/merlin/examples/workflows/optimization/scripts/visualizer.py index f41885bf9..c373c9cd1 100644 --- a/merlin/examples/workflows/optimization/scripts/visualizer.py +++ b/merlin/examples/workflows/optimization/scripts/visualizer.py @@ -9,7 +9,6 @@ import matplotlib.pyplot as plt import numpy as np -from mpl_toolkits.mplot3d import Axes3D plt.style.use("seaborn-white") diff --git a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py index 0e2b624d6..232c43e86 100644 --- a/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/remote_feature_demo/scripts/hello_world.py @@ -1,6 +1,5 @@ import argparse import json -import sys from typing import Dict @@ -42,4 +41,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/restart/scripts/make_samples.py b/merlin/examples/workflows/restart/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/restart/scripts/make_samples.py +++ b/merlin/examples/workflows/restart/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/restart_delay/scripts/make_samples.py b/merlin/examples/workflows/restart_delay/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/restart_delay/scripts/make_samples.py +++ b/merlin/examples/workflows/restart_delay/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/examples/workflows/slurm/scripts/make_samples.py b/merlin/examples/workflows/slurm/scripts/make_samples.py index 913a94062..e6c807bc9 100644 --- a/merlin/examples/workflows/slurm/scripts/make_samples.py +++ b/merlin/examples/workflows/slurm/scripts/make_samples.py @@ -1,6 +1,5 @@ import argparse import ast -import sys import numpy as np @@ -58,4 +57,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 20a32755f..e90fd6afa 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -36,6 +36,7 @@ "RetryException", "SoftFailException", "HardFailException", + "InvalidChainException", "RestartException", ) @@ -76,7 +77,7 @@ class InvalidChainException(Exception): """ def __init__(self): - super(HardFailException, self).__init__() + super(InvalidChainException, self).__init__() class RestartException(Exception): diff --git a/merlin/main.py b/merlin/main.py index 9f6c90a73..dec90a034 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -140,7 +140,7 @@ def parse_override_vars( val = int(val) result[key] = val - except BaseException as excpt: + except Exception as excpt: raise ValueError( f"{excpt} Bad '--vars' formatting on command line. See 'merlin run --help' for an example." ) from excpt @@ -756,4 +756,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 94c57445e..67a9d455f 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -33,7 +33,6 @@ """ import argparse import logging -import sys from merlin.ascii_art import banner_small from merlin.log_formatter import setup_logging @@ -65,4 +64,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 122b8f002..7dda416b1 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -228,7 +228,7 @@ def dump(self): result = result.replace("\n\n\n", "\n\n") try: yaml.safe_load(result) - except BaseException as e: + except Exception as e: raise ValueError(f"Error parsing provenance spec:\n{e}") return result diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index b7045bc46..67598be87 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -155,8 +155,8 @@ def query_celery_queues(queues): try: name, jobs, consumers = channel.queue_declare(queue=queue, passive=True) found_queues.append((name, jobs, consumers)) - except BaseException: - LOG.warning(f"Cannot find queue {queue} on server.") + except Exception as e: + LOG.warning(f"Cannot find queue {queue} on server.{e}") finally: connection.close() return found_queues diff --git a/merlin/utils.py b/merlin/utils.py index 657349fed..0cd06371e 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -37,7 +37,7 @@ import re import socket import subprocess -from contextlib import contextmanager, suppress +from contextlib import contextmanager from copy import deepcopy from datetime import timedelta from types import SimpleNamespace @@ -205,15 +205,14 @@ def get_yaml_var(entry, var, default): :param `var`: a yaml key :param `default`: default value in the absence of data """ - ret = default - if isinstance(entry, dict): - with suppress(KeyError): - ret = entry[var] - else: - with suppress(AttributeError): - ret = getattr(entry, var) - return ret + try: + return entry[var] + except (TypeError, KeyError): + try: + return getattr(entry, var) + except AttributeError: + return default def load_array_file(filename, ndmin=2): diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index c7d3745b1..3a8038d06 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -34,7 +34,6 @@ """ import argparse import shutil -import sys import time from contextlib import suppress from subprocess import PIPE, Popen @@ -155,7 +154,7 @@ def run_tests(args, tests): continue try: passed, info = run_single_test(test_name, test, test_label) - except BaseException as e: + except Exception as e: print(e) passed = False info = None @@ -215,4 +214,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + main()