diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..f568354da --- /dev/null +++ b/.gitignore @@ -0,0 +1,68 @@ +# Merlin .gitignore + +# Python bytecode and serialized files +__pycache__/ +*.py[cod] +*.pkl +*.db +*.npy +*.npz +*.hdf5 +dist/ +build/ + +# Egg packaging +*.egg +*.egg-info/ + +# virtual environments +toss3venv/ +venv/ +chaos5-venv/ +blueos/ +venv_* + +# Merlin-generated output +*_ARCHIVE/ +*_ARCHIVE_DIR/ +ARCHIVE +ARCHIVE_DIR +*_OUTPUT/ +*_OUTPUT_D/ +*_ensemble_*/ + +# Scheduler logs +flux.out +slurm*.out +docs/build/ + +# Tox files +.tox/* + +# Jupyter +jupyter/.ipynb_checkpoints +jupyter/testDistributedSamples.py + +# Images +*.png +*.pdf + +# aspell +*.bak + +# Misc file extension types +*.swl +*.swm +*.swn +*.swo +*.swp +*.db +*.npy +*.log + +# IDEs +*.idea + +# Misc directories +dist/ +build/ diff --git a/.travis.yml b/.travis.yml index 8a9b9ecfc..a0434d2dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,16 +5,16 @@ python: - "3.8" install: - "pip install --upgrade pip" - - "pip install cryptography" - "pip install -r requirements.txt" - - "pip install -r workflows/feature_demo/requirements.txt" + - "pip install -r merlin/examples/workflows/feature_demo/requirements.txt" - "pip install -e ." - "pip install -r requirements/mysql.txt" + - "pip install -r requirements/dev.txt" - "pip install --upgrade sphinx" script: - "merlin config" - "python -m pytest tests/" - - "python tests/integration/run_tests.py --verbose --ids 1 2 3 4 5 6 7 8 9 12 13 18 19 20" + - "python tests/integration/run_tests.py --verbose --ids 1 2 3 4 5 6 7 8 9 12 13 14 15 20 21 22" deploy: provider: pypi user: "__token__" diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e082da8..59bad86c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,33 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.1.0] - 2020-01-07 + +### Added +- Development dependencies install via pip: `pip install "merlinwf[dev]"`. +- `merlin status ` that returns queues, number of connected + workers and number of unused tasks in each of those queues. +- `merlin example` cli command, which allows users to start running the + examples immedately (even after pip-installing). + +### Fixed +- `MANIFEST.in` fixes as required by Spack. +- `requirements.txt` just has release components, not dev deps. +- A bug related to the deprecated word 'unicode' in `openfilelist.py`. +- Broken Merlin logo image on PyPI summary page. +- Documentation typos. + +### Changed +- Made `README.md` more concise and user-friendly. + +### Removed +- Dependencies outside the requirements directory. +- LLNL-specific material in the Makefile. + +### Deprecated +- `merlin-templates` cli command, in favor of `merlin example`. + + ## [1.0.5] - 2019-12-05 ### Fixed diff --git a/MANIFEST.in b/MANIFEST.in index 3a1b2a29d..83521ddcd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,3 @@ -prune docs -prune tests include workflows/* include merlin/data/* include requirements.txt diff --git a/Makefile b/Makefile index c38b1f696..720e8cfcb 100644 --- a/Makefile +++ b/Makefile @@ -31,22 +31,21 @@ PYTHON?=python3 PYV=$(shell $(PYTHON) -c "import sys;t='{v[0]}_{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") PYVD=$(shell $(PYTHON) -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") -VENV?=venv_merlin_$(SYS_TYPE)_py$(PYV) -CERT?=/etc/pki/tls/cert.pem +VENV?=venv_merlin_py$(PYV) PIP?=$(VENV)/bin/pip -MRLN?=merlin/ -TEST?=tests/ +MRLN=merlin/ +TEST=tests/ +WKFW=merlin/examples/workflows/ MAX_COMPLEXITY?=5 -VENVMOD?=venv PENV=merlin$(PYV) .PHONY : all -.PHONY : install +.PHONY : install-dev .PHONY : virtualenv +.PHONY : install-workflow-deps .PHONY : install-pip-mysql -.PHONY : install-tasks -.PHONY : install-scipy +.PHONY : install-merlin .PHONY : update .PHONY : pull .PHONY : clean-output @@ -59,40 +58,34 @@ PENV=merlin$(PYV) .PHONY : check-style .PHONY : check-camel-case .PHONY : checks -.PHONY : start-workers -all: install install-tasks install-pip-mysql install-sphinx +all: install-dev install-merlin install-workflow-deps install-pip-mysql # install requirements -install: virtualenv - $(VENV)/bin/easy_install cryptography - $(PIP) install --cert $(CERT) -r requirements.txt +install-dev: virtualenv + $(PIP) install -r requirements/dev.txt # this only works outside the venv virtualenv: - $(PYTHON) -m $(VENVMOD) $(VENV) --prompt $(PENV) --system-site-packages - $(PIP) install --cert $(CERT) --upgrade pip + $(PYTHON) -m venv $(VENV) --prompt $(PENV) --system-site-packages + $(PIP) install --upgrade pip -install-sphinx: - $(PIP) install --upgrade sphinx +install-workflow-deps: + $(PIP) install -r $(WKFW)feature_demo/requirements.txt install-pip-mysql: $(PIP) install -r requirements/mysql.txt -install-tasks: +install-merlin: $(PIP) install -e . -install-scipy: - $(PIP) install --cert $(CERT) scipy --ignore-installed - - # this only works outside the venv update: pull install clean @@ -110,7 +103,6 @@ clean-py: # remove all studies/ directories clean-output: -find $(MRLN) -name "studies*" -type d -exec rm -rf {} \; - -find workflows/ -name "studies*" -type d -exec rm -rf {} \; -find . -maxdepth 1 -name "studies*" -type d -exec rm -rf {} \; -find . -maxdepth 1 -name "merlin.log" -type f -exec rm -rf {} \; @@ -156,8 +148,3 @@ check-camel-case: clean-py # run all checks checks: check-style check-camel-case - -# basic shortcut for starting celery workers -start-workers: - celery worker -A merlin -l INFO - diff --git a/README.md b/README.md index 9d8be283d..5959c0dcd 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,11 @@ -![alt text][logo] +![Merlin](https://raw.githubusercontent.com/LLNL/merlin/master/docs/images/merlin.png) -[logo]: https://github.com/LLNL/merlin/blob/master/docs/images/merlin.png "Merlin logo" - -Welcome to the Merlin README, a condensed guide. For more in-depth Merlin - information, try our [web docs here](https://merlin.readthedocs.io/). - -See the [CHANGELOG](CHANGELOG.md) for up-to-date details about features, - fixes, etc. - -# A brief introduction to Merlin +## A brief introduction to Merlin Merlin is a tool for running machine learning based workflows. The goal of Merlin is to make it easy to build, run, and process the kinds of large scale HPC workflows needed for cognitive simulation. -At its heart, Merlin is a distributed task queueing system, designed to allow complex +At its heart, Merlin is a distributed task queuing system, designed to allow complex HPC workflows to scale to large numbers of simulations (we've done 100 Million on the Sierra Supercomputer). @@ -85,256 +77,52 @@ For more details, check out the rest of the [documentation](https://merlin.readt Need help? -# Quick Start +## Quick Start -Note: Merlin only supports Python 3.6+. +Note: Merlin supports Python 3.6+. +To install Merlin and its dependencies, run: -To install the project and set up its virtualenv with dependencies, run: - - $ make all - $ source venv_merlin_$SYS_TYPE_py$(PYVERSION)/bin/activate - # activate.csh for cshrc (venv_merlin_$SYS_TYPE_py$(PYVERSION)) $ + $ pip3 install merlinwf That's it. -To update the project: - - $ make update - To run something a little more like what you're interested in, -namely a demo workflow that has simulation and machine-learning: - - (venv) $ merlin run workflows/feature_demo/feature_demo.yaml - (venv) $ merlin run-workers workflows/feature_demo/feature_demo.yaml - -More documentation on the example workflows can be found under -'Running the Examples'. - -# Code of Conduct -Please note that Merlin has a -[**Code of Conduct**](.github/CODE_OF_CONDUCT.md). By participating in -the Merlin community, you agree to abide by its rules. - -# Running the Examples -Example workflows can be found in the `workflows/` directory. -They can be run with the command line interface (CLI). - - # This processes the workflow and creates tasks on the server - (venv) $ merlin run workflows/feature_demo/feature_demo.yaml - # This launches workers that can process those tasks - (venv) $ merlin run-workers workflows/feature_demo/feature_demo.yaml - - -# Using the CLI -A good way to use merlin is through the command line interface (CLI). -This allows you to both create tasks to be run, as well as stand up workers -for those tasks. - -For more information see: - - (venv) $ merlin --help - (venv) $ merlin run --help - (venv) $ merlin run-workers --help - (venv) $ merlin purge --help - -Run a workflow specified by the given file. - -`(venv) $ merlin run ` - -Stand up celery workers with queues specified in the workflow file. - -`(venv) $ merlin run-workers [--echo] [--worker-args "celery args"] [--steps step1 stepN]` - -A note on arguments: - -`[--echo]` Just process the file and print the appropriate command. - -`[--worker-args "celery args"]` Passes arguments to the workers - -`[--steps step1 ... stepN]` Just give workers for specific steps in the file. - - -To remove tasks from the task server, use the purge option: - -`(venv) $ merlin purge ` - -More information can be obtained by running: - - (venv) $ merlin purge --help - - usage: merlin purge [-h] [-f] [--steps PURGE_STEPS [PURGE_STEPS ...]] - specification - - positional arguments: - specification Path to a Merlin YAML spec file - - optional arguments:` - -h, --help Show this help message and exit - -f, --force Purge the tasks without confirmation (default: False) - --steps PURGE_STEPS [PURGE_STEPS ...] - The specific steps in the YAML file from which you - want to purge the queues. The input is a space - separated list. (default: None) - -## Some real-life examples: - -Run workers for the feature_demo.yaml file: - - (venv) $ merlin run-workers workflows/feature_demo/feature_demo.yaml --worker-args "--prefetch-multiplier 1" - $ celery worker -A merlin --prefetch-multiplier 1 -Q merlin,hello_queue,post_process_queue - -Just run workers for the hello step in the file: - - (venv) $ merlin run-workers workflows/feature_demo/feature_demo.yaml --steps hello - $ celery worker -A merlin -Q hello_queue - -Adding `--echo` to a command will just print out the command, so you can move this command into more complex workflows. - -For instance, to put into a batch script, or run many workers you can do stuff like this: - - $ srun -n 5 `merlin run-workers workflows/feature_demo/feature_demo.yaml --echo --steps hello` +namely a demo workflow that has simulation and machine-learning, +first generate an example workflow: -Which is equivalent to + $ merlin example feature_demo - $ srun -n 5 celery worker -A merlin -Q hello_queue +Then install the workflow's dependencies: -Generate a template spec file. + $ pip install -r feature_demo/requirements.txt -`(venv)` $ merlin template +Then process the workflow and create tasks on the server: -Show pip and python versions and locations. This is useful for troubleshooting. + $ merlin run feature_demo/feature_demo.yaml -`(venv) $ merlin info` +And finally, launch workers that can process those tasks: -Display version number. + $ merlin run-workers feature_demo/feature_demo.yaml -`(venv) $ merlin -v` or `(venv) $ merlin --version` -Display these CLI options in-console. +## Documentation +[**Full documentation**](http://merlin.readthedocs.io/) is available, or +run: -`(venv) $ merlin -h` or `(venv) $ merlin --help` + $ merlin --help -# Custom Setup +(or add `--help` to the end of any sub-command you +want to learn more about.) -## Create and activate a Virtual Environment - $ python -m virtualenv venv - $ source venv/bin/activate # Or activate.csh for cshrc. - (venv) $ - -## Upgrade Pip - - (venv) $ pip install -U pip - -## Add requirements to your environment - - # This will install all package dependencies into the virtualenv. - (venv) $ pip install -r requirements.txt - (venv) $ pip install -e . - -## Adding MySQL packages - -If using MySQL, then the following requirements should also be installed: - - (venv) $ pip install -r requirements/mysql.txt - -## Celery - -Celery is a distributed task queue that helps spread work over threads and machines. -Before running a distributed job, use: - - (venv) $ celery worker -A merlin -l INFO - -For very small tests this may done on a login node, but otherwise celery workers should -be scheduled via the system's batch. -Some useful flags are: `--concurrency N`, `-Ofair`, `--prefetch-multiplier M`. -For more details, see the batch scripts in `merlin/examples/` or type `celery -h` for help. - -# redis - -The redis system is currently being used to implement the backend server on -rabbit.llnl.gov. This same redis system can be used as the frontend broker -and can also be run on the local allocation instead of a remote server. The -instructions below detail the method for implementing a local broker or -backend. - -## Build redis - -Download the code from: - https://redis.io/download - -untar the code - - tar xvf redis-4.0.11.tar.gz - -Type make in the top level redis code directory. - - make - -The executables will be in src. - -## Local redis server - -In this example, the same redis server is used as the broker and -backend at port 6397 of localhost. - -### Start the server - -Run redis-server on an allocation node using the default config. - - redis-server redis.conf - -### Configure Merlin to use the local redis server - -Edit the app.yaml file and use these configurations - - broker: - name: redis - server: localhost - port: 6379 - - results_backend: - name: redis - server: localhost - port: 6379 - -## Local redis+socket - -This configuration will use redis+socket for the broker and the same redis as the backend -server connection to port 6397 on the localhost. - -Edit the redis.conf file to turn on the socket interface, this method -only works on a single node: - - unixsocket //redis.sock - unixsocketperm 700 - -### Start the server - -Run redis-server on an allocation node. - - redis-server redis.conf - -### Configure Merlin to use the local redis server - -Edit the `app.yaml` file and use these configurations: - - broker: - name: redis+socket - path: //redis.sock - socketname: redis.sock +## Code of Conduct +Please note that Merlin has a +[**Code of Conduct**](.github/CODE_OF_CONDUCT.md). By participating in +the Merlin community, you agree to abide by its rules. - results_backend: - name: redis - server: localhost - port: 6379 -# Testing -## Unit tests -* `(venv) $ make version-tests` or `(venv) $ tox` runs Merlin tests in different Python version environments. See `tox.ini` file for details on what this runs. -* From the main directory, `(venv) $ py.tests merlin/` runs Merlin unit tests. -## Style checks -* `(venv) $ make check-style` or `(venv) $ pylint merlin/` checks for PEP8 infractions in our Python code. -To customize what is tested for, see `.pylintrc` and `tox.ini` files. +## License +Merlin is distributed under the terms of the [MIT LICENSE](https://github.com/LLNL/merlin/blob/master/LICENSE). +LLNL-CODE-797170 diff --git a/docs/source/app_config/app_amqp.yaml b/docs/source/app_config/app_amqp.yaml index 0b8f35878..921dde755 100644 --- a/docs/source/app_config/app_amqp.yaml +++ b/docs/source/app_config/app_amqp.yaml @@ -1,7 +1,8 @@ + celery: # directory where Merlin looks for the following: # mysql-ca-cert.pem rabbit-client-cert.pem rabbit-client-key.pem redis.pass - certs: /usr/workspace/wsb/icfsi/merlin/config + certs: /path/to/celery/config broker: # can be redis, redis+sock, or rabbitmq @@ -9,7 +10,7 @@ broker: #username: # defaults to your username unless changed here password: ~/.merlin/rabbit-password # server URL - server: jackalope.llnl.gov + server: server.domain.com ### for rabbitmq, redis+sock connections ### #vhost: # defaults to your username unless changed here @@ -26,12 +27,12 @@ broker: results_backend: # must be redis name: redis - - dbname: mlsi - username: mlsi + + dbname: dbname + username: username # name of file where redis password is stored. password: redis.pass - server: jackalope.llnl.gov + server: server.domain.com # merlin will generate this key if it does not exist yet, # and will use it to encrypt all data over the wire to # your redis server. diff --git a/docs/source/app_config/app_redis.yaml b/docs/source/app_config/app_redis.yaml deleted file mode 100644 index 3d30a7174..000000000 --- a/docs/source/app_config/app_redis.yaml +++ /dev/null @@ -1,14 +0,0 @@ -celery: - certs: /usr/workspace/wsb/icfsi/merlin/config - -broker: - name: redis - server: localhost - port: 6379 - -results_backend: - name: redis - server: localhost - port: 6379 - - diff --git a/docs/source/app_config/app_rzamqp.yaml b/docs/source/app_config/app_rzamqp.yaml deleted file mode 100644 index 36364b4c9..000000000 --- a/docs/source/app_config/app_rzamqp.yaml +++ /dev/null @@ -1,17 +0,0 @@ -celery: - certs: /usr/workspace/wsrzd/icfsi/merlin/config - -broker: - name: rabbitmq - username: mlsi - password: ~/.merlin/rzrabbit.password # The filename that contains the password, from ..icfsi/merlin/config dir - server: rzrabbit.llnl.gov - -results_backend: - name: redis - dbname: mlsi - username: mlsi - password: ~/.merlin/rzredis.pass # The filename that contains the password. from ..icfsi/merlin/config dir - server: rzrabbit.llnl.gov - - diff --git a/docs/source/conf.py b/docs/source/conf.py index b609425c4..cee29d8f2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,21 +12,26 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +from datetime import date import os import sys + sys.path.insert(0, os.path.abspath('../../..')) +MERLIN_VERSION = __import__("merlin").VERSION # -- Project information ----------------------------------------------------- +_year = date.today().year + project = u'Merlin' -copyright = u'2018, MLSI' +copyright = f'{_year}, MLSI' author = u'MLSI' # The short X.Y version -version = u'1.0.0' +version = MERLIN_VERSION # The full version, including alpha/beta/rc tags -release = u'1.0.0' +release = MERLIN_VERSION # -- General configuration --------------------------------------------------- diff --git a/docs/source/faq.rst b/docs/source/faq.rst index cf8842ce8..c4fd67e14 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -94,7 +94,7 @@ for more details. What is flux? ~~~~~~~~~~~~~ Flux is a hierarchical scheduler and launcher for parallel simulations. It allows the user -to specifiy the same launch command that will work on different HPC clusters with different +to specify the same launch command that will work on different HPC clusters with different default schedulers such as SLURM or LSF. More information can be found at the `Flux web page `_. @@ -104,7 +104,7 @@ Designing and Building Workflows Where are some example workflows? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``merlin-templates`` +``merlin example --help`` How do I launch a workflow? ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -178,7 +178,7 @@ How do I specify the language used in a step? You can add the field ``shell`` under the ``run`` portion of your step to change the language you write your step in. The default is ``/bin/bash``, but you can do things like ``/usr/bin/env python`` as well. -See the ``basic_ensemble.yaml`` for an example. +Use ``merlin example feature_demo`` to see an example of this. Running Workflows ----------------- @@ -236,3 +236,7 @@ You might also have rogue workers. To find out, try ``merlin query-workers``. Where do tasks get run? ~~~~~~~~~~~~~~~~~~~~~~~ + +Where can I learn more about merlin? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Check out our (paper)[https://arxiv.org/abs/1912.02892] on arXiv. diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index e469a826c..414dbfd72 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -1,58 +1,30 @@ Getting Started ================ -Quick Start Setup -++++++++++++++++++ - -This setup will work in most cases and can be used to quickly setup Merlin. - -Clone the `Merlin `_ -repository. See the :doc:`Spack <./spack>` section for an alternative method to setup merlin. - +Quick Start +++++++++++++++ :: + pip3 install merlinwf + +All set up? See the :doc:`Merlin Commands <./merlin_commands>` section for using merlin. - git clone https://github.com/LLNL/merlin.git - -Then run the following:: - - cd merlin - make all - -for a different python3 version:: - cd merlin - make PYTHON=python3-3.7.2 all - -The Makefile should have created a virtualenv and installed all required -dependencies. - -Activate the virtualenv:: - - $ source venv_merlin__py/bin/activate # Or activate.csh if using .cshrc - # The prompt will be merlin_ - # This can be changed by setting the PENV variable in the Makefile. - (merlin3_7) $ - - with python 3.7.* this will be, - $ source venv_merlin_${SYS_TYPE}_py3_7/bin/activate - -.. note:: The ${SYS_TYPE} variable may not be defined on your machine. - -.. note:: A virtualenv may only be created for the current system. This virtualenv - setup process must be repeated for each different system being run on. - -See the :doc:`Merlin Commands <./merlin_commands>` section for using merlin. +Developer Setup +++++++++++++++++++ +To install with the additional developer dependencies, use:: + pip3 install "merlinwf[dev]" + +or:: + pip3 install -e "git+https://github.com/LLNL/merlin.git@develop#egg=merlinwf[dev]" -To exit the virtualenv:: +See the :doc:`Spack <./spack>` section for an alternative method to setup merlin on supercomputers. - (merlin3_7) $ deactivate - $ Configuring Merlin ******************* Once Merlin has been installed, the installation needs to be configured. -Documentiation for merlin configuration is in the :doc:`Configuring Merlin <./merlin_config>` section. +Documentation for merlin configuration is in the :doc:`Configuring Merlin <./merlin_config>` section. That's it. To start running Merlin see the :doc:`Merlin Workflows. <./merlin_workflows>` diff --git a/docs/source/index.rst b/docs/source/index.rst index 45e7658b5..33c83c380 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -12,7 +12,7 @@ scale HPC workflows needed for cognitive simulation. Merlin Overview --------------- -Merlin is a distributed task queueing system, designed to allow complex +Merlin is a distributed task queuing system, designed to allow complex HPC workflows to scale to large numbers of simulations (we've done 100 Million on the Sierra Supercomputer). diff --git a/docs/source/merlin_commands.rst b/docs/source/merlin_commands.rst index f6d71fe4e..694e00aa0 100644 --- a/docs/source/merlin_commands.rst +++ b/docs/source/merlin_commands.rst @@ -236,34 +236,32 @@ whose name matches a regular expression: only one might get the signal. In this case, you can send it again. -Generate new spec template --------------------------- +Generate working examples +------------------------- -If you want to start a new workflow from a built-in template, use Merlin's -template builder ``merlin-templates``: +If you want to run an example workflow, use Merlin's ``merlin example``: .. code:: bash - (merlin3_7) $ merlin-templates list + (merlin3_7) $ merlin example --help -This will list the available templates and a description for each one. To -select a template: +This will list the available example workflows and a description for each one. To +select one: .. code:: bash - (merlin3_7) $ merlin-templates setup + (merlin3_7) $ merlin example -This will copy the template file to the current working directory. It is -possible to specify a path to copy the template to. +This will copy the example workflow to the current working directory. It is +possible to specify another path to copy to. .. code:: bash - (merlin3_7) $ merlin-templates setup -p path/to/dir + (merlin3_7) $ merlin example -p path/to/dir If the specified directory does not exist Merlin will automatically create it. -This will generate the template file at the specified location, ready for your -editing. +This will generate the example workflow at the specified location, ready to be run. Purging Tasks diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index 8ee49881f..fd312a188 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -38,13 +38,13 @@ Reserved variables * - ``$(OUTPUT_PATH)`` - Directory path the study output will be written to. If not defined will default to the current working directory. May be reassigned or - overriden. + overridden. - ``./studies`` * - ``$(MERLIN_TIMESTAMP)`` - The time a study began. May be used as a unique identifier. - ``"YYYYMMDD-HHMMSS"`` * - ``$(MERLIN_WORKSPACE)`` - - Output directory gererated by a study at ``OUTPUT_PATH``. Ends with + - Output directory generated by a study at ``OUTPUT_PATH``. Ends with ``MERLIN_TIMESTAMP``. - ``$(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP)`` * - ``$(WORKSPACE)`` @@ -130,7 +130,7 @@ Step return variables exit $(MERLIN_RESTART) max_retries: 23 - * - ``$(meriln_soft_fail)`` + * - ``$(merlin_soft_fail)`` - Mark this step as a failure, note in the warning log but keep going. Unknown return codes get translated to soft fails, so that they can be logged. diff --git a/docs/source/testing.rst b/docs/source/testing.rst index d20ee7086..9b6ef1949 100644 --- a/docs/source/testing.rst +++ b/docs/source/testing.rst @@ -50,7 +50,7 @@ Continuous integration testing ****************************** -Continuous integration testing is being implemented for the github repo. +Continuous integration testing is being implemented for the GitHub repo. .. _style-checks: diff --git a/merlin/__init__.py b/merlin/__init__.py index 42427cc35..cc85d79bc 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.0.5" +__version__ = "1.1.0" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 817295473..152fb18bd 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 94c05e09d..71e16ef39 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -29,7 +29,10 @@ ############################################################################### """Updated celery configuration.""" -from __future__ import absolute_import, print_function +from __future__ import ( + absolute_import, + print_function, +) import logging import os @@ -40,7 +43,10 @@ from celery.signals import worker_process_init import merlin.common.security.encrypt_backend_traffic -from merlin.config import broker, results_backend +from merlin.config import ( + broker, + results_backend, +) from merlin.log_formatter import FORMATS from merlin.router import route_for_task diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index abd4eeaa9..e26ce3a51 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index abd4eeaa9..e26ce3a51 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 14f09dc05..00e03caa1 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index a16470e25..31e00be50 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -60,14 +60,11 @@ import copy -unistr = (unicode, str) - - class OpenFileList: openwas = open def __new__(cls, files, *v, **kw): - if isinstance(files, unistr): + if isinstance(files, str): return open(files, *v, **kw) return super(OpenFileList, cls).__new__(cls) diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 9c7e86958..fe9f48452 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 4103e803f..c20840a5d 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 77334dc99..4f1816ee3 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -33,7 +33,10 @@ """ from parse import parse -from merlin.common.sample_index import MAX_SAMPLE, SampleIndex +from merlin.common.sample_index import ( + MAX_SAMPLE, + SampleIndex, +) from merlin.utils import cd diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index abd4eeaa9..e26ce3a51 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 5a48e5d02..93e468451 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 78256de31..3191a1081 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index c2a773d8d..3d5c9ac19 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -29,18 +29,34 @@ ############################################################################### """Test tasks.""" -from __future__ import absolute_import, unicode_literals +from __future__ import ( + absolute_import, + unicode_literals, +) import logging import os -from celery import chain, chord, group, shared_task, signature -from celery.exceptions import OperationalError, TimeoutError +from celery import ( + chain, + chord, + group, + shared_task, + signature, +) +from celery.exceptions import ( + OperationalError, + TimeoutError, +) from merlin.common.abstracts.enums import ReturnCode from merlin.common.sample_index import uniform_directories from merlin.common.sample_index_factory import create_hierarchy -from merlin.exceptions import HardFailException, InvalidChainException, RetryException +from merlin.exceptions import ( + HardFailException, + InvalidChainException, + RetryException, +) from merlin.router import stop_workers from merlin.spec.expansion import ( parameter_substitutions_for_cmd, @@ -245,7 +261,7 @@ def add_merlin_expanded_chain_to_chord( LOG.debug(f"adding chain to chord") add_chains_to_chord(self, all_chains) else: - # recurse down the sample_index heirarchy + # recurse down the sample_index hierarchy for next_index in sample_index.children.values(): next_index.name = os.path.join(sample_index.name, next_index.name) next_step = add_merlin_expanded_chain_to_chord.s( @@ -374,7 +390,7 @@ def expand_tasks_with_samples( :labels : A list of strings containing the label associated with each column in the samples. :task_type : The celery task type to create. Currently always merlin_step. :adapter_config : A dictionary used for configuring maestro script adapters. - :level_max_dirs : The max number of directories per level in the sample heirarchy. + :level_max_dirs : The max number of directories per level in the sample hierarchy. """ LOG.debug(f"expand_tasks_with_samples called with chain,{chain}\n") @@ -413,7 +429,7 @@ def expand_tasks_with_samples( if needs_expansion: prepare_chain_workspace(sample_index, steps) sample_index.name = "" - LOG.debug(f"queueing merlin expansion task") + LOG.debug(f"queuing merlin expansion task") sig = add_merlin_expanded_chain_to_chord.s( task_type, steps, samples, labels, sample_index, adapter_config, 0 ) @@ -424,7 +440,7 @@ def expand_tasks_with_samples( self.add_to_chord(sig, lazy=False) LOG.debug(f"merlin expansion task queued") else: - LOG.debug(f"queueing simple chain task") + LOG.debug(f"queuing simple chain task") add_simple_chain_to_chord(self, task_type, steps, adapter_config) LOG.debug(f"simple chain task queued") diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index d4ad923e1..6f3ca9eb4 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 3d01ee291..209382e13 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index af72bf780..3927b4c76 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index d5992f45a..75a52ce58 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index c8f4a7ea9..771d82ff4 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index abd4eeaa9..e26ce3a51 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index f499af709..42875c767 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -37,7 +37,10 @@ from tabulate import tabulate from merlin.ascii_art import banner_small -from merlin.config import broker, results_backend +from merlin.config import ( + broker, + results_backend, +) from merlin.config.configfile import default_config_info diff --git a/merlin/templates/__init__.py b/merlin/examples/__init__.py similarity index 97% rename from merlin/templates/__init__.py rename to merlin/examples/__init__.py index abd4eeaa9..e26ce3a51 100644 --- a/merlin/templates/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/templates/examples.py b/merlin/examples/examples.py similarity index 99% rename from merlin/templates/examples.py rename to merlin/examples/examples.py index e1a91fe41..cdfb77c6e 100644 --- a/merlin/templates/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py new file mode 100644 index 000000000..78e3fb4a8 --- /dev/null +++ b/merlin/examples/generator.py @@ -0,0 +1,118 @@ +############################################################################### +# Copyright (c) 2019, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.1.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### + +""" +This module contains a list of examples that can be used when learning to use +Merlin, or for setting up new workflows. +""" +import glob +import logging +import os +import shutil + +import tabulate +import yaml + +from merlin.examples import examples + + +LOG = logging.getLogger(__name__) + +EXAMPLES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "workflows") + + +def gather_example_dirs(): + result = {} + for d in os.listdir(EXAMPLES_DIR): + result[d] = d + return result + + +def gather_all_examples(): + path = os.path.join(os.path.join(EXAMPLES_DIR, ""), os.path.join("*", "*.yaml")) + return glob.glob(path) + + +def write_example(src_path, dst_path): + """ + Write out the example workflow to a file. + :param src_path: The path to copy from. + :param content: The formatted content to write the file to. + """ + if os.path.isdir(src_path): + shutil.copytree(src_path, dst_path) + else: + shutil.copy(src_path, dst_path) + + +def list_examples(): + """List all available examples.""" + headers = ["name", "description"] + rows = [] + for example_dir in gather_example_dirs(): + directory = os.path.join(os.path.join(EXAMPLES_DIR, example_dir), "") + specs = glob.glob(directory + "*.yaml") + for spec in specs: + with open(spec) as f: + spec_metadata = yaml.safe_load(f)["description"] + rows.append([spec_metadata["name"], spec_metadata["description"]]) + return "\n" + tabulate.tabulate(rows, headers) + "\n" + + +def setup_example(name, outdir): + """Setup the given example.""" + example = None + spec_paths = gather_all_examples() + for spec_path in spec_paths: + spec = os.path.basename(os.path.normpath(spec_path)).replace(".yaml", "") + if name == spec: + example = os.path.basename(os.path.dirname(spec_path)) + break + if example is None: + LOG.error(f"Example '{name}' not found.") + return None + + if outdir is None: + outdir = os.getcwd() + + # if there is only 1 file in the example, don't bother making a directory for it + if len(os.listdir(os.path.dirname(spec_path))) == 1: + src_path = os.path.join(EXAMPLES_DIR, os.path.join(example, example + ".yaml")) + else: + src_path = os.path.join(EXAMPLES_DIR, example) + + outdir = os.path.join(outdir, example) + if os.path.exists(outdir): + LOG.error(f"File '{outdir}' already exists!") + return None + + LOG.info(f"Copying example '{name}' to {outdir}") + write_example(src_path, outdir) + return example diff --git a/workflows/feature_demo/.gitignore b/merlin/examples/workflows/feature_demo/.gitignore similarity index 100% rename from workflows/feature_demo/.gitignore rename to merlin/examples/workflows/feature_demo/.gitignore diff --git a/workflows/feature_demo/feature_demo.yaml b/merlin/examples/workflows/feature_demo/feature_demo.yaml similarity index 97% rename from workflows/feature_demo/feature_demo.yaml rename to merlin/examples/workflows/feature_demo/feature_demo.yaml index e8e1b0944..59be5b104 100644 --- a/workflows/feature_demo/feature_demo.yaml +++ b/merlin/examples/workflows/feature_demo/feature_demo.yaml @@ -18,10 +18,8 @@ env: TRANSLATE: $(SCRIPTS)/translator.py COLLECT: $(SCRIPTS)/collector.py MAKE_SAMPLES: $(SCRIPTS)/make_samples.py - - SHARED: $(SPECROOT)/../shared - LEARN: $(SHARED)/learn.py - PREDICT: $(SHARED)/predict.py + LEARN: $(SCRIPTS)/learn.py + PREDICT: $(SCRIPTS)/predict.py study: - name: hello diff --git a/workflows/feature_demo/requirements.txt b/merlin/examples/workflows/feature_demo/requirements.txt similarity index 100% rename from workflows/feature_demo/requirements.txt rename to merlin/examples/workflows/feature_demo/requirements.txt diff --git a/workflows/feature_demo/scripts/collector.py b/merlin/examples/workflows/feature_demo/scripts/collector.py similarity index 100% rename from workflows/feature_demo/scripts/collector.py rename to merlin/examples/workflows/feature_demo/scripts/collector.py diff --git a/workflows/feature_demo/scripts/features.json b/merlin/examples/workflows/feature_demo/scripts/features.json similarity index 100% rename from workflows/feature_demo/scripts/features.json rename to merlin/examples/workflows/feature_demo/scripts/features.json diff --git a/workflows/feature_demo/scripts/hello_world.py b/merlin/examples/workflows/feature_demo/scripts/hello_world.py similarity index 100% rename from workflows/feature_demo/scripts/hello_world.py rename to merlin/examples/workflows/feature_demo/scripts/hello_world.py index 1e7e6de13..efd275028 100644 --- a/workflows/feature_demo/scripts/hello_world.py +++ b/merlin/examples/workflows/feature_demo/scripts/hello_world.py @@ -1,6 +1,6 @@ import argparse -import sys import json +import sys def process_args(args): diff --git a/workflows/shared/learn.py b/merlin/examples/workflows/feature_demo/scripts/learn.py similarity index 99% rename from workflows/shared/learn.py rename to merlin/examples/workflows/feature_demo/scripts/learn.py index ed478463d..12570ada8 100644 --- a/workflows/shared/learn.py +++ b/merlin/examples/workflows/feature_demo/scripts/learn.py @@ -1,13 +1,15 @@ import argparse +import sys + +import numpy as np + +from sklearn.ensemble import RandomForestRegressor + try: import cPickle as pickle except ImportError: import pickle -import numpy as np -import sys - -from sklearn.ensemble import RandomForestRegressor FOREST_DEFAULTS = {"max_depth": 2, "random_state": 0, "n_estimators": 100} diff --git a/workflows/feature_demo/scripts/make_samples.py b/merlin/examples/workflows/feature_demo/scripts/make_samples.py similarity index 100% rename from workflows/feature_demo/scripts/make_samples.py rename to merlin/examples/workflows/feature_demo/scripts/make_samples.py index b9edbdd2f..de96a4607 100644 --- a/workflows/feature_demo/scripts/make_samples.py +++ b/merlin/examples/workflows/feature_demo/scripts/make_samples.py @@ -1,7 +1,7 @@ import argparse import sys -import numpy as np +import numpy as np from maestrowf.datastructures.core import ParameterGenerator diff --git a/workflows/shared/predict.py b/merlin/examples/workflows/feature_demo/scripts/predict.py similarity index 99% rename from workflows/shared/predict.py rename to merlin/examples/workflows/feature_demo/scripts/predict.py index 4ee3e9086..a7abe7136 100644 --- a/workflows/shared/predict.py +++ b/merlin/examples/workflows/feature_demo/scripts/predict.py @@ -1,12 +1,13 @@ import argparse +import sys + +import numpy as np + try: import cPickle as pickle except ImportError: import pickle -import sys - -import numpy as np def setup_argparse(): diff --git a/workflows/feature_demo/scripts/translator.py b/merlin/examples/workflows/feature_demo/scripts/translator.py similarity index 88% rename from workflows/feature_demo/scripts/translator.py rename to merlin/examples/workflows/feature_demo/scripts/translator.py index bd1ddbcf6..8413d9b3f 100644 --- a/workflows/feature_demo/scripts/translator.py +++ b/merlin/examples/workflows/feature_demo/scripts/translator.py @@ -1,9 +1,10 @@ import argparse import json -import numpy as np import os import sys +import numpy as np + def setup_argparse(): parser = argparse.ArgumentParser(description="Translate .json into numpy") @@ -23,16 +24,15 @@ def setup_argparse(): def process_args(args): - #data = cb.load_node(args.input) + # data = cb.load_node(args.input) samples = json.load(open(args.input, "r")) schema = json.load(open(args.schema, "r")) - input_array_dict = {} output_array_dict = {} for s in samples: - make_data_array_dict(input_array_dict, s["inputs"],schema["inputs"]) - make_data_array_dict(output_array_dict, s["outputs"],schema["outputs"]) + make_data_array_dict(input_array_dict, s["inputs"], schema["inputs"]) + make_data_array_dict(output_array_dict, s["outputs"], schema["outputs"]) for path in output_array_dict: output_array_dict[path] = np.array(output_array_dict[path]) @@ -58,7 +58,7 @@ def generate_scalar_path_pairs(node, schema, path=""): ): yield pair else: - if not isinstance(schema[child],dict): + if not isinstance(schema[child], dict): yield path + child, node[child] diff --git a/workflows/flux/flux_par.yaml b/merlin/examples/workflows/flux/flux_par.yaml similarity index 94% rename from workflows/flux/flux_par.yaml rename to merlin/examples/workflows/flux/flux_par.yaml index eda977d2c..f5ebecc65 100644 --- a/workflows/flux/flux_par.yaml +++ b/merlin/examples/workflows/flux/flux_par.yaml @@ -1,6 +1,6 @@ description: - description: Run a scan through Merlin/Maestro - name: FLUXPARMERLIN + description: A simple ensemble of parallel MPI jobs run by flux. + name: flux_par batch: type: flux diff --git a/workflows/flux/flux_test.yaml b/merlin/examples/workflows/flux/flux_test.yaml similarity index 91% rename from workflows/flux/flux_test.yaml rename to merlin/examples/workflows/flux/flux_test.yaml index 2245aac1d..c23809c72 100644 --- a/workflows/flux/flux_test.yaml +++ b/merlin/examples/workflows/flux/flux_test.yaml @@ -1,6 +1,6 @@ description: - description: Run a scan through Merlin/Maestro - name: FLUXTESTMERLIN + description: A simple ensemble of echo commands run by flux. + name: flux_test batch: type: flux diff --git a/workflows/flux/paper.yaml b/merlin/examples/workflows/flux/paper.yaml similarity index 92% rename from workflows/flux/paper.yaml rename to merlin/examples/workflows/flux/paper.yaml index 4b554d75a..b15f1bf61 100644 --- a/workflows/flux/paper.yaml +++ b/merlin/examples/workflows/flux/paper.yaml @@ -1,6 +1,6 @@ description: - description: Run a scan through Merlin/Maestro, this yaml file assumes bash and will give timings for all the runs at the end. - name: PAPERFLUXMERLIN + description: Use flux to run single core MPI jobs and record timings. + name: paper_flux batch: type: flux diff --git a/workflows/flux/requirements.txt b/merlin/examples/workflows/flux/requirements.txt similarity index 100% rename from workflows/flux/requirements.txt rename to merlin/examples/workflows/flux/requirements.txt diff --git a/merlin/examples/workflows/flux/scripts/flux_info.py b/merlin/examples/workflows/flux/scripts/flux_info.py new file mode 100755 index 000000000..bf45d6df5 --- /dev/null +++ b/merlin/examples/workflows/flux/scripts/flux_info.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python2 +"""This module will collect information on flux jobs from the live kvs +store and output times for each phase. + +create: The time flux registerd the job +starting: The time the job was created +running: The time the job was running +completing: The time the job started its' completion pahse. +complete: The time the job was complete +walltime: ? Seems to be 0. +""" +import os +import sys + +import flux +from flux import ( + kvs, + kz, +) + + +f = flux.Flux() + +fs = "FLUX_START_SECONDS" +if fs in os.environ: + print("flux start: {0}".format(os.environ[fs])) + +for d in kvs.walk("lwj", flux_handle=f): + try: + # print(type(d)) + fdir = "lwj.{0}".format(d[0]) + + qcreate = "{0}.create-time".format(fdir) + create_time = kvs.get(f, qcreate) + + qstart = "{0}.starting-time".format(fdir) + start_time = kvs.get(f, qstart) + + qrun = "{0}.running-time".format(fdir) + start_time = kvs.get(f, qrun) + + qcomplete = "{0}.complete-time".format(fdir) + complete_time = kvs.get(f, qcomplete) + + qcompleting = "{0}.completing-time".format(fdir) + completing_time = kvs.get(f, qcompleting) + + qwall = "{0}.walltime".format(fdir) + wall_time = kvs.get(f, qwall) + + proto = "Job {0}: create: {1} start {1} run {2} completing {3} complete {4} wall {5}" + print( + proto.format( + d[0], create_time, start_time, completing_time, complete_time, wall_time + ) + ) + except: + pass + + +# vi: ts=4 sw=4 expandtab diff --git a/workflows/flux/scripts/hello.c b/merlin/examples/workflows/flux/scripts/hello.c similarity index 100% rename from workflows/flux/scripts/hello.c rename to merlin/examples/workflows/flux/scripts/hello.c diff --git a/workflows/flux/scripts/hello_sleep.c b/merlin/examples/workflows/flux/scripts/hello_sleep.c similarity index 100% rename from workflows/flux/scripts/hello_sleep.c rename to merlin/examples/workflows/flux/scripts/hello_sleep.c diff --git a/workflows/flux/scripts/make_samples.py b/merlin/examples/workflows/flux/scripts/make_samples.py similarity index 56% rename from workflows/flux/scripts/make_samples.py rename to merlin/examples/workflows/flux/scripts/make_samples.py index 91f32ba5a..f06ba96ba 100644 --- a/workflows/flux/scripts/make_samples.py +++ b/merlin/examples/workflows/flux/scripts/make_samples.py @@ -1,16 +1,18 @@ import argparse import sys + import numpy as np from merlin.common.util_sampling import scale_samples + def process_args(args): n_samples = args.n n_dims = args.dims sample_type = args.sample_type - if sample_type == 'random': + if sample_type == "random": x = np.random.random((n_samples, n_dims)) - elif sample_type == 'grid': + elif sample_type == "grid": subdivision = int(pow(n_samples, 1 / float(n_dims))) temp = [np.linspace(0, 1.0, subdivision) for i in range(n_dims)] X = np.meshgrid(*temp) @@ -22,33 +24,37 @@ def process_args(args): scales = ast.literal_eval(args.scale) for scale in scales: limits.append((scale[0], scale[1])) - if scale[2] == 'log': - do_log.append(True) + if scale[2] == "log": + do_log.append(True) else: - do_log.append(False) + do_log.append(False) x = scale_samples(x, limits, do_log=do_log) np.save(args.outfile, x) def setup_argparse(): - parser = argparse.ArgumentParser('Generate some samples!') - parser.add_argument('-n', help='number of samples', default=100, type=int) - parser.add_argument('-dims', help='number of dimensions', default=2, type=int) - parser.add_argument('-sample_type', - help='type of sampling. options: random, grid. If grid, will try to get close to the correct number of samples', default='random') - parser.add_argument( '-scale', - help='ranges to scale results in form "[(min,max,type),(min, max,type)]" where type = "linear" or "log"') - parser.add_argument('-outfile', help='name of output .npy file', default='samples') + parser = argparse.ArgumentParser("Generate some samples!") + parser.add_argument("-n", help="number of samples", default=100, type=int) + parser.add_argument("-dims", help="number of dimensions", default=2, type=int) + parser.add_argument( + "-sample_type", + help="type of sampling. options: random, grid. If grid, will try to get close to the correct number of samples", + default="random", + ) + parser.add_argument( + "-scale", + help='ranges to scale results in form "[(min,max,type),(min, max,type)]" where type = "linear" or "log"', + ) + parser.add_argument("-outfile", help="name of output .npy file", default="samples") return parser def main(): parser = setup_argparse() - args = parser.parse_args() + args = parser.parse_args() process_args(args) if __name__ == "__main__": sys.exit(main()) - diff --git a/workflows/flux/scripts/paper_workers.sbatch b/merlin/examples/workflows/flux/scripts/paper_workers.sbatch similarity index 100% rename from workflows/flux/scripts/paper_workers.sbatch rename to merlin/examples/workflows/flux/scripts/paper_workers.sbatch diff --git a/workflows/flux/scripts/test_workers.sbatch b/merlin/examples/workflows/flux/scripts/test_workers.sbatch similarity index 100% rename from workflows/flux/scripts/test_workers.sbatch rename to merlin/examples/workflows/flux/scripts/test_workers.sbatch diff --git a/workflows/flux/scripts/workers.bsub b/merlin/examples/workflows/flux/scripts/workers.bsub similarity index 100% rename from workflows/flux/scripts/workers.bsub rename to merlin/examples/workflows/flux/scripts/workers.bsub diff --git a/workflows/flux/scripts/workers.sbatch b/merlin/examples/workflows/flux/scripts/workers.sbatch similarity index 100% rename from workflows/flux/scripts/workers.sbatch rename to merlin/examples/workflows/flux/scripts/workers.sbatch diff --git a/workflows/simple_chain/simple_chain.yaml b/merlin/examples/workflows/simple_chain/simple_chain.yaml similarity index 97% rename from workflows/simple_chain/simple_chain.yaml rename to merlin/examples/workflows/simple_chain/simple_chain.yaml index b43c09169..726d9b6e8 100644 --- a/workflows/simple_chain/simple_chain.yaml +++ b/merlin/examples/workflows/simple_chain/simple_chain.yaml @@ -1,5 +1,5 @@ description: - name: simple chain + name: simple_chain description: test to see that chains are not run in parallel batch: diff --git a/workflows/slurm/requirements.txt b/merlin/examples/workflows/slurm/requirements.txt similarity index 100% rename from workflows/slurm/requirements.txt rename to merlin/examples/workflows/slurm/requirements.txt diff --git a/workflows/slurm/scripts/hello.c b/merlin/examples/workflows/slurm/scripts/hello.c similarity index 100% rename from workflows/slurm/scripts/hello.c rename to merlin/examples/workflows/slurm/scripts/hello.c diff --git a/workflows/slurm/scripts/make_samples.py b/merlin/examples/workflows/slurm/scripts/make_samples.py similarity index 56% rename from workflows/slurm/scripts/make_samples.py rename to merlin/examples/workflows/slurm/scripts/make_samples.py index 91f32ba5a..f06ba96ba 100644 --- a/workflows/slurm/scripts/make_samples.py +++ b/merlin/examples/workflows/slurm/scripts/make_samples.py @@ -1,16 +1,18 @@ import argparse import sys + import numpy as np from merlin.common.util_sampling import scale_samples + def process_args(args): n_samples = args.n n_dims = args.dims sample_type = args.sample_type - if sample_type == 'random': + if sample_type == "random": x = np.random.random((n_samples, n_dims)) - elif sample_type == 'grid': + elif sample_type == "grid": subdivision = int(pow(n_samples, 1 / float(n_dims))) temp = [np.linspace(0, 1.0, subdivision) for i in range(n_dims)] X = np.meshgrid(*temp) @@ -22,33 +24,37 @@ def process_args(args): scales = ast.literal_eval(args.scale) for scale in scales: limits.append((scale[0], scale[1])) - if scale[2] == 'log': - do_log.append(True) + if scale[2] == "log": + do_log.append(True) else: - do_log.append(False) + do_log.append(False) x = scale_samples(x, limits, do_log=do_log) np.save(args.outfile, x) def setup_argparse(): - parser = argparse.ArgumentParser('Generate some samples!') - parser.add_argument('-n', help='number of samples', default=100, type=int) - parser.add_argument('-dims', help='number of dimensions', default=2, type=int) - parser.add_argument('-sample_type', - help='type of sampling. options: random, grid. If grid, will try to get close to the correct number of samples', default='random') - parser.add_argument( '-scale', - help='ranges to scale results in form "[(min,max,type),(min, max,type)]" where type = "linear" or "log"') - parser.add_argument('-outfile', help='name of output .npy file', default='samples') + parser = argparse.ArgumentParser("Generate some samples!") + parser.add_argument("-n", help="number of samples", default=100, type=int) + parser.add_argument("-dims", help="number of dimensions", default=2, type=int) + parser.add_argument( + "-sample_type", + help="type of sampling. options: random, grid. If grid, will try to get close to the correct number of samples", + default="random", + ) + parser.add_argument( + "-scale", + help='ranges to scale results in form "[(min,max,type),(min, max,type)]" where type = "linear" or "log"', + ) + parser.add_argument("-outfile", help="name of output .npy file", default="samples") return parser def main(): parser = setup_argparse() - args = parser.parse_args() + args = parser.parse_args() process_args(args) if __name__ == "__main__": sys.exit(main()) - diff --git a/workflows/slurm/scripts/test_workers.sbatch b/merlin/examples/workflows/slurm/scripts/test_workers.sbatch similarity index 100% rename from workflows/slurm/scripts/test_workers.sbatch rename to merlin/examples/workflows/slurm/scripts/test_workers.sbatch diff --git a/workflows/slurm/scripts/workers.sbatch b/merlin/examples/workflows/slurm/scripts/workers.sbatch similarity index 100% rename from workflows/slurm/scripts/workers.sbatch rename to merlin/examples/workflows/slurm/scripts/workers.sbatch diff --git a/workflows/slurm/slurm_par.yaml b/merlin/examples/workflows/slurm/slurm_par.yaml similarity index 91% rename from workflows/slurm/slurm_par.yaml rename to merlin/examples/workflows/slurm/slurm_par.yaml index 893e1c7a3..4dde22051 100644 --- a/workflows/slurm/slurm_par.yaml +++ b/merlin/examples/workflows/slurm/slurm_par.yaml @@ -1,6 +1,6 @@ description: - description: Run a scan through Merlin/Maestro - name: SLURMPARMERLIN + description: A simple ensemble of parallel MPI jobs run by slurm (srun). + name: slurm_par batch: type: slurm diff --git a/workflows/slurm/slurm_test.yaml b/merlin/examples/workflows/slurm/slurm_test.yaml similarity index 90% rename from workflows/slurm/slurm_test.yaml rename to merlin/examples/workflows/slurm/slurm_test.yaml index dcb5bb598..0013855e6 100644 --- a/workflows/slurm/slurm_test.yaml +++ b/merlin/examples/workflows/slurm/slurm_test.yaml @@ -1,6 +1,6 @@ description: - description: Run a scan through Merlin/Maestro - name: SLURMTESTMERLIN + description: A simple ensemble of serial jobs run by slurm (srun). + name: slurm_test batch: type: slurm diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 9c8fb08ae..31aac4725 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 06d098274..1447513df 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index f3f246cc9..0c5a0379b 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -39,13 +39,24 @@ ArgumentDefaultsHelpFormatter, ArgumentParser, RawDescriptionHelpFormatter, + RawTextHelpFormatter, ) from contextlib import suppress -from merlin import VERSION, router +from merlin import ( + VERSION, + router, +) from merlin.ascii_art import banner_small +from merlin.examples.generator import ( + list_examples, + setup_example, +) from merlin.log_formatter import setup_logging -from merlin.spec.expansion import RESERVED, get_spec_with_expansion +from merlin.spec.expansion import ( + RESERVED, + get_spec_with_expansion, +) from merlin.study.study import MerlinStudy from merlin.utils import ARRAY_FILE_FORMATS @@ -209,6 +220,23 @@ def purge_tasks(args): LOG.info(f"Purge return = {ret} .") +def query_status(args): + """ + CLI command for querying queue status. + + :param 'args': parsed CLI arguments + """ + print(banner_small) + filepath = verify_filepath(args.specification) + variables_dict = parse_override_vars(args.variables) + spec = get_spec_with_expansion(filepath, override_vars=variables_dict) + ret = router.query_status(args.task_server, spec, args.steps) + for name, jobs, consumers in ret: + print(f"{name:30} - Workers: {consumers:10} - Queued Tasks: {jobs:10}") + if args.csv is not None: + router.dump_status(ret, args.csv) + + def query_workers(args): """ CLI command for finding all workers. @@ -253,6 +281,11 @@ def config_merlin(args): _ = router.create_config(args.task_server, output_dir) +def process_example(args): + print(banner_small) + setup_example(args.workflow, args.path) + + def setup_argparse(): """ Setup argparse and any CLI options we want available via the package. @@ -462,6 +495,45 @@ def setup_argparse(): Default: %(default)s", ) + # merlin status + status = subparsers.add_parser( + "status", + help="List server stats (name, number of tasks to do, \ + number of connected workers) for a workflow spec.", + ) + status.set_defaults(func=query_status) + status.add_argument( + "specification", type=str, help="Path to a Merlin YAML spec file" + ) + status.add_argument( + "--steps", + nargs="+", + type=str, + dest="steps", + default=["all"], + help="The specific steps in the YAML file you want to query", + ) + status.add_argument( + "--task_server", + type=str, + default="celery", + help="Task server type from which to stop workers.\ + Default: %(default)s", + ) + status.add_argument( + "--vars", + action="store", + dest="variables", + type=str, + nargs="+", + default=None, + help="Specify desired Merlin variable values to override those found in the specification. Space-delimited. " + "Example: '--vars LEARN=path/to/new_learn.py EPOCHS=3'", + ) + status.add_argument( + "--csv", type=str, help="csv file to dump status report to", default=None, + ) + # merlin info info = subparsers.add_parser( "info", help="show pip and python versions and locations" @@ -490,6 +562,29 @@ def setup_argparse(): Default: ~/.merlin", ) + # merlin example + example = subparsers.add_parser( + "example", + help="Generate an example merlin workflow.", + formatter_class=RawTextHelpFormatter, + ) + example.add_argument( + "workflow", + action="store", + type=str, + help="The name of the example workflow to setup.\n" + list_examples(), + ) + example.add_argument( + "-p", + "--path", + action="store", + type=str, + default=None, + help="Specify a path to write the workflow to. Defaults to current " + "working directory", + ) + example.set_defaults(func=process_example) + return parser diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 4e29f42fa..9034fb5f3 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -29,90 +29,45 @@ ############################################################################### """ -This module handles the CLI for the merlin-templates. +This module handles the CLI for the deprecated `merlin-templates` command. """ import argparse import logging import os import sys -from merlin import router from merlin.ascii_art import banner_small +from merlin.examples.generator import ( + list_examples, + setup_example, +) from merlin.log_formatter import setup_logging -from merlin.templates.generator import list_templates LOG = logging.getLogger("merlin-templates") -DEFAULT_LOG_LEVEL = "INFO" +DEFAULT_LOG_LEVEL = "ERROR" -def setup_template(args): - - print(banner_small) - - outdir = os.getcwd() - - if args.path: - output = args.path - - router.templates(args.template, args.path) - - -def template_list(args): - print(banner_small) - list_templates() +def process_templates(args): + LOG.error( + "The command `merlin-templates` has been deprecated in favor of `merlin example`." + ) def setup_argparse(): parser = argparse.ArgumentParser( - prog="Merlin Templates", + prog="Merlin Examples", description=banner_small, formatter_class=argparse.RawDescriptionHelpFormatter, - epilog="See merlin-templates --help for more info.", - ) - subparsers = parser.add_subparsers(dest="subparsers") - subparsers.required = True - - parser.add_argument( - "-lvl", - "--level", - action="store", - dest="level", - type=str, - default=DEFAULT_LOG_LEVEL, - help="Set the log level. Options: DEBUG, INFO, WARNING, ERROR. " - "[Default: %(default)s]", - ) - - # Naming variable subparser _list to avoid conflict with Python's list - # reserved word. - _list = subparsers.add_parser("list", help="List available templates.") - _list.set_defaults(func=template_list) - - setup = subparsers.add_parser("setup", help="Setup a new template.") - setup.add_argument( - "template", action="store", type=str, help="The name of the template to setup." ) - setup.add_argument( - "-p", - "--path", - action="store", - type=str, - default=os.getcwd(), - help="Specify a path to write the workflow to. Defaults to current " - "working directory", - ) - setup.set_defaults(func=setup_template) - + parser.set_defaults(func=process_templates) return parser def main(): parser = setup_argparse() args = parser.parse_args() - - setup_logging(logger=LOG, log_level=args.level.upper(), colors=True) - + setup_logging(logger=LOG, log_level=DEFAULT_LOG_LEVEL, colors=True) args.func(args) diff --git a/merlin/router.py b/merlin/router.py index b2a39ae73..e0b8d65c5 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -38,16 +38,17 @@ import logging import os from contextlib import suppress +from datetime import datetime from merlin.study.celeryadapter import ( create_celery_config, purge_celery_tasks, + query_celery_queues, query_celery_workers, run_celery, start_celery_workers, stop_celery_workers, ) -from merlin.templates.generator import setup_template try: @@ -109,6 +110,47 @@ def purge_tasks(task_server, spec, force, steps): LOG.error("Celery is not specified as the task server!") +def query_status(task_server, spec, steps): + """ + Queries status of queues in spec file from server. + + :param `task_server`: The task server from which to purge tasks. + :param `spec`: A MerlinSpec object + :param `steps`: Spaced-separated list of stepnames to query. Default is all + """ + LOG.info(f"Querying queues for steps = {steps}") + + if task_server == "celery": + queues = spec.get_queue_list(steps) + # Query the queues + return query_celery_queues(queues) + else: + LOG.error("Celery is not specified as the task server!") + + +def dump_status(query_return, csv_file): + """ + Dump the results of a query_status to a csv file. + + :param `query_return`: The output of query_status + :param `csv_file`: The csv file to append + """ + if os.path.exists(csv_file): + fmode = "a" + else: + fmode = "w" + with open(csv_file, mode=fmode) as f: + if f.mode == "w": # add the header + f.write("# time") + for name, job, consumer in query_return: + f.write(f",{name}:tasks,{name}:consumers") + f.write("\n") + f.write(f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + for name, job, consumer in query_return: + f.write(f",{job},{consumer}") + f.write("\n") + + def query_workers(task_server): """ Gets info from workers. @@ -168,16 +210,3 @@ def create_config(task_server, config_dir): create_celery_config(config_dir, config_file, data_file) else: LOG.error("Only celery can be configured currently.") - - -def templates(template_name, outdir): - """ - Setup a Merlin template spec. - - :param template_name: Then name of the template to copy into the workspace. - :param outdir: The directory to copy the template to. - """ - with suppress(FileExistsError): - os.makedirs(outdir) - - template = setup_template(template_name, outdir) diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index abd4eeaa9..e26ce3a51 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index a6830da98..b81862672 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index ecdbada7c..62e82c509 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -30,10 +30,16 @@ import logging from collections import ChainMap -from os.path import expanduser, expandvars +from os.path import ( + expanduser, + expandvars, +) from merlin.common.abstracts.enums import ReturnCode -from merlin.spec.override import dump_with_overrides, error_override_vars +from merlin.spec.override import ( + dump_with_overrides, + error_override_vars, +) from merlin.spec.specification import MerlinSpec diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index eb4a5a64b..aed55a012 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -31,7 +31,7 @@ """ This module contains a class, MerlinSpec, which holds the unchanged data from the Merlin specification file. -To see an example of a yaml specification, run `merlin-templates`. +To see examples of yaml specifications, run `merlin example`. """ import logging import os @@ -101,7 +101,7 @@ def load_merlin_block(stream): merlin_block = {} LOG.warning( f"Workflow specification missing \n " - f"encouraged 'merlin' section! Run 'merlin-templates' for an example.\n" + f"encouraged 'merlin' section! Run 'merlin example' for examples.\n" f"Using default configuration with no sampling." ) return merlin_block @@ -176,11 +176,11 @@ def get_task_queues(self): queues[step.name] = step.run["task_queue"] return queues - def make_queue_string(self, steps): + def get_queue_list(self, steps): """ - Return a unique queue string for the steps + Return a sorted list of queues corresponding to spec steps - param steps: a list of step names + param steps: a list of step names or 'all' """ queues = self.get_task_queues() if steps[0] == "all": @@ -197,4 +197,12 @@ def make_queue_string(self, steps): f"Invalid steps '{steps}'! Try one of these (or 'all'):\n{nl.join(queues.keys())}" ) raise - return ",".join(set(task_queues)) + return sorted(set(task_queues)) + + def make_queue_string(self, steps): + """ + Return a unique queue string for the steps + + param steps: a list of step names + """ + return ",".join(set(self.get_queue_list(steps))) diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index abd4eeaa9..e26ce3a51 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 7678e8f6c..825622d1a 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 7aee233f1..66a8d8c28 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -37,8 +37,16 @@ import time from contextlib import suppress -from merlin.study.batch import batch_check_parallel, batch_worker_launch -from merlin.utils import get_procs, get_yaml_var, is_running, regex_list_filter +from merlin.study.batch import ( + batch_check_parallel, + batch_worker_launch, +) +from merlin.utils import ( + get_procs, + get_yaml_var, + is_running, + regex_list_filter, +) LOG = logging.getLogger(__name__) @@ -141,6 +149,29 @@ def query_celery_workers(): LOG.warning("No workers found!") +def query_celery_queues(queues): + """Return stats for queues specified. + + Send results to the log. + """ + from merlin.celery import app + + connection = app.connection() + found_queues = [] + try: + channel = connection.channel() + for queue in queues: + try: + name, jobs, consumers = channel.queue_declare(queue=queue, passive=True) + found_queues.append((name, jobs, consumers)) + LOG.info(f"Found queue {queue}.") + except: + LOG.warning(f"Cannot find queue {queue} on server.") + finally: + connection.close() + return found_queues + + def get_workers(app): """Get all workers connected to a celery application. diff --git a/merlin/study/dag.py b/merlin/study/dag.py index d42be9772..4814927c6 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 0f87f64ba..5a0735f30 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index c1c7842a5..b35f0ae47 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 968151c33..6aad73364 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -41,8 +41,14 @@ from merlin.common.abstracts.enums import ReturnCode from merlin.spec import defaults -from merlin.spec.expansion import determine_user_variables, expand_line -from merlin.spec.override import dump_with_overrides, error_override_vars +from merlin.spec.expansion import ( + determine_user_variables, + expand_line, +) +from merlin.spec.override import ( + dump_with_overrides, + error_override_vars, +) from merlin.spec.specification import MerlinSpec from merlin.study.dag import DAG from merlin.utils import load_array_file @@ -80,7 +86,7 @@ def __init__( self.label_clash_error() self.dry_run = dry_run - # If we load from a file, record that in the object for provenence + # If we load from a file, record that in the object for provenance # downstream if self.samples_file is not None: self.spec.merlin["samples"]["file"] = self.samples_file @@ -146,7 +152,7 @@ def write_expanded_spec(self, dest): :param `dest`: destination for fully expanded yaml file """ - # specification text including defaults and overriden user variables + # specification text including defaults and overridden user variables full_spec = dump_with_overrides(self.spec, self.override_vars) with open(dest, "w") as dumped_file: diff --git a/merlin/templates/generator.py b/merlin/templates/generator.py deleted file mode 100644 index 3dd8903da..000000000 --- a/merlin/templates/generator.py +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################### -# Copyright (c) 2019, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory -# Written by the Merlin dev team, listed in the CONTRIBUTORS file. -# -# -# LLNL-CODE-797170 -# All rights reserved. -# This file is part of Merlin, Version: 1.0.5. -# -# For details, see https://github.com/LLNL/merlin. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -############################################################################### - -""" -This module contains a list of example templates that can be used for setting -up new workflows. -""" -import logging -import os - -import tabulate - -from merlin.templates import examples - - -LOG = logging.getLogger("merlin-templates") - - -def find_template(name): - for template in examples.TEMPLATES: - if template["name"] == name: - return template # Return only the first template for now. - - -def write_template(filepath, content): - """ - Write out the template workflow to a file. - - :param filepath: The path to write the template file to. - :param content: The formatted content to write the file to. - """ - with open(filepath, "w") as _file: - _file.write(content) - - -def list_templates(): - """List all available templates.""" - templates = examples.TEMPLATES - - print("") - headers = ["name", "description"] - rows = [] - for template in templates: - rows.append([template["name"], template["description"]]) - print(tabulate.tabulate(rows, headers)) - print("") - - -def setup_template(name, outdir=None): - """Setup the given template.""" - template = find_template(name) - - if template is None: - LOG.error(f"Template '{name}' not found.") - return None - - if outdir: - filepath = os.path.join(outdir, template["filename"]) - else: - filepath = template["filename"] - - if os.path.isfile(filepath): - LOG.error(f"Filename '{filepath}' already exists!") - return None - - LOG.info(f"Copying template '{name}' to {outdir}") - write_template(filepath, template["content"]) - return template diff --git a/merlin/utils.py b/merlin/utils.py index 56299a94e..8bed55b58 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.0.5. +# This file is part of Merlin, Version: 1.1.0. # # For details, see https://github.com/LLNL/merlin. # @@ -36,7 +36,10 @@ import os import re import subprocess -from contextlib import contextmanager, suppress +from contextlib import ( + contextmanager, + suppress, +) from copy import deepcopy from types import SimpleNamespace @@ -239,7 +242,7 @@ def load_array_file(filename, ndmin=2): f"Array in {filename} has fewer than the required \ minimum dimensions ({array.ndim} < {ndmin})!" ) - # Make sure text files load as strings with mininum number of dimensions + # Make sure text files load as strings with minimum number of dimensions elif protocol == "csv": array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str) elif protocol == "tab": diff --git a/requirements.txt b/requirements.txt index d300b663b..7b608af11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1 @@ -# Development dependencies. -black -dep-license -flake8 -isort -pytest -pytest-runner -sphinx -sphinx_rtd_theme - -r requirements/release.txt diff --git a/requirements/dev.txt b/requirements/dev.txt index bd3b12e13..d570d3af8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,5 +6,3 @@ isort pytest sphinx sphinx_rtd_theme - --r requirements/release.txt diff --git a/setup.py b/setup.py index a033f0212..c0b90dbd9 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ version = __import__("merlin").VERSION -extras = ["mysql"] +extras = ["mysql","dev"] def readme(): diff --git a/tests/common/test_sample_index.py b/tests/common/test_sample_index.py index ba18db1e8..e5d34f32b 100644 --- a/tests/common/test_sample_index.py +++ b/tests/common/test_sample_index.py @@ -3,7 +3,10 @@ import unittest from contextlib import suppress -from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy +from merlin.common.sample_index_factory import ( + create_hierarchy, + read_hierarchy, +) TEST_DIR = "UNIT_TEST_SPACE" diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 795feb8f1..1d0e31d8c 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -40,7 +40,10 @@ from contextlib import suppress from glob import glob from re import search -from subprocess import PIPE, Popen +from subprocess import ( + PIPE, + Popen, +) OUTPUT_DIR = "cli_test_studies" @@ -121,7 +124,6 @@ def process_test_result(passed, info, is_verbose, exit): def run_tests(args, tests): """ Run all inputted tests. - :param `tests`: a dictionary of {"test_name" : ("test_command", [conditions])} """ @@ -337,8 +339,11 @@ def define_tests(): run = "merlin run" restart = "merlin restart" purge = "merlin purge" - demo = "workflows/feature_demo/feature_demo.yaml" - simple = "workflows/simple_chain/simple_chain.yaml" + examples = "merlin/examples/workflows" + demo = f"{examples}/feature_demo/feature_demo.yaml" + simple = f"{examples}/simple_chain/simple_chain.yaml" + slurm = f"{examples}/slurm/slurm_test.yaml" + flux = f"{examples}/flux/flux_test.yaml" black = "black --check --target-version py36" config_dir = "./CLI_TEST_MERLIN_CONFIG" @@ -359,11 +364,11 @@ def define_tests(): [ReturnCodeCond(), RegexCond(celery_regex)], ), "run-workers echo slurm_test": ( - f"{workers} workflows/slurm/slurm_test.yaml --echo", + f"{workers} {slurm} --echo", [ReturnCodeCond(), RegexCond(celery_regex)], ), "run-workers echo flux_test": ( - f"{workers} workflows/flux/flux_test.yaml --echo", + f"{workers} {flux} --echo", [ReturnCodeCond(), RegexCond(celery_regex)], ), "run-workers echo override feature_demo": ( @@ -383,6 +388,11 @@ def define_tests(): f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", ReturnCodeCond(), ), + "example failure": (f"merlin example failure", RegexCond("not found"),), + "example simple_chain": ( + f"merlin example simple_chain ; {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; rm simple_chain.yaml", + ReturnCodeCond(), + ), # "restart local simple_chain": ( # f"{restart} --local $(find studies/ -type d -name 'simple_chain_*')", # [ReturnCodeCond(), NoStderrCond()], diff --git a/workflows/flux/scripts/flux_info.py b/workflows/flux/scripts/flux_info.py deleted file mode 100755 index 132a4492a..000000000 --- a/workflows/flux/scripts/flux_info.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python2 -"""This module will collect information on flux jobs from the live kvs -store and output times for each phase. - -create: The time flux registerd the job -starting: The time the job was created -running: The time the job was running -completing: The time the job started its' completion pahse. -complete: The time the job was complete -walltime: ? Seems to be 0. -""" -import flux -import os -import sys -from flux import kvs, kz - -f = flux.Flux() - -fs='FLUX_START_SECONDS' -if fs in os.environ: - print("flux start: {0}".format(os.environ[fs])) - -for d in kvs.walk('lwj', flux_handle=f): - try: - #print(type(d)) - fdir = "lwj.{0}".format(d[0]) - - qcreate="{0}.create-time".format(fdir) - create_time=kvs.get(f, qcreate) - - qstart="{0}.starting-time".format(fdir) - start_time=kvs.get(f, qstart) - - qrun="{0}.running-time".format(fdir) - start_time=kvs.get(f, qrun) - - qcomplete="{0}.complete-time".format(fdir) - complete_time=kvs.get(f, qcomplete) - - qcompleting="{0}.completing-time".format(fdir) - completing_time=kvs.get(f, qcompleting) - - qwall="{0}.walltime".format(fdir) - wall_time=kvs.get(f, qwall) - - proto = "Job {0}: create: {1} start {1} run {2} completing {3} complete {4} wall {5}" - print(proto.format(d[0],create_time,start_time,completing_time,complete_time,wall_time)) - except: - pass - - -# vi: ts=4 sw=4 expandtab