From fee19005472c00b4e7754e07e2ec453d5df39f43 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 18 Dec 2023 08:30:32 -0700 Subject: [PATCH 1/4] some fixes --- qiita_db/metadata_template/prep_template.py | 14 ++++++++++++-- qiita_db/processing_job.py | 14 ++++++++++++-- qiita_pet/templates/workflows.html | 4 ++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index bc097dba6..526dcf710 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -851,20 +851,30 @@ def _get_predecessors(workflow, node): starting_job = None pt_artifact = self.artifact.artifact_type - workflows = [] + all_workflows = [wk for wk in qdb.software.DefaultWorkflow.iter()] + # are there any workflows with parameters? + check_requirements = False + default_parameters = {'prep': {}, 'sample': {}} + if [wk for wk in all_workflows if wk.parameters != default_parameters]: + check_requirements = True ST = qdb.metadata_template.sample_template.SampleTemplate - for wk in qdb.software.DefaultWorkflow.iter(): + workflows = [] + for wk in all_workflows: if wk.artifact_type == pt_artifact and pt_dt in wk.data_type: + if check_requirements and wk.parameters == default_parameters: + continue wk_params = wk.parameters reqs_satisfied = True if wk_params['sample']: + check_requirements = True df = ST(self.study_id).to_dataframe(samples=list(self)) for k, v in wk_params['sample'].items(): if k not in df.columns or v not in df[k].unique(): reqs_satisfied = False if wk_params['prep']: + check_requirements = True df = self.to_dataframe() for k, v in wk_params['prep'].items(): if k not in df.columns or v not in df[k].unique(): diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 45e90d98f..8f048cb57 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -999,7 +999,9 @@ def submit(self, parent_job_id=None, dependent_jobs_list=None): qdb.sql_connection.TRN.commit() job_dir = join(qdb.util.get_work_base_dir(), self.id) - software = self.command.software + command = self.command + software = command.software + cname = command.name plugin_start_script = software.start_script plugin_env_script = software.environment_script @@ -1011,7 +1013,15 @@ def submit(self, parent_job_id=None, dependent_jobs_list=None): # case where we are going to execute some command and then wait for the # plugin to return their own id (first implemented for # fast-bowtie2+woltka) - if 'ENVIRONMENT' in plugin_env_script: + # + # This is the hardcoded lines described in issue: + # https://github.com/qiita-spots/qiita/issues/3340 + # the idea is that in the future we shouldn't check specific command + # names to know if it should be executed differently and the + # plugin should let Qiita know that a specific command should be ran + # as job array or not + cnames_to_skip = {'Calculate Cell Counts'} + if 'ENVIRONMENT' in plugin_env_script and not cname in cnames_to_skip: # the job has to be in running state so the plugin can change its` # status with qdb.sql_connection.TRN: diff --git a/qiita_pet/templates/workflows.html b/qiita_pet/templates/workflows.html index d436e181c..a5a49b112 100644 --- a/qiita_pet/templates/workflows.html +++ b/qiita_pet/templates/workflows.html @@ -73,9 +73,9 @@

Recommended Default Workflows

default Earth Microbiome Project protocol and so assumes the uploaded data are multiplexed sequences with the reversed barcodes in your mapping file and index sequence file (see here for more details). Thus, if the protocol does not apply to your data you can still use the Default Workflow, however, you should first manually process your data using the appropriate steps until you have a defined step; in our example, - demultiplexed your reads. After demultiplexing the Default Workflow is safe to use with any protocol. + demultiplex your reads. After demultiplexing, the Default Workflow is safe to use with any protocol.

- If you have already manually performed one of the processing steps in the Defaul Workflow pipeline, the "Add Default Workflow" button will not re-select those steps but + If you have already manually performed one of the processing steps in the Default Workflow pipeline, the "Add Default Workflow" button will not re-select those steps but instead will only select any remaining steps that have not been completed. You can also add additional workflows on top of the recommended Default Workflow at any time.

Note that this is not a full inclusive list of data types accepted by Qiita but only those that have a defined workflow. From d0b50fc1039f4118d29843b2383d148dd75ec0d3 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 18 Dec 2023 08:50:44 -0700 Subject: [PATCH 2/4] flake8 --- qiita_db/processing_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 8f048cb57..9c9aa1186 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1021,7 +1021,7 @@ def submit(self, parent_job_id=None, dependent_jobs_list=None): # plugin should let Qiita know that a specific command should be ran # as job array or not cnames_to_skip = {'Calculate Cell Counts'} - if 'ENVIRONMENT' in plugin_env_script and not cname in cnames_to_skip: + if 'ENVIRONMENT' in plugin_env_script and cname not in cnames_to_skip: # the job has to be in running state so the plugin can change its` # status with qdb.sql_connection.TRN: From 4615dadac1ab79bad18a2073604e2927cccb9f29 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 18 Dec 2023 11:03:28 -0700 Subject: [PATCH 3/4] 2023.12 --- CHANGELOG.md | 15 +++++++++++++++ qiita_core/__init__.py | 2 +- qiita_db/__init__.py | 2 +- qiita_pet/__init__.py | 2 +- qiita_pet/handlers/api_proxy/__init__.py | 2 +- .../processingdata/processing-recommendations.rst | 4 ++-- qiita_ware/__init__.py | 2 +- setup.py | 2 +- 8 files changed, 23 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0df0f0092..ce053b5c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Qiita changelog +Version 2023.12 +--------------- + +* The sample and preparation information pages will display the timestamp of their last update. +* Added a ProcessingJob.complete_processing_job method to retrieve the job that is completing the current job. +* Added a ProcessingJob.complete_processing_job method to retrieve the job that is completing the current job. +* Added a ProcessingJob.trace method to trace all the jobs of a processing_job. +* Analyses now accept SLURM reservation's via the GUI; this will be [helpful for workshops or classes](https://qiita.ucsd.edu/static/doc/html/faq.html#are-you-planning-a-workshop-or-class). +* Admins can now add per-user-level SLURM submission parameters via the DB; this is helpful to prioritize wet-lab and admin jobs. +* Workflow definitions can now use sample or preparation information columns/values to differentiate between them. +* Updated the Adapter and host filtering plugin (qp-fastp-minimap2) to v2023.12 addressing a bug in adapter filtering; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/qp-fastp-minimap2.html). +* Other fixes: [3334](https://github.com/qiita-spots/qiita/pull/3334), [3338](https://github.com/qiita-spots/qiita/pull/3338). Thank you @sjanssen2. +* The internal Sequence Processing Pipeline is now using the human pan-genome reference, together with the GRCh38 genome + PhiX and CHM13 genome for human host filtering. + + Version 2023.10 --------------- diff --git a/qiita_core/__init__.py b/qiita_core/__init__.py index 7d3e184ba..349b460d8 100644 --- a/qiita_core/__init__.py +++ b/qiita_core/__init__.py @@ -6,4 +6,4 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- -__version__ = "2023.10" +__version__ = "2023.12" diff --git a/qiita_db/__init__.py b/qiita_db/__init__.py index d47d522c9..c2b6f34a3 100644 --- a/qiita_db/__init__.py +++ b/qiita_db/__init__.py @@ -27,7 +27,7 @@ from . import user from . import processing_job -__version__ = "2023.10" +__version__ = "2023.12" __all__ = ["analysis", "artifact", "archive", "base", "commands", "environment_manager", "exceptions", "investigation", "logger", diff --git a/qiita_pet/__init__.py b/qiita_pet/__init__.py index 7d3e184ba..349b460d8 100644 --- a/qiita_pet/__init__.py +++ b/qiita_pet/__init__.py @@ -6,4 +6,4 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- -__version__ = "2023.10" +__version__ = "2023.12" diff --git a/qiita_pet/handlers/api_proxy/__init__.py b/qiita_pet/handlers/api_proxy/__init__.py index ffdb40112..85dace443 100644 --- a/qiita_pet/handlers/api_proxy/__init__.py +++ b/qiita_pet/handlers/api_proxy/__init__.py @@ -38,7 +38,7 @@ from .user import (user_jobs_get_req) from .util import check_access, check_fp -__version__ = "2023.10" +__version__ = "2023.12" __all__ = ['prep_template_summary_get_req', 'data_types_get_req', 'study_get_req', 'sample_template_filepaths_get_req', diff --git a/qiita_pet/support_files/doc/source/processingdata/processing-recommendations.rst b/qiita_pet/support_files/doc/source/processingdata/processing-recommendations.rst index 482d9ea2a..532ec2302 100755 --- a/qiita_pet/support_files/doc/source/processingdata/processing-recommendations.rst +++ b/qiita_pet/support_files/doc/source/processingdata/processing-recommendations.rst @@ -45,7 +45,7 @@ Shotgun sequencing ------------------ Qiita currently has one active shotgun metagenomics data analysis pipeline: a per sample -bowtie2 alignment step with Woltka classification using either the WoLr1, WoLr2 (default) or RS210 databases. +bowtie2 alignment step with Woltka classification using either the WoLr2 (default) or RS210 databases. Below you will find more information about each of these options. .. note:: @@ -87,7 +87,7 @@ Note that the command produces up to 6 output artifacts based on the aligner and - Alignment Profile: contains the raw alignment file and the no rank classification BIOM table - Per genome Predictions: contains the per genome level predictions BIOM table -- Per gene Predictions: Only WoLr1 & WoLr2, contains the per gene level predictions BIOM table +- Per gene Predictions: Only WoLr2, contains the per gene level predictions BIOM table - KEGG Pathways: Only WoLr2, contains the functional profile - KEGG Ontology (KO): Only WoLr2, contains the functional profile - KEGG Enzyme (EZ): Only WoLr2, contains the functional profile diff --git a/qiita_ware/__init__.py b/qiita_ware/__init__.py index 7d3e184ba..349b460d8 100644 --- a/qiita_ware/__init__.py +++ b/qiita_ware/__init__.py @@ -6,4 +6,4 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- -__version__ = "2023.10" +__version__ = "2023.12" diff --git a/setup.py b/setup.py index 016c93d35..7649c4fc5 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ from setuptools import setup from glob import glob -__version__ = "2023.10" +__version__ = "2023.12" classes = """ From 74d282f706f62a5875228b8153e39f43be5b481b Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 18 Dec 2023 11:39:24 -0700 Subject: [PATCH 4/4] addressing @charles-cowart comments --- CHANGELOG.md | 1 - qiita_db/metadata_template/prep_template.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce053b5c1..2d1b55336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ Version 2023.12 * The sample and preparation information pages will display the timestamp of their last update. * Added a ProcessingJob.complete_processing_job method to retrieve the job that is completing the current job. -* Added a ProcessingJob.complete_processing_job method to retrieve the job that is completing the current job. * Added a ProcessingJob.trace method to trace all the jobs of a processing_job. * Analyses now accept SLURM reservation's via the GUI; this will be [helpful for workshops or classes](https://qiita.ucsd.edu/static/doc/html/faq.html#are-you-planning-a-workshop-or-class). * Admins can now add per-user-level SLURM submission parameters via the DB; this is helpful to prioritize wet-lab and admin jobs. diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 526dcf710..1dfbff4de 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -867,14 +867,12 @@ def _get_predecessors(workflow, node): reqs_satisfied = True if wk_params['sample']: - check_requirements = True df = ST(self.study_id).to_dataframe(samples=list(self)) for k, v in wk_params['sample'].items(): if k not in df.columns or v not in df[k].unique(): reqs_satisfied = False if wk_params['prep']: - check_requirements = True df = self.to_dataframe() for k, v in wk_params['prep'].items(): if k not in df.columns or v not in df[k].unique():