Merge changes from main into develop

.github/workflows/pypi_publish.yml

@@ -0,0 +1,49 @@
+name: Publish package python distribution to Pypi
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+    build:
+      name: Build distribution
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12.7
+      - name: Install pypi/build
+        run: >-
+          python3 -m
+          pip install
+          build
+          --user
+      - name: Build a binary wheel and a source tarball
+        run: python3 -m build
+      - name: Store the distribution packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+    publish-to-pypi:
+      name: Publish dist to PyPI
+      needs:
+      - build
+      runs-on: ubuntu-latest
+      environment:
+        name: pypi
+        url: https://pypi.org/p/buisciii-tools
+      permissions:
+        id-token: write
+      steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/python_lint.yml

@@ -2,11 +2,10 @@ name: python_lint
 
 on:
   push:
-    paths:
-      - '**/*.py'
+    branches: "**"
   pull_request:
-    paths:
-      - '**/*.py'
+    types: [opened, reopened, synchronize, closed]
+    branches: "**"
 
 jobs:
   flake8_py3:
@@ -19,17 +18,42 @@ jobs:
           architecture: x64
       - name: Checkout PyTorch
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Install flake8
         run: pip install flake8
+      - name: Check for Python file changes
+        id: file_check
+        uses: tj-actions/changed-files@v44
+        with:
+          sha: ${{ github.event.pull_request.head.sha }}
+          files: |
+             **.py
       - name: Run flake8
+        if: steps.file_check.outputs.any_changed == 'true'
         run: flake8 --ignore E501,W503,E203,W605
+      - name: No Python files changed
+        if: steps.file_check.outputs.any_changed != 'true'
+        run: echo "No Python files have been changed."
 
   black_lint:
     runs-on: ubuntu-latest
     steps:
         - name: Setup
           uses: actions/checkout@v2
+          with:
+            fetch-depth: 0
         - name: Install black in jupyter
           run: pip install black[jupyter]
+        - name: Check for Python file changes
+          id: file_check
+          uses: tj-actions/changed-files@v44
+          with:
+            sha: ${{ github.event.pull_request.head.sha }}
+            files: '**.py'
         - name: Check code lints with Black
+          if: steps.file_check.outputs.any_changed == 'true'
           uses: psf/black@stable
+        - name: No Python files changed
+          if: steps.file_check.outputs.any_changed != 'true'
+          run: echo "No Python files have been changed."

CHANGELOG.md

@@ -4,7 +4,163 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [2.2.Xdev] - 2024-0X-XX : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.2.X
+## [2.2.4] - 2024-12-27 : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.2.4
+
+### Credits
+
+- [Victor Lopez](https://github.com/victor5lm)
+
+### Template fixes and updates
+
+- Replaced /data/bi/ by /data/ucct/bi where necessary [#385](https://github.com/BU-ISCIII/buisciii-tools/pull/385).
+- Removed middle_obx from config files [#385](https://github.com/BU-ISCIII/buisciii-tools/pull/385).
+
+### Modules
+
+#### Added enhancements
+
+#### Fixes
+
+#### Changed
+
+#### Removed
+
+### Requirements
+
+## [2.2.3] - 2024-12-23 : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.2.3
+
+### Credits
+
+Code contributions to the new version:
+
+- [Victor Lopez](https://github.com/victor5lm)
+- [Sarai Varona](https://github.com/svarona)
+
+### Template fixes and updates
+
+- Updated sftp_user.json, added the locus-tag option for the PROKKA process in the bacass config file and changed new_service.py so that integrity is checked only for the samples of interest [#363](https://github.com/BU-ISCIII/buisciii-tools/pull/363).
+- Replaced /data/bi/ by /data/ucct/bi/ [#380](https://github.com/BU-ISCIII/buisciii-tools/pull/380).
+- Updated bacass version in all pertinent files [#380](https://github.com/BU-ISCIII/buisciii-tools/pull/380).
+- Updated read length variable definition when creating the mapping_illumina.tab file [#380](https://github.com/BU-ISCIII/buisciii-tools/pull/380).
+- Updated create_irma_stats.sh to include %mapped_reads [#380](https://github.com/BU-ISCIII/buisciii-tools/pull/380).
+- Changed "Buenas" by "Estimado/a" in email.j2 [#380](https://github.com/BU-ISCIII/buisciii-tools/pull/380).
+
+### Modules
+
+#### Added enhancements
+
+#### Fixes
+
+- Fixed new-service to correctly handle when there are no samples in service [#372](https://github.com/BU-ISCIII/buisciii-tools/pull/372). Fixes issue [#371](https://github.com/BU-ISCIII/buisciii-tools/issues/371)
+
+#### Changed
+
+#### Removed
+
+### Requirements
+
+## [2.X.Xhot] - 2024-0X-0X : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.X.2
+
+### Credits
+
+### Template fixes and updates
+
+### Modules
+
+#### Added enhancements
+
+#### Fixes
+
+#### Changed
+
+#### Removed
+
+### Requirements
+
+## [2.2.2] - 2024-10-28 : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.2.2
+
+### Credits
+
+Code contributions to the new version:
+
+- [Pablo Mata](https://github.com/Shettland)
+- [Victor Lopez](https://github.com/victor5lm)
+
+### Template fixes and updates
+
+- Updated the fix-permissions module in __main__.py [#356](https://github.com/BU-ISCIII/buisciii-tools/pull/356).
+- Fixed the singularity cache directory in taxprofiler.config [#356](https://github.com/BU-ISCIII/buisciii-tools/pull/356).
+- Updated sftp_user.json [#356](https://github.com/BU-ISCIII/buisciii-tools/pull/356).
+- Fixed viralrecon's lablog and the remove_columns_mapping_table.sh auxiliary script [#356](https://github.com/BU-ISCIII/buisciii-tools/pull/356).
+- Updated the singularity image in the mtbseq templates [#356](https://github.com/BU-ISCIII/buisciii-tools/pull/356).
+- Fixed a bug in bioinfo_doc.py [#356](https://github.com/BU-ISCIII/buisciii-tools/pull/356).
+- Updated new_service.py to check FASTQ integrity via md5sum [#356](https://github.com/BU-ISCIII/buisciii-tools/pull/356).
+
+### Modules
+
+#### Added enhancements
+
+- Included a new github action to automatically publish releases to pypi [#351](https://github.com/BU-ISCIII/buisciii-tools/pull/351)
+
+#### Fixes
+
+#### Changed
+
+#### Removed
+
+### Requirements
+
+## [2.X.Xhot] - 2024-0X-0X : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.X.1
+
+### Credits
+
+Code contributions to the hotfix:
+
+### Template fixes and updates
+
+### Modules
+
+#### Added enhancements
+
+#### Fixes
+
+#### Changed
+
+#### Removed
+
+### Requirements
+
+## [2.2.1] - 2024-10-01 : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.2.1
+
+### Credits
+
+Code contributions to the new version:
+
+- [Daniel Valle](https://github.com/Daniel-VM)
+- [Sarai Varona](https://github.com/svarona)
+- [Victor Lopez](https://github.com/victor5lm)
+- [Sergio Olmos](https://github.com/OPSergio)
+
+### Template fixes and updates
+
+- Fixed path to blast database and update Emmtyper params [#339](https://github.com/BU-ISCIII/buisciii-tools/pull/339)
+- Updated sarek version (v3.4.4) in ExomeEB-ExomeTrio-WGSTrio templates [#341] (https://github.com/BU-ISCIII/buisciii-tools/pull/341)
+- Fixed IRMAs config for amended consensus [#325](https://github.com/BU-ISCIII/buisciii-tools/pull/325).
+- Improved excel_generator.py and bioinfo_doc.py email creation function, and updated sftp_user.json, setup.py, main.py and some lablogs [#344](https://github.com/BU-ISCIII/buisciii-tools/pull/344).
+
+### Modules
+
+#### Added enhancements
+
+#### Fixes
+
+#### Changed
+
+#### Removed
+
+### Requirements
+
+## [2.2.0] - 2024-09-12 : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.2.0
 
 ### Credits
 
@@ -15,6 +171,7 @@ Code contributions to the new version:
 - [Sarai Varona](https://github.com/svarona)
 - [Daniel Valle](https://github.com/Daniel-VM)
 - [Víctor López](https://github.com/victor5lm)
+- [Juan Ledesma](https://github.com/juanledesma78)
 
 ### Template fixes and updates
 
@@ -43,38 +200,54 @@ Code contributions to the new version:
 - Included annotated tab description in exome-trios markdowns [#273](https://github.com/BU-ISCIII/buisciii-tools/pull/273)
 - Installed all necessary singularity images and modified all templates so that, instead of using conda environments or loaded modules, the corresponding singularity images are used [#272](https://github.com/BU-ISCIII/buisciii-tools/pull/272)
 - Updated sarek version in exomeeb, exometrio and wgstrio templates [#277](https://github.com/BU-ISCIII/buisciii-tools/pull/277)
+- Extension file of all_samples_virus_table_filtered (from csv to tsv) in lablog_viralrecon_results changed [#278](https://github.com/BU-ISCIII/buisciii-tools/pull/278)
+- Fixed singularity-images path when updating pangolin database in lablog_viralrecon. Added line break after prompted input. [#282](https://github.com/BU-ISCIII/buisciii-tools/pull/282)
+- Updated characterization and snippy templates to fit bacass pipeline. Corrected path in 05-iqtree in snippy template. [#283](https://github.com/BU-ISCIII/buisciii-tools/pull/283)
+- Included multiqc_report.html in RESULTS folder in every service, where necessary [#265] (https://github.com/BU-ISCIII/buisciii-tools/pull/265)
+- Added MAG tempalte and removed MAG from other templates [#288](https://github.com/BU-ISCIII/buisciii-tools/pull/288)
+- Added amrfinderplus to characterization template. [#289] (https://github.com/BU-ISCIII/buisciii-tools/pull/289)
+- Updated all files so that paths referring to /pipelines/ are updated according to the new structure [#287](https://github.com/BU-ISCIII/buisciii-tools/pull/287)
+- Updated assembly, ariba, snippy, amrfinderplus and iqtree templates, removed genomeev and mtbseq_assembly templates and updated services.json [#295](https://github.com/BU-ISCIII/buisciii-tools/pull/295)
+- Changed viralrecon's lablog so that references are available within refgenie [#296](https://github.com/BU-ISCIII/buisciii-tools/pull/296)
+- Updated services.json, mtbseq's lablog, viralrecon's lablog and assembly's config file [#299](https://github.com/BU-ISCIII/buisciii-tools/pull/299)
+- Added lablog to automate gene characterization with emmtyper, including unzipping assemblies. [#300](https://github.com/BU-ISCIII/buisciii-tools/pull/300)
+- Fixed 99-stats (MAG) template. [#301](https://github.com/BU-ISCIII/buisciii-tools/pull/301)
+- Created a python script to process IRMA's results and create a standard vcf file against reference. [#304](https://github.com/BU-ISCIII/buisciii-tools/pull/304)
+- Fixed IRMA's lablog so that the sequences of the samples are not displayed several times neither in the .txt files of each influenza type nor in all_samples_completo.txt [#305](https://github.com/BU-ISCIII/buisciii-tools/pull/305)
+- Modified bioinfo_doc.py so that new lines in the delivery message are applied in the email [#307](https://github.com/BU-ISCIII/buisciii-tools/pull/307)
+- Added several improvements in lablog_viralrecon (created log files, modified check_references function behaviour, enabled config files regeneration) [#306](https://github.com/BU-ISCIII/buisciii-tools/pull/306)
+- Fixed bug when lablog_viralrecon tries to download references that don't belong to any family. [#310](https://github.com/BU-ISCIII/buisciii-tools/pull/310)
+- Added mvmoneo to SFTP users. [#317](https://github.com/BU-ISCIII/buisciii-tools/pull/317)
+- Added scripts for time series RNAseq and updated differential expression code for differentially expressed transcripts [#316](https://github.com/BU-ISCIII/buisciii-tools/pull/316).
+- Added bbaladron to SFTP users [#316](https://github.com/BU-ISCIII/buisciii-tools/pull/316).
+- Added new template for comprehensive taxonomy profiling using the nf-core/taxprofiler pipeline [#320](https://github.com/BU-ISCIII/buisciii-tools/pull/320).
+- Added full execution support for the MAG template [#321](https://github.com/BU-ISCIII/buisciii-tools/pull/321).
+- Added labels to services.json and updated bioinfo_doc.py and jinja_template_delivery.j2 so that software versions data is displayed in the delivery pdf [#330](https://github.com/BU-ISCIII/buisciii-tools/pull/330).
+- Updated several templates (singularity images, outdated paths, improvements, etc) [#331](https://github.com/BU-ISCIII/buisciii-tools/pull/331)
+- Added permissions fixing after running scratch_copy, as well as a new fix-permissions module in the tools [#332](https://github.com/BU-ISCIII/buisciii-tools/pull/332).
+- Updated MAG lablogs and utils.py [#334](https://github.com/BU-ISCIII/buisciii-tools/pull/334).
+- Updated some files (setup.py, __main__.py, README, etc) for the 2.2.0 release [#335](https://github.com/BU-ISCIII/buisciii-tools/pull/335).
 
 ### Modules
 
 #### Added enhancements
 
 - PR [#274](https://github.com/BU-ISCIII/buisciii-tools/pull/274): added `--dev` option, configuration dev and test folder structure.
+- PR [#276](https://github.com/BU-ISCIII/buisciii-tools/pull/276): wkhtmlpdf does not need absolute path to executable. Added better error handling when executable does not exists.
+- PR [#288](https://github.com/BU-ISCIII/buisciii-tools/pull/288) Allowed to handle more than one service at a time, related to issue [#217](https://github.com/BU-ISCIII/buisciii-tools/issues/217)
 
 #### Fixes
 
 - Fixed archive module. Updated correct header for scout tsv [#258](https://github.com/BU-ISCIII/buisciii-tools/pull/258).
+- Fixed clean module. Corrected purge_files function. Renaming stage moved from clean to rename_nocopy option. Updated services.json file with correct paths for some services. [#280](https://github.com/BU-ISCIII/buisciii-tools/pull/280)
+- Fixed autoclean-sftp function. [#281](https://github.com/BU-ISCIII/buisciii-tools/pull/281)
+- Fixed bioinfo_doc.py. Modified it so that this module creates a .pdf file including new-line characters, without merging lines into one single line [#259](https://github.com/BU-ISCIII/buisciii-tools/pull/259).
+- PR [#288](https://github.com/BU-ISCIII/buisciii-tools/pull/288) Fixed updating service's state to in_progress multiple times, related with issue [#285](https://github.com/BU-ISCIII/buisciii-tools/issues/285)
+- Review and update of services.json for files and folders cleaning [#318](https://github.com/BU-ISCIII/buisciii-tools/pull/318).
 
 #### Changed
 
-#### Removed
-
-### Requirements
-
-## [2.X.1hot] - 2024-0X-0X : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.X.1
-
-### Credits
-
-Code contributions to the hotfix:
-
-### Template fixes and updates
-
-### Modules
-
-#### Added enhancements
-
-#### Fixes
-
-#### Changed
+- Forcing python lint to success if no .py files are in PR [#279](https://github.com/BU-ISCIII/buisciii-tools/pull/279)
 
 #### Removed