Restructure installation and use src, pyproject.toml

.github/actions/build-release/action.yml

@@ -85,11 +85,9 @@ runs:
         # Add source files
         rsync --archive \
           "--include=autocompletion/***" \
-          "--include=bin/***" \
-          "--include=classes/***" \
-          "--include=subcommands/***" \
-          "--include=utils/***" \
+          "--include=src/***" \
           "--include=*-conda-env.yaml" \
+          "--include=pyproject.toml" \
           "--include=install.sh" \
           "--exclude=*" \
           "./" "release-${{ inputs.git_tag }}/"

Dockerfile

@@ -7,8 +7,9 @@ ARG CONDA_USER_NAME="cwl_ica_user"
 ARG CONDA_USER_ID=1000
 ARG CONDA_ENV_NAME="cwl-ica"
 ARG YQ_VERSION="v4.35.2"
-ARG ICAV2_PLUGINS_CLI_VERSION="v2.15.2"
-ARG ICAV2_PLUGINS_CLI_CONDA_ENV_NAME="python3.10"
+ARG ICAV2_PLUGINS_CLI_VERSION="v2.20.0"
+ARG ICAV2_PLUGINS_CLI_CONDA_PYTHON_VERSION="3.11"
+ARG ICAV2_PLUGINS_CLI_CONDA_ENV_NAME="python3.11"
 ARG CURL_VERSION="7.81.0"
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
@@ -129,10 +130,10 @@ ENV CONDA_DEFAULT_ENV="${CONDA_ENV_NAME}"
 
 RUN ( \
       cd "/home/${CONDA_USER_NAME}" && \
-      echo "Creating python3.10 environment for icav2 cli plugins" 1>&2 && \
+      echo "Creating ${ICAV2_PLUGINS_CLI_CONDA_ENV_NAME} environment for icav2 cli plugins" 1>&2 && \
       mamba create --yes \
         --name "${ICAV2_PLUGINS_CLI_CONDA_ENV_NAME}" \
-        python=3.10 && \
+        python="${ICAV2_PLUGINS_CLI_CONDA_PYTHON_VERSION}" && \
       echo "Installing ICAv2 CLI Plugins" 1>&2 && \
       wget --quiet \
         --output-document "icav2-plugins-cli--${ICAV2_PLUGINS_CLI_VERSION}.zip" \

cwl-ica-conda-env.yaml

@@ -2,31 +2,6 @@ name: cwl-ica
 channels:
   - defaults
 dependencies:
-  - python=3.10
-  - numpy
-  - pandas
-  - matplotlib
+  - python>=3.11  # Required for tomllib
   - pip
   - nodejs>=18
-  - pip:
-    - beautifulsoup4
-    - cwlref-runner
-    - cwltool
-    - cwl-utils
-    - pytest
-    - humanfriendly
-    - libica==2.3.0
-    - pyyaml
-    - requests
-    - verboselogs
-    - semantic_version
-    - python-dateutil
-    - docopt
-    - PyJWT
-    - tabulate
-    - mdutils==1.4.0
-    - deepdiff
-    - in-place
-    - pre-commit
-    - websocket
-    - websocket-client

install.sh

@@ -122,11 +122,7 @@ get_lib_path(){
 
   local conda_env_prefix="$1"
 
-  if [[ "${OSTYPE}" == "msys" ]]; then
-    echo "${conda_env_prefix}/Lib"
-  else
-    echo "${conda_env_prefix}/lib/python3.10"
-  fi
+  find "${conda_env_prefix}/lib" -name "cwl_ica" | sort | tail -n1
 }
 
 get_bin_path(){
@@ -240,7 +236,7 @@ update_conda_env() {
 
   if ! has_conda_env "${conda_env_name}"; then
     if [[ "${yes}" == "true" ]]; then
-      echo_stderr "Creating cwltool-icav1 conda env"
+      echo_stderr "Creating ${conda_env_name} conda env"
       run_conda_create "${conda_env_name}" "${conda_env_file}"
     else
       echo_stderr "'${conda_env_name}' conda env does not exist - would you like to create one?"
@@ -257,6 +253,7 @@ update_conda_env() {
     fi
   else
     if [[ "${yes}" == "true" ]]; then
+      # Install conda updates
       echo_stderr "Updating '${conda_env_name}' conda env"
       run_conda_update "${conda_env_name}" "${conda_env_file}"
     else
@@ -273,6 +270,11 @@ update_conda_env() {
       done
     fi
   fi
+
+  # Install other deps through pyproject.toml
+  if [[ "${conda_env_name}" == "${CWL_ICA_CONDA_ENV_NAME}" ]]; then
+    conda run --name "${conda_env_name}" pip install "$(get_this_path)/."
+  fi
 }
 
 check_conda_version() {
@@ -431,67 +433,48 @@ update_conda_env "${CWLTOOL_ICAV1_CONDA_ENV_NAME}" "${yes}"
 # Now we can obtain the env prefix which is where we will place our
 conda_cwl_ica_env_prefix="$(get_conda_env_prefix)"
 
-###########
-# COPY BINS
-###########
-
-# Copy over to conda env
-echo_stderr "Adding scripts to \"${conda_cwl_ica_env_prefix}/bin\""
-
-# Add scripts to bin
-chmod +x "$(get_this_path)/bin/"*
-rsync --archive \
-  --exclude "__init__.py" \
-  "$(get_this_path)/bin/" "$(get_bin_path "${conda_cwl_ica_env_prefix}")/"
-
-###########
-# COPY LIBS
-###########
-: '
-Copy over utils/classes/subcommands to library path
-'
-
-rsync --delete --archive \
-  "$(get_this_path)/utils/" "$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/"
-
-rsync --delete --archive \
-  "$(get_this_path)/classes/" "$(get_lib_path "${conda_cwl_ica_env_prefix}")/classes/"
-
-rsync --delete --archive \
-  "$(get_this_path)/subcommands/" "$(get_lib_path "${conda_cwl_ica_env_prefix}")/subcommands/"
-
-rsync --delete --archive \
-  --include "*.py" \
-  --exclude "*" \
-  "$(get_this_path)/bin/" "$(get_lib_path "${conda_cwl_ica_env_prefix}")/bin/"
-
 #####################
 # REPLACE __VERSION__
 #####################
 : '
 Only needed in the event that one is installing from source
 '
 
-sed "s/__VERSION__/latest/" \
-  "$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/__version__.py" > \
-  "$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/__version__.py.tmp"
+# From toml lib
 
-mv "$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/__version__.py.tmp" \
-  "$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/__version__.py"
+version_path="$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/__version__.py"
+version_number="$( \
+"${conda_cwl_ica_env_prefix}/bin/python3" - <<EOF
 
+# Imports
+import tomllib
+
+# Open toml file
+with open("$(get_this_path)/pyproject.toml", mode="rb") as toml_h:
+    config = tomllib.load(toml_h)
+
+# Get from toml file
+print(config["project"]["version"])
+EOF
+)"
+
+# Replace __VERSION__ with version number from toml lib
+sed "s/__VERSION__/${version_number}/" \
+  "${version_path}" > \
+  "${version_path}.tmp"
+mv "${version_path}.tmp" \
+  "${version_path}"
+
+# Get the latest cwltool version
 latest_cwltool_version="$(
   conda run --name "cwl-ica" \
   cwltool --version | cut -f2 -d' '
 )"
-
+globals_path="$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/globals.py"
 sed "s/__LATEST_CWLTOOL_VERSION__/${latest_cwltool_version}/" \
-  "$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/globals.py" > \
-  "$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/globals.py.tmp"
-
-  mv "$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/globals.py.tmp" \
-  "$(get_lib_path "${conda_cwl_ica_env_prefix}")/utils/globals.py"
-
-
+  "${globals_path}" > \
+  "${globals_path}.tmp"
+mv "${globals_path}.tmp" "${globals_path}"
 
 ##################################
 # Update npm and yarn

pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cwl-ica"
+version = "v1.7"
+authors = [
+  { name="Alexis Lucattini", email="[email protected]" },
+]
+description = "cwl-ica"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+dependencies = [
+  # Requirements created by pip-reqs
+  "beautifulsoup4 >= 4.12.2, < 5",
+  "cwl_utils >= 0.32, < 1",
+  "deepdiff >= 6.7.1, < 7",
+  "docopt >= 0.6.2, < 1",
+  "in_place >= 1.0.0, < 2",
+  "libica >= 2.3.0, < 3",
+  "matplotlib >= 3.8.0, < 4",
+  "mdutils >= 1.4.0, < 2",
+  "numpy >= 1.24.2, < 2",
+  "pandas >= 2.1.4, < 3",
+  "pydot >= 1.4.2, < 2",
+  "PyJWT >= 2.8.0, < 3",
+  "PyJWT >= 2.8.0, < 3",
+  "python_dateutil >= 2.8.2, < 3",
+  "Requests >= 2.31.0, < 3",
+  "ruamel.base >= 1.0.0, < 2",
+  "semantic_version >= 2.10.0, < 3",
+  "verboselogs >= 1.7, < 2",
+  "websocket_client >= 1.7.0, < 2",
+  "cwltool"  # Always the latest version
+]
+
+[project.urls]
+"Homepage" = "https://github.com/umccr/cwl-ica-cli"
+"Bug Tracker" = "https://github.com/umccr/cwl-ica-cli/issues"
+
+[project.scripts]
+"cwl-ica" = "cwl_ica.utils.cli:main"
+
+[tool.setuptools]
+script-files = [
+  "scripts/initialise_typescript_expression_directory.sh",
+  "scripts/update_yarn_dependencies.sh",
+  "scripts/validate_typescript_expressions_directory.sh",
+  "scripts/create_typescript_interface_from_schema.py"
+]

classes/command.py → src/cwl_ica/classes/command.py

@@ -1,7 +1,13 @@
+#!/usr/bin/env python3
+
+# External imports
 import sys
 from docopt import docopt
-from utils.logging import get_logger
 
+# Utils
+from ..utils.logging import get_logger
+
+# Set logger
 logger = get_logger()
 
 

classes/cwl.py → src/cwl_ica/classes/cwl.py

@@ -12,19 +12,26 @@
   - cwlschema -> alternate model for reading schema object
 """
 
+# Extenral imports
 from pathlib import Path
 from os import environ
-from utils.globals import ICAV1_CWLTOOL_VERSION, ICAV1_CWLTOOL_CONDA_ENV_NAME, LATEST_CWLTOOL_CONDA_ENV_NAME, \
-    LATEST_CWLTOOL_VERSION
-from utils.logging import get_logger
-from utils.errors import CWLPackagingError, CWLValidationError, InvalidAuthorshipError
-from tempfile import NamedTemporaryFile
-import json
-from utils.subprocess_handler import run_subprocess_proc
+import cwl_utils.parser as parser
 from hashlib import md5
 from string import ascii_lowercase, digits
-import cwl_utils.parser as parser
+from tempfile import NamedTemporaryFile
+import json
 
+# Utils
+from ..utils.globals import (
+    ICAV1_CWLTOOL_VERSION,
+    ICAV1_CWLTOOL_CONDA_ENV_NAME,
+    LATEST_CWLTOOL_CONDA_ENV_NAME,
+    LATEST_CWLTOOL_VERSION)
+from ..utils.logging import get_logger
+from ..utils.errors import CWLPackagingError, CWLValidationError, InvalidAuthorshipError
+from ..utils.subprocess_handler import run_subprocess_proc
+
+# Set logger
 logger = get_logger()
 
 

classes/cwl_expression.py → src/cwl_ica/classes/cwl_expression.py

@@ -5,16 +5,21 @@
 Based mostly on the cwl-utils package
 """
 
-from classes.cwl import CWL
-from utils.logging import get_logger
-from utils.errors import CWLValidationError, CWLPackagingError
+# External imports
 from tempfile import NamedTemporaryFile
 import os
 from pathlib import Path
 from cwl_utils.parser.latest import \
     ExpressionTool, LoadingOptions  # For creation of tool
 from ruamel.yaml.comments import CommentedMap as OrderedDict
-from utils.yaml import dump_cwl_yaml as dump_yaml, to_multiline_string
+
+# Utils
+from ..utils.logging import get_logger
+from ..utils.errors import CWLValidationError, CWLPackagingError
+from ..utils.yaml import dump_cwl_yaml as dump_yaml, to_multiline_string
+
+# Local imports
+from .cwl import CWL
 
 logger = get_logger()
 

classes/cwl_schema.py → src/cwl_ica/classes/cwl_schema.py

@@ -4,23 +4,29 @@
 A subclass of cwl, this function implements the validate and create for a cwl object
 Based mostly on the cwl-utils package
 """
+# External imports
 import re
 from copy import deepcopy
 from typing import List, Dict
-
 from ruamel.yaml import YAML
-
-from classes.cwl import CWL
-from utils.cwl_helper_utils import split_cwl_id_to_path_and_fragment
-from utils.logging import get_logger
-from utils.errors import CWLSchemaError
 from tempfile import NamedTemporaryFile
 from pathlib import Path
 from ruamel.yaml.comments import CommentedMap as OrderedDict
 import json
+
+# CWL Utils
 from cwl_utils.parser.latest import \
     RecordSchema
 
+# Utils
+from ..utils.cwl_helper_utils import split_cwl_id_to_path_and_fragment
+from ..utils.logging import get_logger
+from ..utils.errors import CWLSchemaError
+
+# Locals
+from .cwl import CWL
+
+# Set logger
 logger = get_logger()