From 918d77be2406f36e2d9a7d22cf659418785ff291 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 23 Nov 2023 14:57:45 +0100 Subject: [PATCH] remove last usage of pkg_resources --- cwltool/main.py | 2 +- cwltool/process.py | 2 +- cwltool/utils.py | 15 +++++++++++---- cwltool/validate_js.py | 2 +- docs/requirements.txt | 3 +-- pyproject.toml | 2 +- requirements.txt | 2 +- setup.py | 2 +- tests/test_cuda.py | 4 ++-- tests/test_load_tool.py | 2 +- tests/test_mpi.py | 2 +- tests/util.py | 16 +++++++++++----- tox.ini | 1 - 13 files changed, 33 insertions(+), 22 deletions(-) diff --git a/cwltool/main.py b/cwltool/main.py index 6a8e86d12..9c7343154 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -34,7 +34,6 @@ import argcomplete import coloredlogs import ruamel.yaml -from importlib_resources import files from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML from schema_salad.exceptions import ValidationException @@ -108,6 +107,7 @@ CWLOutputType, HasReqsHints, adjustDirObjs, + files, normalizeFilesDirs, processes_to_kill, trim_listing, diff --git a/cwltool/process.py b/cwltool/process.py index 7072d5c08..6c08007f5 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -33,7 +33,6 @@ ) from cwl_utils import expression -from importlib_resources import files from mypy_extensions import mypyc_attr from rdflib import Graph from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -70,6 +69,7 @@ aslist, cmp_like_py2, ensure_writable, + files, get_listing, normalizeFilesDirs, random_outdir, diff --git a/cwltool/utils.py b/cwltool/utils.py index aa25c171c..654b4f448 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -53,12 +53,19 @@ from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader +if sys.version_info >= (3, 9): + from importlib.resources import as_file, files +else: + from importlib_resources import as_file, files + if TYPE_CHECKING: from .command_line_tool import CallbackJob, ExpressionJob from .job import CommandLineJob, JobBase from .stdfsaccess import StdFsAccess from .workflow_job import WorkflowJob +__all__ = ["files", "as_file"] + __random_outdir: Optional[str] = None CONTENT_LIMIT = 64 * 1024 @@ -368,8 +375,8 @@ def add_writable_flag(p: str) -> None: if os.path.isdir(path): if include_root: add_writable_flag(path) - for root, dirs, files in os.walk(path): - for name in files: + for root, dirs, files_ in os.walk(path): + for name in files_: add_writable_flag(os.path.join(root, name)) for name in dirs: add_writable_flag(os.path.join(root, name)) @@ -380,8 +387,8 @@ def add_writable_flag(p: str) -> None: def ensure_non_writable(path: str) -> None: """Attempt to change permissions to ensure that a path is not writable.""" if os.path.isdir(path): - for root, dirs, files in os.walk(path): - for name in files: + for root, dirs, files_ in os.walk(path): + for name in files_: j = os.path.join(root, name) st = os.stat(j) mode = stat.S_IMODE(st.st_mode) diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 3b54c4d37..de4adaa14 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -18,7 +18,6 @@ from cwl_utils.errors import SubstitutionError from cwl_utils.expression import scanner as scan_expression from cwl_utils.sandboxjs import code_fragment_to_js, exec_js_process -from importlib_resources import files from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import ( ArraySchema, @@ -33,6 +32,7 @@ from .errors import WorkflowException from .loghandler import _logger +from .utils import files def is_expression(tool: Any, schema: Optional[Schema]) -> bool: diff --git a/docs/requirements.txt b/docs/requirements.txt index 7d9f9d0d4..f6be1e81c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,6 +2,5 @@ sphinx >= 2.2 sphinx-rtd-theme==1.3.0 sphinx-autoapi sphinx-autodoc-typehints -typed_ast;python_version<'3.8' sphinxcontrib-autoprogram -importlib_resources +importlib_resources;python_version<'3.9' diff --git a/pyproject.toml b/pyproject.toml index 7729cbc57..941e55bb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "mypy==1.7.1", # also update mypy-requirements.txt "types-requests", "types-psutil", - "importlib_resources>=1.4", # equivalent to Python 3.9 + "importlib_resources>=1.4;python_version<'3.9'", "ruamel.yaml>=0.16.0,<0.18", "schema-salad>=8.4.20230426093816,<9", "cwl-utils>=0.22,!=0.30", diff --git a/requirements.txt b/requirements.txt index 7d14648e0..63abe8ec2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ schema-salad>=8.4.20230426093816,<9 prov==1.5.1 mypy-extensions psutil>=5.6.6 -importlib_resources>=1.4 # equivalent to Python 3.9 +importlib_resources>=1.4;python_version<'3.9' coloredlogs pydot>=1.4.1 argcomplete>=1.12.0 diff --git a/setup.py b/setup.py index df80c4378..03fd68e89 100644 --- a/setup.py +++ b/setup.py @@ -109,7 +109,7 @@ "prov == 1.5.1", "mypy-extensions", "psutil >= 5.6.6", - "importlib_resources>=1.4", + "importlib_resources>=1.4;python_version<'3.9'", "coloredlogs", "pydot >= 1.4.1", "argcomplete", diff --git a/tests/test_cuda.py b/tests/test_cuda.py index e8de7cd63..27dfae39d 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -284,7 +284,7 @@ def test_cuda_job_setup_check_err_wrong_type_cuda_version( def test_cuda_eval_resource_range() -> None: - with open(get_data("cwltool/extensions-v1.1.yml")) as res: + with open(get_data("extensions-v1.1.yml")) as res: use_custom_schema("v1.2", "http://commonwl.org/cwltool", res.read()) joborder = {} # type: CWLObjectType @@ -301,7 +301,7 @@ def test_cuda_eval_resource_range() -> None: def test_cuda_eval_resource_max() -> None: - with open(get_data("cwltool/extensions-v1.1.yml")) as res: + with open(get_data("extensions-v1.1.yml")) as res: use_custom_schema("v1.2", "http://commonwl.org/cwltool", res.read()) joborder = {} # type: CWLObjectType diff --git a/tests/test_load_tool.py b/tests/test_load_tool.py index cf2cb620b..a74d64840 100644 --- a/tests/test_load_tool.py +++ b/tests/test_load_tool.py @@ -87,7 +87,7 @@ def test_load_graph_fragment_from_packed() -> None: loadingContext = LoadingContext() uri = Path(get_data("tests/wf/packed-with-loadlisting.cwl")).as_uri() + "#main" try: - with open(get_data("cwltool/extensions.yml")) as res: + with open(get_data("extensions.yml")) as res: use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read()) # The updater transforms LoadListingRequirement from an diff --git a/tests/test_mpi.py b/tests/test_mpi.py index e90d5d642..f7903d7b6 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -7,7 +7,6 @@ from typing import Any, Generator, List, MutableMapping, Optional, Tuple import pytest -from importlib_resources import files from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Names from schema_salad.utils import yaml_no_ts @@ -19,6 +18,7 @@ from cwltool.context import LoadingContext, RuntimeContext from cwltool.main import main from cwltool.mpi import MpiConfig, MPIRequirementName +from cwltool.utils import files from .util import get_data, working_directory diff --git a/tests/util.py b/tests/util.py index 7f33f1c40..82e49e560 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,3 +1,5 @@ +"""Test functions.""" +import atexit import contextlib import io import json @@ -5,15 +7,16 @@ import shutil import subprocess import sys +from contextlib import ExitStack from pathlib import Path from typing import Dict, Generator, List, Mapping, Optional, Tuple, Union import pytest -from pkg_resources import Requirement, ResolutionError, resource_filename from cwltool.env_to_stdout import deserialize_env from cwltool.main import main from cwltool.singularity import is_version_2_6, is_version_3_or_newer +from cwltool.utils import as_file, files def force_default_container(default_container_id: str, _: str) -> str: @@ -25,12 +28,15 @@ def get_data(filename: str) -> str: filename = os.path.normpath(filename) filepath = None try: - filepath = resource_filename(Requirement.parse("cwltool"), filename) - except ResolutionError: + file_manager = ExitStack() + atexit.register(file_manager.close) + traversable = files("cwltool") / filename + filepath = file_manager.enter_context(as_file(traversable)) + except ModuleNotFoundError: pass if not filepath or not os.path.isfile(filepath): - filepath = os.path.join(os.path.dirname(__file__), os.pardir, filename) - return str(Path(filepath).resolve()) + filepath = Path(os.path.dirname(__file__)) / ".." / filename + return str(filepath.resolve()) needs_docker = pytest.mark.skipif( diff --git a/tox.ini b/tox.ini index ed8f5c243..31b0fd91a 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,6 @@ deps = py3{8,9,10,11,12}-{unit,mypy}: -rtest-requirements.txt py3{8,9,10,11,12}-lint: -rlint-requirements.txt py3{8,9,10,11,12}-bandit: bandit - py3{8,9,10,11,12}-bandit: importlib_metadata != 4.8.0 py3{8,9,10,11,12}-mypy: -rmypy-requirements.txt py312-pydocstyle: pydocstyle py312-pydocstyle: diff-cover