From e80dafe12f71339e11cf3925874bbfa0a04cefb2 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 29 Jul 2022 10:22:30 -0700 Subject: [PATCH 01/45] test thingy # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- .../python/packaging/pyoxidizer/config.py | 3 ++- .../python/packaging/pyoxidizer/subsystem.py | 2 +- src/python/pants/bin/BUILD | 13 +++++++++++++ src/python/pants/bin/pants_loader.py | 4 +++- src/python/pants/core/goals/check.py | 3 +++ src/python/pants/engine/rules.py | 16 ++++++++++++++-- 6 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/python/pants/backend/python/packaging/pyoxidizer/config.py b/src/python/pants/backend/python/packaging/pyoxidizer/config.py index f04262ad7fe..2a51aea0485 100644 --- a/src/python/pants/backend/python/packaging/pyoxidizer/config.py +++ b/src/python/pants/backend/python/packaging/pyoxidizer/config.py @@ -9,7 +9,7 @@ DEFAULT_TEMPLATE = """ def make_exe(): - dist = default_python_distribution() + dist = default_python_distribution(python_version="3.9") policy = dist.make_python_packaging_policy() # Note: Adding this for pydanic and libs that have the "unable to load from memory" error @@ -17,6 +17,7 @@ def make_exe(): policy.resources_location_fallback = "filesystem-relative:lib" python_config = dist.make_python_interpreter_config() + $RUN_MODULE exe = dist.to_python_executable( diff --git a/src/python/pants/backend/python/packaging/pyoxidizer/subsystem.py b/src/python/pants/backend/python/packaging/pyoxidizer/subsystem.py index af84bceab5e..f8268cb2240 100644 --- a/src/python/pants/backend/python/packaging/pyoxidizer/subsystem.py +++ b/src/python/pants/backend/python/packaging/pyoxidizer/subsystem.py @@ -23,6 +23,6 @@ class PyOxidizer(PythonToolBase): default_main = ConsoleScript("pyoxidizer") register_interpreter_constraints = True - default_interpreter_constraints = ["CPython>=3.8,<4"] + default_interpreter_constraints = ["CPython==3.9.*"] args = ArgsListOption(example="--release") diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index 74ecd622770..6a6e3005f6a 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -77,4 +77,17 @@ pex_binary( strip_pex_env=False, ) + +python_distribution( + name="pants_bin", + dependencies=[":pants_loader"], + provides=python_artifact(name="pantsbuild.pants.oxidized", description="test oxidized pants build"), +) + +pyoxidizer_binary( + name="pants_oxidized_experimental", + dependencies=[":pants_bin"], + entry_point="pants.bin.pants_loader", +) + python_tests(name="tests") diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 37fb0ed86d5..39d43b17224 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -17,7 +17,6 @@ ) from pants.bin.pants_runner import PantsRunner - class PantsLoader: """Initial entrypoint for pants. @@ -86,6 +85,9 @@ def run_alternate_entrypoint(entrypoint: str) -> None: def run_default_entrypoint() -> None: start_time = time.time() try: + # Patch for PyOxidizer + if not sys.argv[0]: + sys.argv[0] = "PLACEHOLDER_BINARY" runner = PantsRunner(args=sys.argv, env=os.environ) exit_code = runner.run(start_time) except KeyboardInterrupt as e: diff --git a/src/python/pants/core/goals/check.py b/src/python/pants/core/goals/check.py index 9335ddd2cf5..6d135592efe 100644 --- a/src/python/pants/core/goals/check.py +++ b/src/python/pants/core/goals/check.py @@ -31,6 +31,9 @@ logger = logging.getLogger(__name__) +from os import environ +if environ.get("EX"): + raise Exception() @dataclass(frozen=True) class CheckResult: diff --git a/src/python/pants/engine/rules.py b/src/python/pants/engine/rules.py index 889c47c4992..747fcafe5ef 100644 --- a/src/python/pants/engine/rules.py +++ b/src/python/pants/engine/rules.py @@ -23,6 +23,7 @@ get_type_hints, ) + from typing_extensions import ParamSpec from pants.engine.engine_aware import SideEffecting @@ -42,6 +43,10 @@ PANTS_RULES_MODULE_KEY = "__pants_rules__" +from os import environ +if environ.get("EX"): + raise Exception() + # NB: This violates Python naming conventions of using snake_case for functions. This is because # SubsystemRule behaves very similarly to UnionRule and RootRule, and we want to use the same @@ -383,12 +388,19 @@ def collect_rules(*namespaces: Union[ModuleType, Mapping[str, Any]]) -> Iterable If no namespaces are given, collects all the @rules in the caller's module namespace. """ + + open("logfile.txt", "w").write("hello") + if not namespaces: currentframe = inspect.currentframe() - assert isinstance(currentframe, FrameType) + if not isinstance(currentframe, FrameType): + raise Exception(f"{currentframe=}") + #assert isinstance(currentframe, FrameType) caller_frame = currentframe.f_back caller_module = inspect.getmodule(caller_frame) - assert isinstance(caller_module, ModuleType) + if not isinstance(caller_module, ModuleType): + raise Exception(f"{currentframe=} {caller_frame=} {caller_module=}") + #assert isinstance(caller_module, ModuleType) namespaces = (caller_module,) def iter_rules(): From 30493dfbb1b8465a6dd196b40e29f6dd894a9d4b Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 29 Jul 2022 11:20:02 -0700 Subject: [PATCH 02/45] Package now builds and self-hosts # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- pants.toml | 1 + src/python/pants/bin/BUILD | 8 +------- src/python/pants/bin/py03-build.md | 13 +++++++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 src/python/pants/bin/py03-build.md diff --git a/pants.toml b/pants.toml index 08c5170d813..129024c3e5a 100644 --- a/pants.toml +++ b/pants.toml @@ -6,6 +6,7 @@ pythonpath = ["%(buildroot)s/pants-plugins"] backend_packages.add = [ "pants.backend.python", "pants.backend.experimental.python.lint.autoflake", + "pants.backend.experimental.python.packaging.pyoxidizer", "pants.backend.explorer", "pants.backend.python.lint.black", "pants.backend.python.lint.docformatter", diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index 6a6e3005f6a..86763d9d919 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -78,15 +78,9 @@ pex_binary( ) -python_distribution( - name="pants_bin", - dependencies=[":pants_loader"], - provides=python_artifact(name="pantsbuild.pants.oxidized", description="test oxidized pants build"), -) - pyoxidizer_binary( name="pants_oxidized_experimental", - dependencies=[":pants_bin"], + dependencies=["src/python/pants:pants-packaged"], entry_point="pants.bin.pants_loader", ) diff --git a/src/python/pants/bin/py03-build.md b/src/python/pants/bin/py03-build.md new file mode 100644 index 00000000000..a4ce3f036ba --- /dev/null +++ b/src/python/pants/bin/py03-build.md @@ -0,0 +1,13 @@ +NOTES: + +Obtain a self-signed certificate using info at at https://developer.apple.com/library/archive/documentation/Security/Conceptual/CodeSigningGuide/Procedures/Procedures.html + +I've called mine "py03-test" + +Get native engine to build first. + +Run `codesign -s py03-test /Users/chrisjrn/src/pants/dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/lib/pants/engine/internals/native_engine.so` + +Run `./pants --no-pantsd --no-remote-cache-write package --pyoxidizer-args="--system-rust --target-triple=aarch64-apple-darwin" src/python/pants/bin:pants_oxidized_experimental` + +Run `_PANTS_VERSION_OVERRIDE="2.14.0.dev" dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental --no-pantsd` \ No newline at end of file From bf95375b65b69394656da35ae6cfb9a52047b750 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 29 Jul 2022 12:17:49 -0700 Subject: [PATCH 03/45] Remove extraneous irrelevant stuff # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/engine/rules.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/python/pants/engine/rules.py b/src/python/pants/engine/rules.py index 747fcafe5ef..19438bb7170 100644 --- a/src/python/pants/engine/rules.py +++ b/src/python/pants/engine/rules.py @@ -43,10 +43,6 @@ PANTS_RULES_MODULE_KEY = "__pants_rules__" -from os import environ -if environ.get("EX"): - raise Exception() - # NB: This violates Python naming conventions of using snake_case for functions. This is because # SubsystemRule behaves very similarly to UnionRule and RootRule, and we want to use the same @@ -389,8 +385,6 @@ def collect_rules(*namespaces: Union[ModuleType, Mapping[str, Any]]) -> Iterable If no namespaces are given, collects all the @rules in the caller's module namespace. """ - open("logfile.txt", "w").write("hello") - if not namespaces: currentframe = inspect.currentframe() if not isinstance(currentframe, FrameType): From 0669daedd70a165ab307f52da1efc1f5f36ea2b2 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 29 Jul 2022 12:19:22 -0700 Subject: [PATCH 04/45] style changes # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/bin/pants_loader.py | 1 + src/python/pants/core/goals/check.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 39d43b17224..717d2244750 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -17,6 +17,7 @@ ) from pants.bin.pants_runner import PantsRunner + class PantsLoader: """Initial entrypoint for pants. diff --git a/src/python/pants/core/goals/check.py b/src/python/pants/core/goals/check.py index 6d135592efe..bf8e3208acc 100644 --- a/src/python/pants/core/goals/check.py +++ b/src/python/pants/core/goals/check.py @@ -32,9 +32,11 @@ logger = logging.getLogger(__name__) from os import environ + if environ.get("EX"): raise Exception() + @dataclass(frozen=True) class CheckResult: exit_code: int From 4c6c0642b7a7dc2e412177737dc5ed3454a63600 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 29 Jul 2022 12:20:00 -0700 Subject: [PATCH 05/45] Use `f_globals` instead of `inspect.getmodule` in `collect_rules` # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/engine/rules.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/python/pants/engine/rules.py b/src/python/pants/engine/rules.py index 19438bb7170..41fdf19e679 100644 --- a/src/python/pants/engine/rules.py +++ b/src/python/pants/engine/rules.py @@ -23,7 +23,6 @@ get_type_hints, ) - from typing_extensions import ParamSpec from pants.engine.engine_aware import SideEffecting @@ -389,18 +388,14 @@ def collect_rules(*namespaces: Union[ModuleType, Mapping[str, Any]]) -> Iterable currentframe = inspect.currentframe() if not isinstance(currentframe, FrameType): raise Exception(f"{currentframe=}") - #assert isinstance(currentframe, FrameType) caller_frame = currentframe.f_back - caller_module = inspect.getmodule(caller_frame) - if not isinstance(caller_module, ModuleType): - raise Exception(f"{currentframe=} {caller_frame=} {caller_module=}") - #assert isinstance(caller_module, ModuleType) - namespaces = (caller_module,) + global_items = caller_frame.f_globals + namespaces = (global_items,) def iter_rules(): for namespace in namespaces: mapping = namespace.__dict__ if isinstance(namespace, ModuleType) else namespace - for name, item in mapping.items(): + for item in mapping.values(): if not callable(item): continue rule = getattr(item, "rule", None) From 335c8984b282e738c3458d3a46f0d87c5a7cedb3 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 29 Jul 2022 12:20:00 -0700 Subject: [PATCH 06/45] Use `f_globals` instead of `inspect.getmodule` in `collect_rules` [ci skip-rust] (cherry picked from commit 4c6c0642b7a7dc2e412177737dc5ed3454a63600) [ci skip-build-wheels] --- src/python/pants/engine/rules.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/python/pants/engine/rules.py b/src/python/pants/engine/rules.py index 889c47c4992..e10009396cf 100644 --- a/src/python/pants/engine/rules.py +++ b/src/python/pants/engine/rules.py @@ -387,14 +387,15 @@ def collect_rules(*namespaces: Union[ModuleType, Mapping[str, Any]]) -> Iterable currentframe = inspect.currentframe() assert isinstance(currentframe, FrameType) caller_frame = currentframe.f_back - caller_module = inspect.getmodule(caller_frame) - assert isinstance(caller_module, ModuleType) - namespaces = (caller_module,) + assert isinstance(caller_frame, FrameType) + + global_items = caller_frame.f_globals + namespaces = (global_items,) def iter_rules(): for namespace in namespaces: mapping = namespace.__dict__ if isinstance(namespace, ModuleType) else namespace - for name, item in mapping.items(): + for item in mapping.values(): if not callable(item): continue rule = getattr(item, "rule", None) From 85f00b8d7aead9d24b68e68a5c4d57d21322ad54 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 29 Jul 2022 12:55:23 -0700 Subject: [PATCH 07/45] Adds `ox.py` construct, which we can hopefully remove sometime --- src/python/pants/bin/pants_loader.py | 2 ++ src/python/pants/ox.py | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 src/python/pants/ox.py diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 717d2244750..0f303b6e26c 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -9,6 +9,7 @@ import warnings from textwrap import dedent +from pants import ox from pants.base.exiter import PANTS_FAILED_EXIT_CODE from pants.bin.pants_env_vars import ( DAEMON_ENTRYPOINT, @@ -89,6 +90,7 @@ def run_default_entrypoint() -> None: # Patch for PyOxidizer if not sys.argv[0]: sys.argv[0] = "PLACEHOLDER_BINARY" + ox.is_oxidized = True runner = PantsRunner(args=sys.argv, env=os.environ) exit_code = runner.run(start_time) except KeyboardInterrupt as e: diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py new file mode 100644 index 00000000000..58ec3fa2814 --- /dev/null +++ b/src/python/pants/ox.py @@ -0,0 +1,5 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# True if this is being run as an oxidized binary. +is_oxidized = False \ No newline at end of file From 78deb85ca0bfa975751acad134a5445c020f9e43 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 29 Jul 2022 12:56:29 -0700 Subject: [PATCH 08/45] Adds workaround for subsystem initialisation error --- src/python/pants/option/subsystem.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/python/pants/option/subsystem.py b/src/python/pants/option/subsystem.py index cdb6b6ae8f8..603a86fab1e 100644 --- a/src/python/pants/option/subsystem.py +++ b/src/python/pants/option/subsystem.py @@ -9,6 +9,7 @@ from abc import ABCMeta from typing import Any, ClassVar, TypeVar +from pants import ox from pants.engine.internals.selectors import AwaitableConstraints, Get from pants.option.errors import OptionsError from pants.option.option_types import collect_options_info @@ -55,7 +56,12 @@ def signature(cls): partial_construct_subsystem.__name__ = name partial_construct_subsystem.__module__ = cls.__module__ partial_construct_subsystem.__doc__ = cls.help - _, class_definition_lineno = inspect.getsourcelines(cls) + + # `inspect.getsourcelines` does not work under oxidation + if not ox.is_oxidized: + _, class_definition_lineno = inspect.getsourcelines(cls) + else: + class_definition_lineno = 57 partial_construct_subsystem.__line_number__ = class_definition_lineno return dict( From e6f162edf9cef57df04b9848c6fd9ea1cb324c6f Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Tue, 2 Aug 2022 13:53:56 -0700 Subject: [PATCH 09/45] Replaces `pkgutil.get_data` with `resources.read_resource` # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/__init__.py | 0 .../backend/codegen/protobuf/scala/rules.py | 4 +-- .../docker/subsystems/dockerfile_parser.py | 4 +-- .../backend/go/go_sources/load_go_binary.py | 4 +-- .../dependency_inference/kotlin_parser.py | 4 +-- .../parse_python_dependencies.py | 4 +-- .../dependency_inference/scripts/__init__.py | 0 .../dependency_inference/scala_parser.py | 4 +-- .../backend/terraform/dependency_inference.py | 4 +-- src/python/pants/util/resources.py | 36 +++++++++++++++++++ src/python/pants/version.py | 5 +-- 11 files changed, 53 insertions(+), 16 deletions(-) create mode 100644 src/python/pants/__init__.py create mode 100644 src/python/pants/backend/python/dependency_inference/scripts/__init__.py create mode 100644 src/python/pants/util/resources.py diff --git a/src/python/pants/__init__.py b/src/python/pants/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/codegen/protobuf/scala/rules.py b/src/python/pants/backend/codegen/protobuf/scala/rules.py index ce33a42713d..42f9ac8442a 100644 --- a/src/python/pants/backend/codegen/protobuf/scala/rules.py +++ b/src/python/pants/backend/codegen/protobuf/scala/rules.py @@ -3,7 +3,6 @@ from __future__ import annotations import os -import pkgutil from dataclasses import dataclass from pants.backend.codegen import export_codegen_goal @@ -55,6 +54,7 @@ from pants.source.source_root import SourceRoot, SourceRootRequest from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource class GenerateScalaFromProtobufRequest(GenerateSourcesRequest): @@ -245,7 +245,7 @@ async def setup_scalapb_shim_classfiles( ) -> ScalaPBShimCompiledClassfiles: dest_dir = "classfiles" - scalapb_shim_content = pkgutil.get_data( + scalapb_shim_content = read_resource( "pants.backend.codegen.protobuf.scala", "ScalaPBShim.scala" ) if not scalapb_shim_content: diff --git a/src/python/pants/backend/docker/subsystems/dockerfile_parser.py b/src/python/pants/backend/docker/subsystems/dockerfile_parser.py index 53ae379494c..85fcc1a601d 100644 --- a/src/python/pants/backend/docker/subsystems/dockerfile_parser.py +++ b/src/python/pants/backend/docker/subsystems/dockerfile_parser.py @@ -4,7 +4,6 @@ from __future__ import annotations import json -import pkgutil from dataclasses import dataclass from pathlib import PurePath @@ -32,6 +31,7 @@ from pants.engine.unions import UnionRule from pants.util.docutil import git_url from pants.util.logging import LogLevel +from pants.util.resources import read_resource _DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py" _DOCKERFILE_PACKAGE = "pants.backend.docker.subsystems" @@ -74,7 +74,7 @@ class ParserSetup: @rule async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup: - parser_script_content = pkgutil.get_data(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL) + parser_script_content = read_resource(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL) if not parser_script_content: raise ValueError( f"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}." diff --git a/src/python/pants/backend/go/go_sources/load_go_binary.py b/src/python/pants/backend/go/go_sources/load_go_binary.py index 68c65369bd0..ca84f78e742 100644 --- a/src/python/pants/backend/go/go_sources/load_go_binary.py +++ b/src/python/pants/backend/go/go_sources/load_go_binary.py @@ -3,7 +3,6 @@ from __future__ import annotations -import pkgutil from dataclasses import dataclass from pants.backend.go.util_rules.build_pkg import BuildGoPackageRequest, BuiltGoPackage @@ -12,6 +11,7 @@ from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.util.resources import read_resource @dataclass(frozen=True) @@ -31,7 +31,7 @@ def debug_hint(self) -> str: def setup_files(dir_name: str, file_names: tuple[str, ...]) -> tuple[FileContent, ...]: def get_file(file_name: str) -> bytes: - content = pkgutil.get_data(f"pants.backend.go.go_sources.{dir_name}", file_name) + content = read_resource(f"pants.backend.go.go_sources.{dir_name}", file_name) if not content: raise AssertionError(f"Unable to find resource for `{file_name}`.") return content diff --git a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py index 95c60952da5..b7925c3cdc5 100644 --- a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py +++ b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py @@ -4,7 +4,6 @@ import json import os -import pkgutil from dataclasses import dataclass from typing import Any, Iterator @@ -25,6 +24,7 @@ from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource _PARSER_KOTLIN_VERSION = "1.6.20" @@ -230,7 +230,7 @@ async def resolve_fallible_result_to_analysis( async def setup_kotlin_parser_classfiles(jdk: InternalJdk) -> KotlinParserCompiledClassfiles: dest_dir = "classfiles" - parser_source_content = pkgutil.get_data( + parser_source_content = read_resource( "pants.backend.kotlin.dependency_inference", "KotlinParser.kt" ) if not parser_source_content: diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py index 183ded7992f..a404e0bc832 100644 --- a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py +++ b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import json -import pkgutil from dataclasses import dataclass from pants.backend.python.target_types import PythonSourceField @@ -16,6 +15,7 @@ from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel +from pants.util.resources import read_resource @dataclass(frozen=True) @@ -61,7 +61,7 @@ class ParserScript: @rule async def parser_script() -> ParserScript: - script = pkgutil.get_data(__name__, "scripts/dependency_parser.py") + script = read_resource(__name__, "scripts/dependency_parser.py") assert script is not None return ParserScript( await Get(Digest, CreateDigest([FileContent("__parse_python_dependencies.py", script)])) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/__init__.py b/src/python/pants/backend/python/dependency_inference/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/scala/dependency_inference/scala_parser.py b/src/python/pants/backend/scala/dependency_inference/scala_parser.py index 08fe72df3a0..c415d678034 100644 --- a/src/python/pants/backend/scala/dependency_inference/scala_parser.py +++ b/src/python/pants/backend/scala/dependency_inference/scala_parser.py @@ -5,7 +5,6 @@ import json import logging import os -import pkgutil from dataclasses import dataclass from typing import Any, Iterator, Mapping @@ -39,6 +38,7 @@ from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource logger = logging.getLogger(__name__) @@ -316,7 +316,7 @@ async def resolve_fallible_result_to_analysis( async def setup_scala_parser_classfiles(jdk: InternalJdk) -> ScalaParserCompiledClassfiles: dest_dir = "classfiles" - parser_source_content = pkgutil.get_data( + parser_source_content = read_resource( "pants.backend.scala.dependency_inference", "ScalaParser.scala" ) if not parser_source_content: diff --git a/src/python/pants/backend/terraform/dependency_inference.py b/src/python/pants/backend/terraform/dependency_inference.py index 928661e8239..a43bf8923f7 100644 --- a/src/python/pants/backend/terraform/dependency_inference.py +++ b/src/python/pants/backend/terraform/dependency_inference.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations -import pkgutil from dataclasses import dataclass from pathlib import PurePath @@ -33,6 +32,7 @@ from pants.util.docutil import git_url from pants.util.logging import LogLevel from pants.util.ordered_set import OrderedSet +from pants.util.resources import read_resource class TerraformHcl2Parser(PythonToolRequirementsBase): @@ -72,7 +72,7 @@ class ParserSetup: @rule async def setup_parser(hcl2_parser: TerraformHcl2Parser) -> ParserSetup: - parser_script_content = pkgutil.get_data("pants.backend.terraform", "hcl2_parser.py") + parser_script_content = read_resource("pants.backend.terraform", "hcl2_parser.py") if not parser_script_content: raise ValueError("Unable to find source to hcl2_parser.py wrapper script.") diff --git a/src/python/pants/util/resources.py b/src/python/pants/util/resources.py new file mode 100644 index 00000000000..761e02aadde --- /dev/null +++ b/src/python/pants/util/resources.py @@ -0,0 +1,36 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +import importlib +from importlib import resources +from itertools import chain + + +def read_resource(package_or_module: str, resource: str) -> bytes: + """Reads a resource file from within the Pants package itself. + + This helper function is designed for compatibility with `pkgutil.get_data()` wherever possible, + but also allows compability with PEP302 pluggable importers such as included with PyOxidizer. + This requires that resources are loaded from a valid Python package (i.e. must have an + `__init__.py` file in the directory). + """ + + a = importlib.import_module(package_or_module) + package_ = a.__package__ + + if package_ is None: + raise ValueError( + "`read_resource` can only help find resources for packages or modules that live in " + "a package." + ) + + resource_parts = resource.split("/") + + if len(resource_parts) == 1: + package = package_ + else: + package = ".".join(chain((package_,), resource_parts[:-1])) + resource = resource_parts[-1] + + return resources.read_binary(package, resource) diff --git a/src/python/pants/version.py b/src/python/pants/version.py index adee488c8b5..6d9192af32b 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -2,10 +2,11 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os -import pkgutil from packaging.version import Version +from pants.util.resources import read_resource + # Set this env var to override the version pants reports. Useful for testing. _PANTS_VERSION_OVERRIDE = "_PANTS_VERSION_OVERRIDE" @@ -14,7 +15,7 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - pkgutil.get_data(__name__, "VERSION").decode().strip() # type: ignore[union-attr] + read_resource(__name__, "VERSION").decode().strip() ) PANTS_SEMVER = Version(VERSION) From a9f4f7fcc9fdd039d7fcb1ac8dcc63c5dc3cceef Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Tue, 2 Aug 2022 13:55:44 -0700 Subject: [PATCH 10/45] Update pyo3 run instructions # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/bin/pants_loader.py | 5 ----- src/python/pants/bin/py03-build.md | 2 +- src/python/pants/ox.py | 10 +++++++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 0f303b6e26c..37fb0ed86d5 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -9,7 +9,6 @@ import warnings from textwrap import dedent -from pants import ox from pants.base.exiter import PANTS_FAILED_EXIT_CODE from pants.bin.pants_env_vars import ( DAEMON_ENTRYPOINT, @@ -87,10 +86,6 @@ def run_alternate_entrypoint(entrypoint: str) -> None: def run_default_entrypoint() -> None: start_time = time.time() try: - # Patch for PyOxidizer - if not sys.argv[0]: - sys.argv[0] = "PLACEHOLDER_BINARY" - ox.is_oxidized = True runner = PantsRunner(args=sys.argv, env=os.environ) exit_code = runner.run(start_time) except KeyboardInterrupt as e: diff --git a/src/python/pants/bin/py03-build.md b/src/python/pants/bin/py03-build.md index a4ce3f036ba..a8e723a84e6 100644 --- a/src/python/pants/bin/py03-build.md +++ b/src/python/pants/bin/py03-build.md @@ -10,4 +10,4 @@ Run `codesign -s py03-test /Users/chrisjrn/src/pants/dist/src.python.pants.bin/p Run `./pants --no-pantsd --no-remote-cache-write package --pyoxidizer-args="--system-rust --target-triple=aarch64-apple-darwin" src/python/pants/bin:pants_oxidized_experimental` -Run `_PANTS_VERSION_OVERRIDE="2.14.0.dev" dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental --no-pantsd` \ No newline at end of file +Run `dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental --no-pantsd` \ No newline at end of file diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index 58ec3fa2814..672352a15db 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -1,5 +1,13 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import sys + # True if this is being run as an oxidized binary. -is_oxidized = False \ No newline at end of file +is_oxidized = False + + +# Patch for PyOxidizer +if not sys.argv[0]: + sys.argv[0] = "PLACEHOLDER_BINARY" + is_oxidized = True From 2b4c73dce76c5a63750c7d5d34d55c2ef1075f97 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Tue, 2 Aug 2022 17:27:53 -0700 Subject: [PATCH 11/45] Make `VERSION` resource loadable without upsetting the rest of the pants distribution # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/BUILD | 4 ++-- src/python/pants/__init__.py | 0 src/python/pants/backend/python/dependency_inference/BUILD | 4 +++- .../dependency_inference/parse_python_dependencies.py | 2 +- .../pants/backend/python/dependency_inference/scripts/BUILD | 6 ++++-- .../scripts/{dependency_parser.py => dependency_parser_py} | 0 src/python/pants/bin/BUILD | 2 ++ src/python/pants/bin/VERSION | 1 + src/python/pants/version.py | 2 +- 9 files changed, 14 insertions(+), 7 deletions(-) delete mode 100644 src/python/pants/__init__.py rename src/python/pants/backend/python/dependency_inference/scripts/{dependency_parser.py => dependency_parser_py} (100%) create mode 120000 src/python/pants/bin/VERSION diff --git a/src/python/pants/BUILD b/src/python/pants/BUILD index e09db9f10dd..1f33ff68322 100644 --- a/src/python/pants/BUILD +++ b/src/python/pants/BUILD @@ -5,7 +5,7 @@ python_sources( overrides={ # Enable `python -m pants ...` style execution ala `json.tool` or `venv`. "__main__.py": {"dependencies": ["src/python/pants/bin:pants_loader"]}, - "version.py": {"dependencies": ["./VERSION:resources"]}, + "version.py": {"dependencies": ["./bin/VERSION:resources"]}, }, ) @@ -13,7 +13,7 @@ python_test_utils(name="test_utils") python_distribution( name="pants-packaged", - dependencies=["./__main__.py", ":resources"], + dependencies=["./__main__.py", ":resources", ], # Because we have native code, this will cause the wheel to use whatever the ABI is for the # interpreter used to run setup.py, e.g. `cp36m-macosx_10_15_x86_64`. sdist=False, diff --git a/src/python/pants/__init__.py b/src/python/pants/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/python/pants/backend/python/dependency_inference/BUILD b/src/python/pants/backend/python/dependency_inference/BUILD index 0290569f3ec..74d7180d82c 100644 --- a/src/python/pants/backend/python/dependency_inference/BUILD +++ b/src/python/pants/backend/python/dependency_inference/BUILD @@ -1,7 +1,9 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -python_sources() +python_sources( + dependencies=["./scripts:dependency_parser",], +) python_tests( name="tests", diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py index a404e0bc832..96b897397b9 100644 --- a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py +++ b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py @@ -61,7 +61,7 @@ class ParserScript: @rule async def parser_script() -> ParserScript: - script = read_resource(__name__, "scripts/dependency_parser.py") + script = read_resource(__name__, "scripts/dependency_parser_py") assert script is not None return ParserScript( await Get(Digest, CreateDigest([FileContent("__parse_python_dependencies.py", script)])) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/BUILD b/src/python/pants/backend/python/dependency_inference/scripts/BUILD index e142aded947..60134271a58 100644 --- a/src/python/pants/backend/python/dependency_inference/scripts/BUILD +++ b/src/python/pants/backend/python/dependency_inference/scripts/BUILD @@ -1,11 +1,13 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -resource(name="dependency_parser", source="dependency_parser.py") +resource(name="dependency_parser", source="dependency_parser_py", dependencies=[":init_py",]) +resource(name="init_py", source="__init__.py",) + # Also expose scripts as python sources so they get formatted/linted/checked. python_source( name="dependency_parser_source", - source="dependency_parser.py", + source="dependency_parser_py", # This is run with Python 2.7 and 3.5+, so we shouldn't be running pyupgrade. # skip_pyupgrade=True, ) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/dependency_parser.py b/src/python/pants/backend/python/dependency_inference/scripts/dependency_parser_py similarity index 100% rename from src/python/pants/backend/python/dependency_inference/scripts/dependency_parser.py rename to src/python/pants/backend/python/dependency_inference/scripts/dependency_parser_py diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index 86763d9d919..2ebfbc49f75 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -85,3 +85,5 @@ pyoxidizer_binary( ) python_tests(name="tests") + +resources(name="resources", sources=["VERSION"]) diff --git a/src/python/pants/bin/VERSION b/src/python/pants/bin/VERSION new file mode 120000 index 00000000000..6ff19de4b80 --- /dev/null +++ b/src/python/pants/bin/VERSION @@ -0,0 +1 @@ +../VERSION \ No newline at end of file diff --git a/src/python/pants/version.py b/src/python/pants/version.py index 6d9192af32b..b4121605887 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -15,7 +15,7 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - read_resource(__name__, "VERSION").decode().strip() + read_resource("pants.bin", "VERSION").decode().strip() ) PANTS_SEMVER = Version(VERSION) From 913e894925476a390cf9f8bf522972e6bc952ea7 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Tue, 2 Aug 2022 13:53:56 -0700 Subject: [PATCH 12/45] Replaces `pkgutil.get_data` with `resources.read_resource` # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/__init__.py | 0 .../backend/codegen/protobuf/scala/rules.py | 4 +-- .../docker/subsystems/dockerfile_parser.py | 4 +-- .../backend/go/go_sources/load_go_binary.py | 4 +-- .../dependency_inference/kotlin_parser.py | 4 +-- .../parse_python_dependencies.py | 4 +-- .../dependency_inference/scripts/__init__.py | 0 .../dependency_inference/scala_parser.py | 4 +-- .../backend/terraform/dependency_inference.py | 4 +-- src/python/pants/util/resources.py | 36 +++++++++++++++++++ src/python/pants/version.py | 5 +-- 11 files changed, 53 insertions(+), 16 deletions(-) create mode 100644 src/python/pants/__init__.py create mode 100644 src/python/pants/backend/python/dependency_inference/scripts/__init__.py create mode 100644 src/python/pants/util/resources.py diff --git a/src/python/pants/__init__.py b/src/python/pants/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/codegen/protobuf/scala/rules.py b/src/python/pants/backend/codegen/protobuf/scala/rules.py index ce33a42713d..42f9ac8442a 100644 --- a/src/python/pants/backend/codegen/protobuf/scala/rules.py +++ b/src/python/pants/backend/codegen/protobuf/scala/rules.py @@ -3,7 +3,6 @@ from __future__ import annotations import os -import pkgutil from dataclasses import dataclass from pants.backend.codegen import export_codegen_goal @@ -55,6 +54,7 @@ from pants.source.source_root import SourceRoot, SourceRootRequest from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource class GenerateScalaFromProtobufRequest(GenerateSourcesRequest): @@ -245,7 +245,7 @@ async def setup_scalapb_shim_classfiles( ) -> ScalaPBShimCompiledClassfiles: dest_dir = "classfiles" - scalapb_shim_content = pkgutil.get_data( + scalapb_shim_content = read_resource( "pants.backend.codegen.protobuf.scala", "ScalaPBShim.scala" ) if not scalapb_shim_content: diff --git a/src/python/pants/backend/docker/subsystems/dockerfile_parser.py b/src/python/pants/backend/docker/subsystems/dockerfile_parser.py index 53ae379494c..85fcc1a601d 100644 --- a/src/python/pants/backend/docker/subsystems/dockerfile_parser.py +++ b/src/python/pants/backend/docker/subsystems/dockerfile_parser.py @@ -4,7 +4,6 @@ from __future__ import annotations import json -import pkgutil from dataclasses import dataclass from pathlib import PurePath @@ -32,6 +31,7 @@ from pants.engine.unions import UnionRule from pants.util.docutil import git_url from pants.util.logging import LogLevel +from pants.util.resources import read_resource _DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py" _DOCKERFILE_PACKAGE = "pants.backend.docker.subsystems" @@ -74,7 +74,7 @@ class ParserSetup: @rule async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup: - parser_script_content = pkgutil.get_data(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL) + parser_script_content = read_resource(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL) if not parser_script_content: raise ValueError( f"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}." diff --git a/src/python/pants/backend/go/go_sources/load_go_binary.py b/src/python/pants/backend/go/go_sources/load_go_binary.py index 68c65369bd0..ca84f78e742 100644 --- a/src/python/pants/backend/go/go_sources/load_go_binary.py +++ b/src/python/pants/backend/go/go_sources/load_go_binary.py @@ -3,7 +3,6 @@ from __future__ import annotations -import pkgutil from dataclasses import dataclass from pants.backend.go.util_rules.build_pkg import BuildGoPackageRequest, BuiltGoPackage @@ -12,6 +11,7 @@ from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.util.resources import read_resource @dataclass(frozen=True) @@ -31,7 +31,7 @@ def debug_hint(self) -> str: def setup_files(dir_name: str, file_names: tuple[str, ...]) -> tuple[FileContent, ...]: def get_file(file_name: str) -> bytes: - content = pkgutil.get_data(f"pants.backend.go.go_sources.{dir_name}", file_name) + content = read_resource(f"pants.backend.go.go_sources.{dir_name}", file_name) if not content: raise AssertionError(f"Unable to find resource for `{file_name}`.") return content diff --git a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py index 95c60952da5..b7925c3cdc5 100644 --- a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py +++ b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py @@ -4,7 +4,6 @@ import json import os -import pkgutil from dataclasses import dataclass from typing import Any, Iterator @@ -25,6 +24,7 @@ from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource _PARSER_KOTLIN_VERSION = "1.6.20" @@ -230,7 +230,7 @@ async def resolve_fallible_result_to_analysis( async def setup_kotlin_parser_classfiles(jdk: InternalJdk) -> KotlinParserCompiledClassfiles: dest_dir = "classfiles" - parser_source_content = pkgutil.get_data( + parser_source_content = read_resource( "pants.backend.kotlin.dependency_inference", "KotlinParser.kt" ) if not parser_source_content: diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py index 183ded7992f..a404e0bc832 100644 --- a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py +++ b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import json -import pkgutil from dataclasses import dataclass from pants.backend.python.target_types import PythonSourceField @@ -16,6 +15,7 @@ from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel +from pants.util.resources import read_resource @dataclass(frozen=True) @@ -61,7 +61,7 @@ class ParserScript: @rule async def parser_script() -> ParserScript: - script = pkgutil.get_data(__name__, "scripts/dependency_parser.py") + script = read_resource(__name__, "scripts/dependency_parser.py") assert script is not None return ParserScript( await Get(Digest, CreateDigest([FileContent("__parse_python_dependencies.py", script)])) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/__init__.py b/src/python/pants/backend/python/dependency_inference/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/scala/dependency_inference/scala_parser.py b/src/python/pants/backend/scala/dependency_inference/scala_parser.py index 08fe72df3a0..c415d678034 100644 --- a/src/python/pants/backend/scala/dependency_inference/scala_parser.py +++ b/src/python/pants/backend/scala/dependency_inference/scala_parser.py @@ -5,7 +5,6 @@ import json import logging import os -import pkgutil from dataclasses import dataclass from typing import Any, Iterator, Mapping @@ -39,6 +38,7 @@ from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource logger = logging.getLogger(__name__) @@ -316,7 +316,7 @@ async def resolve_fallible_result_to_analysis( async def setup_scala_parser_classfiles(jdk: InternalJdk) -> ScalaParserCompiledClassfiles: dest_dir = "classfiles" - parser_source_content = pkgutil.get_data( + parser_source_content = read_resource( "pants.backend.scala.dependency_inference", "ScalaParser.scala" ) if not parser_source_content: diff --git a/src/python/pants/backend/terraform/dependency_inference.py b/src/python/pants/backend/terraform/dependency_inference.py index 928661e8239..a43bf8923f7 100644 --- a/src/python/pants/backend/terraform/dependency_inference.py +++ b/src/python/pants/backend/terraform/dependency_inference.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations -import pkgutil from dataclasses import dataclass from pathlib import PurePath @@ -33,6 +32,7 @@ from pants.util.docutil import git_url from pants.util.logging import LogLevel from pants.util.ordered_set import OrderedSet +from pants.util.resources import read_resource class TerraformHcl2Parser(PythonToolRequirementsBase): @@ -72,7 +72,7 @@ class ParserSetup: @rule async def setup_parser(hcl2_parser: TerraformHcl2Parser) -> ParserSetup: - parser_script_content = pkgutil.get_data("pants.backend.terraform", "hcl2_parser.py") + parser_script_content = read_resource("pants.backend.terraform", "hcl2_parser.py") if not parser_script_content: raise ValueError("Unable to find source to hcl2_parser.py wrapper script.") diff --git a/src/python/pants/util/resources.py b/src/python/pants/util/resources.py new file mode 100644 index 00000000000..761e02aadde --- /dev/null +++ b/src/python/pants/util/resources.py @@ -0,0 +1,36 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +import importlib +from importlib import resources +from itertools import chain + + +def read_resource(package_or_module: str, resource: str) -> bytes: + """Reads a resource file from within the Pants package itself. + + This helper function is designed for compatibility with `pkgutil.get_data()` wherever possible, + but also allows compability with PEP302 pluggable importers such as included with PyOxidizer. + This requires that resources are loaded from a valid Python package (i.e. must have an + `__init__.py` file in the directory). + """ + + a = importlib.import_module(package_or_module) + package_ = a.__package__ + + if package_ is None: + raise ValueError( + "`read_resource` can only help find resources for packages or modules that live in " + "a package." + ) + + resource_parts = resource.split("/") + + if len(resource_parts) == 1: + package = package_ + else: + package = ".".join(chain((package_,), resource_parts[:-1])) + resource = resource_parts[-1] + + return resources.read_binary(package, resource) diff --git a/src/python/pants/version.py b/src/python/pants/version.py index adee488c8b5..6d9192af32b 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -2,10 +2,11 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os -import pkgutil from packaging.version import Version +from pants.util.resources import read_resource + # Set this env var to override the version pants reports. Useful for testing. _PANTS_VERSION_OVERRIDE = "_PANTS_VERSION_OVERRIDE" @@ -14,7 +15,7 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - pkgutil.get_data(__name__, "VERSION").decode().strip() # type: ignore[union-attr] + read_resource(__name__, "VERSION").decode().strip() ) PANTS_SEMVER = Version(VERSION) From ac24a35474fc62cc2a87e606b5d59ee62c1f3d93 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Tue, 2 Aug 2022 17:27:53 -0700 Subject: [PATCH 13/45] Make `VERSION` resource loadable without upsetting the rest of the pants distribution # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/BUILD | 4 ++-- src/python/pants/__init__.py | 0 src/python/pants/backend/python/dependency_inference/BUILD | 4 +++- .../dependency_inference/parse_python_dependencies.py | 2 +- .../pants/backend/python/dependency_inference/scripts/BUILD | 6 ++++-- .../scripts/{dependency_parser.py => dependency_parser_py} | 0 src/python/pants/bin/BUILD | 2 ++ src/python/pants/bin/VERSION | 1 + src/python/pants/version.py | 2 +- 9 files changed, 14 insertions(+), 7 deletions(-) delete mode 100644 src/python/pants/__init__.py rename src/python/pants/backend/python/dependency_inference/scripts/{dependency_parser.py => dependency_parser_py} (100%) create mode 120000 src/python/pants/bin/VERSION diff --git a/src/python/pants/BUILD b/src/python/pants/BUILD index e09db9f10dd..1f33ff68322 100644 --- a/src/python/pants/BUILD +++ b/src/python/pants/BUILD @@ -5,7 +5,7 @@ python_sources( overrides={ # Enable `python -m pants ...` style execution ala `json.tool` or `venv`. "__main__.py": {"dependencies": ["src/python/pants/bin:pants_loader"]}, - "version.py": {"dependencies": ["./VERSION:resources"]}, + "version.py": {"dependencies": ["./bin/VERSION:resources"]}, }, ) @@ -13,7 +13,7 @@ python_test_utils(name="test_utils") python_distribution( name="pants-packaged", - dependencies=["./__main__.py", ":resources"], + dependencies=["./__main__.py", ":resources", ], # Because we have native code, this will cause the wheel to use whatever the ABI is for the # interpreter used to run setup.py, e.g. `cp36m-macosx_10_15_x86_64`. sdist=False, diff --git a/src/python/pants/__init__.py b/src/python/pants/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/python/pants/backend/python/dependency_inference/BUILD b/src/python/pants/backend/python/dependency_inference/BUILD index 0290569f3ec..74d7180d82c 100644 --- a/src/python/pants/backend/python/dependency_inference/BUILD +++ b/src/python/pants/backend/python/dependency_inference/BUILD @@ -1,7 +1,9 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -python_sources() +python_sources( + dependencies=["./scripts:dependency_parser",], +) python_tests( name="tests", diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py index a404e0bc832..96b897397b9 100644 --- a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py +++ b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py @@ -61,7 +61,7 @@ class ParserScript: @rule async def parser_script() -> ParserScript: - script = read_resource(__name__, "scripts/dependency_parser.py") + script = read_resource(__name__, "scripts/dependency_parser_py") assert script is not None return ParserScript( await Get(Digest, CreateDigest([FileContent("__parse_python_dependencies.py", script)])) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/BUILD b/src/python/pants/backend/python/dependency_inference/scripts/BUILD index e142aded947..60134271a58 100644 --- a/src/python/pants/backend/python/dependency_inference/scripts/BUILD +++ b/src/python/pants/backend/python/dependency_inference/scripts/BUILD @@ -1,11 +1,13 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -resource(name="dependency_parser", source="dependency_parser.py") +resource(name="dependency_parser", source="dependency_parser_py", dependencies=[":init_py",]) +resource(name="init_py", source="__init__.py",) + # Also expose scripts as python sources so they get formatted/linted/checked. python_source( name="dependency_parser_source", - source="dependency_parser.py", + source="dependency_parser_py", # This is run with Python 2.7 and 3.5+, so we shouldn't be running pyupgrade. # skip_pyupgrade=True, ) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/dependency_parser.py b/src/python/pants/backend/python/dependency_inference/scripts/dependency_parser_py similarity index 100% rename from src/python/pants/backend/python/dependency_inference/scripts/dependency_parser.py rename to src/python/pants/backend/python/dependency_inference/scripts/dependency_parser_py diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index 74ecd622770..8a4ba73a622 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -78,3 +78,5 @@ pex_binary( ) python_tests(name="tests") + +resources(name="resources", sources=["VERSION"]) diff --git a/src/python/pants/bin/VERSION b/src/python/pants/bin/VERSION new file mode 120000 index 00000000000..6ff19de4b80 --- /dev/null +++ b/src/python/pants/bin/VERSION @@ -0,0 +1 @@ +../VERSION \ No newline at end of file diff --git a/src/python/pants/version.py b/src/python/pants/version.py index 6d9192af32b..b4121605887 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -15,7 +15,7 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - read_resource(__name__, "VERSION").decode().strip() + read_resource("pants.bin", "VERSION").decode().strip() ) PANTS_SEMVER = Version(VERSION) From 353f05dcbe178e2ffe6163cd67cd4f7719c6a248 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Wed, 3 Aug 2022 07:53:36 -0700 Subject: [PATCH 14/45] Lint build files # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/BUILD | 5 ++++- .../pants/backend/python/dependency_inference/BUILD | 4 +++- .../python/dependency_inference/scripts/BUILD | 13 +++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/python/pants/BUILD b/src/python/pants/BUILD index 1f33ff68322..d04565ff2d4 100644 --- a/src/python/pants/BUILD +++ b/src/python/pants/BUILD @@ -13,7 +13,10 @@ python_test_utils(name="test_utils") python_distribution( name="pants-packaged", - dependencies=["./__main__.py", ":resources", ], + dependencies=[ + "./__main__.py", + ":resources", + ], # Because we have native code, this will cause the wheel to use whatever the ABI is for the # interpreter used to run setup.py, e.g. `cp36m-macosx_10_15_x86_64`. sdist=False, diff --git a/src/python/pants/backend/python/dependency_inference/BUILD b/src/python/pants/backend/python/dependency_inference/BUILD index 74d7180d82c..a7935536dbc 100644 --- a/src/python/pants/backend/python/dependency_inference/BUILD +++ b/src/python/pants/backend/python/dependency_inference/BUILD @@ -2,7 +2,9 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). python_sources( - dependencies=["./scripts:dependency_parser",], + dependencies=[ + "./scripts:dependency_parser", + ], ) python_tests( diff --git a/src/python/pants/backend/python/dependency_inference/scripts/BUILD b/src/python/pants/backend/python/dependency_inference/scripts/BUILD index 60134271a58..41c5d4acb49 100644 --- a/src/python/pants/backend/python/dependency_inference/scripts/BUILD +++ b/src/python/pants/backend/python/dependency_inference/scripts/BUILD @@ -1,8 +1,17 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -resource(name="dependency_parser", source="dependency_parser_py", dependencies=[":init_py",]) -resource(name="init_py", source="__init__.py",) +resource( + name="dependency_parser", + source="dependency_parser_py", + dependencies=[ + ":init_py", + ], +) +resource( + name="init_py", + source="__init__.py", +) # Also expose scripts as python sources so they get formatted/linted/checked. python_source( From 6582d443aca65dbdcb03b9980844b34b23aaedeb Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 10:34:23 -0700 Subject: [PATCH 15/45] Creates the `pants._version` package, designed to ensure that the `VERSION` resource is reliably available. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/BUILD | 1 - src/python/pants/_version/BUILD | 12 ++++++++++++ src/python/pants/{bin => _version}/VERSION | 0 src/python/pants/_version/__init__.py | 0 src/python/pants/bin/BUILD | 2 -- src/python/pants/version.py | 8 +++++++- 6 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 src/python/pants/_version/BUILD rename src/python/pants/{bin => _version}/VERSION (100%) create mode 100644 src/python/pants/_version/__init__.py diff --git a/src/python/pants/BUILD b/src/python/pants/BUILD index d04565ff2d4..fe6ee11f410 100644 --- a/src/python/pants/BUILD +++ b/src/python/pants/BUILD @@ -5,7 +5,6 @@ python_sources( overrides={ # Enable `python -m pants ...` style execution ala `json.tool` or `venv`. "__main__.py": {"dependencies": ["src/python/pants/bin:pants_loader"]}, - "version.py": {"dependencies": ["./bin/VERSION:resources"]}, }, ) diff --git a/src/python/pants/_version/BUILD b/src/python/pants/_version/BUILD new file mode 100644 index 00000000000..dd4c37fe926 --- /dev/null +++ b/src/python/pants/_version/BUILD @@ -0,0 +1,12 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# This package is used to provide a non-namespaced Python package from which +# `importlib.resources.read_binary` can read our `VERSION` file. The `VERSION` file is a +# symlink to the concrete `VERSION` file in `src/python/pants`. This creates a minimal package +# that can be imported by `pants.version` during tests, and inferred by Pants as a dependency. +# Future versions of `importlib.resources` will be able to read resources from namespace +# packages, at which point, this package will no longer need to exist. + +python_sources(dependencies=["./VERSION:resources"]) +resources(name="resources", sources=["VERSION"]) diff --git a/src/python/pants/bin/VERSION b/src/python/pants/_version/VERSION similarity index 100% rename from src/python/pants/bin/VERSION rename to src/python/pants/_version/VERSION diff --git a/src/python/pants/_version/__init__.py b/src/python/pants/_version/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index 8a4ba73a622..74ecd622770 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -78,5 +78,3 @@ pex_binary( ) python_tests(name="tests") - -resources(name="resources", sources=["VERSION"]) diff --git a/src/python/pants/version.py b/src/python/pants/version.py index b4121605887..24c6013ffc1 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -3,6 +3,9 @@ import os +# Generate a inferrable dependency on the `pants._version` package and its associated resources. +import pants._version + from packaging.version import Version from pants.util.resources import read_resource @@ -15,7 +18,10 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - read_resource("pants.bin", "VERSION").decode().strip() + # `pants._version` is a non-namespace package that can be loaded by `importlib.resources` in + # pre-3.11 versions of Python. Once we can read `pants.VERSION` using the resource loader, + # we can remove the `pants._version` package. See #16379. + read_resource(pants._version.__name__, "VERSION").decode().strip() ) PANTS_SEMVER = Version(VERSION) From a914acc9930fb8ae2f96dc6f7719414da627e38d Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 10:34:55 -0700 Subject: [PATCH 16/45] lint # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/version.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/python/pants/version.py b/src/python/pants/version.py index 24c6013ffc1..c896838311c 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -3,11 +3,10 @@ import os -# Generate a inferrable dependency on the `pants._version` package and its associated resources. -import pants._version - from packaging.version import Version +# Generate a inferrable dependency on the `pants._version` package and its associated resources. +import pants._version from pants.util.resources import read_resource # Set this env var to override the version pants reports. Useful for testing. @@ -18,8 +17,8 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - # `pants._version` is a non-namespace package that can be loaded by `importlib.resources` in - # pre-3.11 versions of Python. Once we can read `pants.VERSION` using the resource loader, + # `pants._version` is a non-namespace package that can be loaded by `importlib.resources` in + # pre-3.11 versions of Python. Once we can read `pants.VERSION` using the resource loader, # we can remove the `pants._version` package. See #16379. read_resource(pants._version.__name__, "VERSION").decode().strip() ) From 184940f1d52c5892dc2b78bd0cfee1c40d3412f1 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 11:12:56 -0700 Subject: [PATCH 17/45] Patch `sys.argv[0]` to be a correct value under pyoxidizer --- src/python/pants/ox.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index 672352a15db..ed4476947fd 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -9,5 +9,9 @@ # Patch for PyOxidizer if not sys.argv[0]: - sys.argv[0] = "PLACEHOLDER_BINARY" + # A giant pile of pants consumer code copies around `sys.argv`, which is modified in an + # invalid way by python's `pymain_run_module` support. For our purposes, the executable + # distribution is the correct `argv[0]`. + # See https://github.com/indygreg/PyOxidizer/issues/307 + sys.argv[0] = sys.executable is_oxidized = True From a226425c601b8bedfc06bfa5f0923cb4db9313b0 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 11:15:50 -0700 Subject: [PATCH 18/45] Use a better sentinel for `class_definition_lineno` --- src/python/pants/option/subsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/option/subsystem.py b/src/python/pants/option/subsystem.py index 603a86fab1e..237eeb1f6c2 100644 --- a/src/python/pants/option/subsystem.py +++ b/src/python/pants/option/subsystem.py @@ -61,7 +61,7 @@ def signature(cls): if not ox.is_oxidized: _, class_definition_lineno = inspect.getsourcelines(cls) else: - class_definition_lineno = 57 + class_definition_lineno = 0 # `inspect.getsourcelines` returns 0 when undefined. partial_construct_subsystem.__line_number__ = class_definition_lineno return dict( From c633384b0ae65c64b833133ad8351d590c41ac78 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 11:47:47 -0700 Subject: [PATCH 19/45] Tidy up `ox.py` # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/ox.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index ed4476947fd..8908d6c2201 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -3,15 +3,20 @@ import sys -# True if this is being run as an oxidized binary. -is_oxidized = False +# Provide the `is_oxidized` symbol, to allow for workarounds in Pants code where we use things +# that don't work under PyOxidizer's custom importer. `oxidized_importer` is only accessible +# in Pants under PyOxidizer, so an import failure will occur if we're not oxidized. +try: + import oxidized_importer # type: ignore # pants: no-infer-dep # noqa: F401 + + is_oxidized = True +except ModuleNotFoundError: + is_oxidized = False -# Patch for PyOxidizer -if not sys.argv[0]: - # A giant pile of pants consumer code copies around `sys.argv`, which is modified in an +if is_oxidized and not sys.argv[0]: + # A not insignificant amount of Pants code relies on `sys.argv[0]`, which is modified in an # invalid way by python's `pymain_run_module` support. For our purposes, the executable # distribution is the correct `argv[0]`. # See https://github.com/indygreg/PyOxidizer/issues/307 sys.argv[0] = sys.executable - is_oxidized = True From 32207de11a98374368c7aeb3cbd98ce8112e9e4f Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 11:51:04 -0700 Subject: [PATCH 20/45] Ensure that `ox` is bootstrapped before anything that depends on `sys.argv`. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/bin/pants_loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 37fb0ed86d5..526865b2f8e 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -9,6 +9,7 @@ import warnings from textwrap import dedent +from pants import ox # noqa: F401 from pants.base.exiter import PANTS_FAILED_EXIT_CODE from pants.bin.pants_env_vars import ( DAEMON_ENTRYPOINT, From 3b651efdcaf80d115d68458a495184f519083b4b Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 11:51:36 -0700 Subject: [PATCH 21/45] Remove customisations to `pyoxidizer` config # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/python/packaging/pyoxidizer/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/backend/python/packaging/pyoxidizer/config.py b/src/python/pants/backend/python/packaging/pyoxidizer/config.py index 38f2c36f8a5..64ca0794643 100644 --- a/src/python/pants/backend/python/packaging/pyoxidizer/config.py +++ b/src/python/pants/backend/python/packaging/pyoxidizer/config.py @@ -9,7 +9,7 @@ DEFAULT_TEMPLATE = """ def make_exe(): - dist = default_python_distribution(python_version="3.9") + dist = default_python_distribution() policy = dist.make_python_packaging_policy() # Note: Adding this for pydanic and libs that have the "unable to load from memory" error From b7b2f064cc2240bb80383343489480b19329f050 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 11:52:02 -0700 Subject: [PATCH 22/45] rename `pyO3-build.md` # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/bin/{py03-build.md => pyO3-build.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/python/pants/bin/{py03-build.md => pyO3-build.md} (84%) diff --git a/src/python/pants/bin/py03-build.md b/src/python/pants/bin/pyO3-build.md similarity index 84% rename from src/python/pants/bin/py03-build.md rename to src/python/pants/bin/pyO3-build.md index a8e723a84e6..f6437f15b1d 100644 --- a/src/python/pants/bin/py03-build.md +++ b/src/python/pants/bin/pyO3-build.md @@ -8,6 +8,6 @@ Get native engine to build first. Run `codesign -s py03-test /Users/chrisjrn/src/pants/dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/lib/pants/engine/internals/native_engine.so` -Run `./pants --no-pantsd --no-remote-cache-write package --pyoxidizer-args="--system-rust --target-triple=aarch64-apple-darwin" src/python/pants/bin:pants_oxidized_experimental` +Run `./pants --no-pantsd --no-remote-cache-write package --pyoxidizer-args="--target-triple=aarch64-apple-darwin" src/python/pants/bin:pants_oxidized_experimental` Run `dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental --no-pantsd` \ No newline at end of file From 8e2ee9ee165795acb229810bb7c9a5a446fd8091 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 12:55:45 -0700 Subject: [PATCH 23/45] Reset the interpreter constraints for pyoxidizer # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/python/packaging/pyoxidizer/subsystem.py | 2 +- src/python/pants/bin/pyO3-build.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/pants/backend/python/packaging/pyoxidizer/subsystem.py b/src/python/pants/backend/python/packaging/pyoxidizer/subsystem.py index f8268cb2240..af84bceab5e 100644 --- a/src/python/pants/backend/python/packaging/pyoxidizer/subsystem.py +++ b/src/python/pants/backend/python/packaging/pyoxidizer/subsystem.py @@ -23,6 +23,6 @@ class PyOxidizer(PythonToolBase): default_main = ConsoleScript("pyoxidizer") register_interpreter_constraints = True - default_interpreter_constraints = ["CPython==3.9.*"] + default_interpreter_constraints = ["CPython>=3.8,<4"] args = ArgsListOption(example="--release") diff --git a/src/python/pants/bin/pyO3-build.md b/src/python/pants/bin/pyO3-build.md index f6437f15b1d..28df3814265 100644 --- a/src/python/pants/bin/pyO3-build.md +++ b/src/python/pants/bin/pyO3-build.md @@ -8,6 +8,6 @@ Get native engine to build first. Run `codesign -s py03-test /Users/chrisjrn/src/pants/dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/lib/pants/engine/internals/native_engine.so` -Run `./pants --no-pantsd --no-remote-cache-write package --pyoxidizer-args="--target-triple=aarch64-apple-darwin" src/python/pants/bin:pants_oxidized_experimental` +Run `./pants --no-pantsd --no-remote-cache-write package --pyoxidizer-args="--target-triple=aarch64-apple-darwin" --pyoxidizer-interpreter-constraints="['CPython==3.9.*']" src/python/pants/bin:pants_oxidized_experimental` -Run `dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental --no-pantsd` \ No newline at end of file +Run `dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental` (outside of Pants itself) \ No newline at end of file From 96029000a0d55098dbae0ff7ee9db856b579d134 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 12:57:59 -0700 Subject: [PATCH 24/45] Remove `bin/VERSION` # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/bin/BUILD | 2 -- src/python/pants/bin/VERSION | 1 - 2 files changed, 3 deletions(-) delete mode 120000 src/python/pants/bin/VERSION diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index 2ebfbc49f75..86763d9d919 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -85,5 +85,3 @@ pyoxidizer_binary( ) python_tests(name="tests") - -resources(name="resources", sources=["VERSION"]) diff --git a/src/python/pants/bin/VERSION b/src/python/pants/bin/VERSION deleted file mode 120000 index 6ff19de4b80..00000000000 --- a/src/python/pants/bin/VERSION +++ /dev/null @@ -1 +0,0 @@ -../VERSION \ No newline at end of file From 1b710738953dedaa80780d605f5a41b8eec57276 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 13:04:35 -0700 Subject: [PATCH 25/45] reorganise pyoxidizer-howto # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../bin/{pyO3-build.md => pyoxidizer-howto.md} | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) rename src/python/pants/bin/{pyO3-build.md => pyoxidizer-howto.md} (56%) diff --git a/src/python/pants/bin/pyO3-build.md b/src/python/pants/bin/pyoxidizer-howto.md similarity index 56% rename from src/python/pants/bin/pyO3-build.md rename to src/python/pants/bin/pyoxidizer-howto.md index 28df3814265..b25dc8a75e7 100644 --- a/src/python/pants/bin/pyO3-build.md +++ b/src/python/pants/bin/pyoxidizer-howto.md @@ -1,13 +1,17 @@ NOTES: -Obtain a self-signed certificate using info at at https://developer.apple.com/library/archive/documentation/Security/Conceptual/CodeSigningGuide/Procedures/Procedures.html +Run `./pants --no-pantsd --no-remote-cache-write package --pyoxidizer-args="--target-triple=aarch64-apple-darwin" --pyoxidizer-interpreter-constraints="['CPython==3.9.*']" src/python/pants/bin:pants_oxidized_experimental` -I've called mine "py03-test" +The binary will be `dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental` -- this will not work on the pants repo itself (yet?) -Get native engine to build first. -Run `codesign -s py03-test /Users/chrisjrn/src/pants/dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/lib/pants/engine/internals/native_engine.so` +# Code signing errors? -Run `./pants --no-pantsd --no-remote-cache-write package --pyoxidizer-args="--target-triple=aarch64-apple-darwin" --pyoxidizer-interpreter-constraints="['CPython==3.9.*']" src/python/pants/bin:pants_oxidized_experimental` -Run `dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental` (outside of Pants itself) \ No newline at end of file +Obtain a self-signed certificate using info at at https://developer.apple.com/library/archive/documentation/Security/Conceptual/CodeSigningGuide/Procedures/Procedures.html + +I've called mine "pyox-test". + +Get native engine to build first (i.e. run `./pants` and wait for rust to finish doing its thing). + +Run `codesign -s pyox-test /Users/chrisjrn/src/pants/dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/lib/pants/engine/internals/native_engine.so` From aaadeaae96ec477a422470bcb30858c7da5b7551 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 4 Aug 2022 13:14:46 -0700 Subject: [PATCH 26/45] Disable static linking against copyleft libraries # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/python/packaging/pyoxidizer/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/pants/backend/python/packaging/pyoxidizer/config.py b/src/python/pants/backend/python/packaging/pyoxidizer/config.py index 64ca0794643..9f2f3cee32b 100644 --- a/src/python/pants/backend/python/packaging/pyoxidizer/config.py +++ b/src/python/pants/backend/python/packaging/pyoxidizer/config.py @@ -11,6 +11,7 @@ def make_exe(): dist = default_python_distribution() policy = dist.make_python_packaging_policy() + policy.extension_module_filter = "no-copyleft" # Note: Adding this for pydanic and libs that have the "unable to load from memory" error # https://github.com/indygreg/PyOxidizer/issues/438 From 03478d35df69f721cdb1672c365ff4b8c39b0274 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 5 Aug 2022 08:37:43 -0700 Subject: [PATCH 27/45] Adds no-op `boostrap_pyoxidizer` function # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/bin/pants_loader.py | 3 ++- src/python/pants/ox.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 526865b2f8e..c2d2d1df0e4 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -9,7 +9,7 @@ import warnings from textwrap import dedent -from pants import ox # noqa: F401 +from pants import ox from pants.base.exiter import PANTS_FAILED_EXIT_CODE from pants.bin.pants_env_vars import ( DAEMON_ENTRYPOINT, @@ -110,6 +110,7 @@ def main(cls) -> None: def main() -> None: + ox.bootstrap_pyoxidizer() PantsLoader.main() diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index 8908d6c2201..f9261a0787d 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -1,8 +1,17 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import logging import sys +_logger = logging.getLogger(__name__) + + +def bootstrap_pyoxidizer() -> None: + if is_oxidized: + _logger.info("Pants is running as a PyOxidizer binary.") + + # Provide the `is_oxidized` symbol, to allow for workarounds in Pants code where we use things # that don't work under PyOxidizer's custom importer. `oxidized_importer` is only accessible # in Pants under PyOxidizer, so an import failure will occur if we're not oxidized. From 6cf9a62180c82187d586d5d55f40217eb44ff5ce Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Tue, 9 Aug 2022 10:13:18 -0700 Subject: [PATCH 28/45] Just enough machinery to get `pex` to run `pip`. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/bin/pants_loader.py | 6 ++ src/python/pants/ox.py | 105 +++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index c2d2d1df0e4..03b3209f0fa 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -1,6 +1,7 @@ # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). + import importlib import locale import os @@ -111,8 +112,13 @@ def main(cls) -> None: def main() -> None: ox.bootstrap_pyoxidizer() + + if ox.is_oxidized and ox.pex_main(): + return + PantsLoader.main() if __name__ == "__main__": + print(f"welcome to pants! {sys.argv=} {sys.executable=} {os.getcwd()=}", file=sys.stderr) main() diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index f9261a0787d..0f785d01304 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -1,8 +1,12 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import importlib.machinery import logging +import os +import runpy import sys +import zipimport _logger = logging.getLogger(__name__) @@ -29,3 +33,104 @@ def bootstrap_pyoxidizer() -> None: # distribution is the correct `argv[0]`. # See https://github.com/indygreg/PyOxidizer/issues/307 sys.argv[0] = sys.executable + + +def pex_main() -> bool: + + with open("/Users/chrisjrn/src/pants/oxidized-invocations.txt", "a") as f: + import datetime + + f.write(f"{datetime.datetime.now()}: {sys.argv=} {sys.path=}\n") + + if sys.argv[1] == "./pex": + run_as_pex() + return True + + if sys.argv[1] == "-sE" and sys.argv[2].endswith("/pex"): + print("bloop", file=sys.stderr) + run_as_pip() + return True + + if len(sys.argv) == 4 and sys.argv[1:3] == ["-s", "-c"]: + run_as_dash_c() + return True + + if ("-m", "venv") in zip(sys.argv, sys.argv[1:]): + run_as_venv() + return True + + return False + + +def prepare_import_machinery(): + print(f"pex main? {sys.version_info=} {sys.argv=}", flush=True) + + # pex relies heavily on `__file__`, which the Oxidized importer does not + # believe in. This reinstates the default Python import machinery before + # loading and running `pex`, but keeps the PyOxidizer machinery at lowest + # priority, so we can still load interned `.py` sources (e.g. the stdlib) + sys.meta_path = [ + importlib.machinery.BuiltinImporter, + importlib.machinery.FrozenImporter, + importlib.machinery.PathFinder, + ] + sys.meta_path + sys.path_hooks = [ + zipimport.zipimporter, + importlib.machinery.FileFinder.path_hook( + (importlib.machinery.ExtensionFileLoader, [".cpython-39-darwin.so", ".abi3.so", ".so"]), + (importlib.machinery.SourceFileLoader, [".py"]), + (importlib.machinery.SourcelessFileLoader, [".pyc"]), + ), + ] + sys.path_hooks + + +def run_as_pex(): + prepare_import_machinery() + g = {} + f = runpy.run_path("./pex", init_globals=g) + del sys.argv[1] + # sys.argv += ["-v"] * 3 + f["bootstrap_pex"]("./pex") + sys.exit(0) + + +def run_as_venv(): + prepare_import_machinery() + index = sys.argv.index("-m") + + import venv + + venv.__file__ = "EXTREMELY/FLAH/BLAH" + + sys.argv[1:] = sys.argv[index + 2 :] + runpy.run_module("venv") + sys.exit(0) + + +def run_as_dash_c(): + prepare_import_machinery() + print(f"{os.environ=}", file=sys.stderr) + if "PEX" in os.environ: + # Get pex into the modules cache + pex = runpy.run_path(os.environ["PEX"]) + pex["bootstrap_pex"](os.environ["PEX"], execute=False) + + index = sys.argv.index("-c") + exec(sys.argv[index + 1], {}, {}) + sys.exit(0) + + +def run_as_pip(): + prepare_import_machinery() + print("floop", file=sys.stderr) + + start_path = os.path.join(os.path.dirname(sys.argv[2]), "lib") + ver = sys.version_info + py_name = f"python{ver.major}.{ver.minor}" + for (dirname, _, _) in os.walk(start_path): + if dirname.endswith(py_name) or dirname.endswith(os.path.join(py_name, "site-packages")): + sys.path.append(dirname) + + del sys.argv[1:3] + runpy.run_module(mod_name="pip", run_name="__main__", alter_sys=True) + sys.exit(0) From 53f9e3f9a43ef084263d552b8c0c8e5d57c57bef Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Tue, 9 Aug 2022 13:30:49 -0700 Subject: [PATCH 29/45] Makes `run_pex_venv` script work # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/ox.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index 0f785d01304..a7690788fae 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -5,6 +5,7 @@ import logging import os import runpy +import site import sys import zipimport @@ -39,16 +40,18 @@ def pex_main() -> bool: with open("/Users/chrisjrn/src/pants/oxidized-invocations.txt", "a") as f: import datetime + import site - f.write(f"{datetime.datetime.now()}: {sys.argv=} {sys.path=}\n") + f.write( + f"{datetime.datetime.now()}: {sys.argv=} {sys.path=} {site.PREFIXES=} {site.getsitepackages()=}\n" + ) if sys.argv[1] == "./pex": run_as_pex() return True if sys.argv[1] == "-sE" and sys.argv[2].endswith("/pex"): - print("bloop", file=sys.stderr) - run_as_pip() + run_pex_venv() return True if len(sys.argv) == 4 and sys.argv[1:3] == ["-s", "-c"]: @@ -63,7 +66,7 @@ def pex_main() -> bool: def prepare_import_machinery(): - print(f"pex main? {sys.version_info=} {sys.argv=}", flush=True) + # print(f"pex main? {sys.version_info=} {sys.argv=}", flush=True) # pex relies heavily on `__file__`, which the Oxidized importer does not # believe in. This reinstates the default Python import machinery before @@ -109,7 +112,6 @@ def run_as_venv(): def run_as_dash_c(): prepare_import_machinery() - print(f"{os.environ=}", file=sys.stderr) if "PEX" in os.environ: # Get pex into the modules cache pex = runpy.run_path(os.environ["PEX"]) @@ -120,17 +122,11 @@ def run_as_dash_c(): sys.exit(0) -def run_as_pip(): +def run_pex_venv(): prepare_import_machinery() - print("floop", file=sys.stderr) - - start_path = os.path.join(os.path.dirname(sys.argv[2]), "lib") - ver = sys.version_info - py_name = f"python{ver.major}.{ver.minor}" - for (dirname, _, _) in os.walk(start_path): - if dirname.endswith(py_name) or dirname.endswith(os.path.join(py_name, "site-packages")): - sys.path.append(dirname) - + path_to_run = sys.argv[2] + site.PREFIXES = [os.path.dirname(path_to_run)] + site.addsitepackages(set()) del sys.argv[1:3] - runpy.run_module(mod_name="pip", run_name="__main__", alter_sys=True) + runpy.run_path(path_to_run, run_name="__main__") sys.exit(0) From cc60f16b57b12138dffdbd3f4aa75c56266f67aa Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 11 Aug 2022 12:02:35 -0700 Subject: [PATCH 30/45] Creates `traditional_import_machinery()` context manager and uses in plugin loading # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/init/plugin_resolver.py | 18 ++++--- src/python/pants/ox.py | 65 ++++++++++++++++-------- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/src/python/pants/init/plugin_resolver.py b/src/python/pants/init/plugin_resolver.py index 4eaf127fd67..54023e676b3 100644 --- a/src/python/pants/init/plugin_resolver.py +++ b/src/python/pants/init/plugin_resolver.py @@ -128,13 +128,17 @@ def resolve( env: CompleteEnvironment, ) -> WorkingSet: """Resolves any configured plugins and adds them to the working_set.""" - for resolved_plugin_location in self._resolve_plugins( - options_bootstrapper, env, self._request - ): - site.addsitedir( - resolved_plugin_location - ) # Activate any .pth files plugin wheels may have. - self._working_set.add_entry(resolved_plugin_location) + from pants import ox + + with ox.traditional_import_machinery(): + for resolved_plugin_location in self._resolve_plugins( + options_bootstrapper, env, self._request + ): + logger.warning(f"{resolved_plugin_location=}") + site.addsitedir( + resolved_plugin_location + ) # Activate any .pth files plugin wheels may have. + self._working_set.add_entry(resolved_plugin_location) return self._working_set def _resolve_plugins( diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index a7690788fae..7cf651ea76a 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -1,6 +1,8 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import contextlib +import functools import importlib.machinery import logging import os @@ -65,30 +67,51 @@ def pex_main() -> bool: return False -def prepare_import_machinery(): - # print(f"pex main? {sys.version_info=} {sys.argv=}", flush=True) - +@contextlib.contextmanager +def traditional_import_machinery(): # pex relies heavily on `__file__`, which the Oxidized importer does not # believe in. This reinstates the default Python import machinery before # loading and running `pex`, but keeps the PyOxidizer machinery at lowest # priority, so we can still load interned `.py` sources (e.g. the stdlib) - sys.meta_path = [ - importlib.machinery.BuiltinImporter, - importlib.machinery.FrozenImporter, - importlib.machinery.PathFinder, - ] + sys.meta_path - sys.path_hooks = [ - zipimport.zipimporter, - importlib.machinery.FileFinder.path_hook( - (importlib.machinery.ExtensionFileLoader, [".cpython-39-darwin.so", ".abi3.so", ".so"]), - (importlib.machinery.SourceFileLoader, [".py"]), - (importlib.machinery.SourcelessFileLoader, [".pyc"]), - ), - ] + sys.path_hooks + old_sys_meta_path = sys.meta_path + old_sys_path_hooks = sys.path_hooks + if is_oxidized: + sys.meta_path = [ + importlib.machinery.BuiltinImporter, + importlib.machinery.FrozenImporter, + importlib.machinery.PathFinder, + ] + sys.meta_path + sys.path_hooks = [ + zipimport.zipimporter, + importlib.machinery.FileFinder.path_hook( + ( + importlib.machinery.ExtensionFileLoader, + [".cpython-39-darwin.so", ".abi3.so", ".so"], + ), + (importlib.machinery.SourceFileLoader, [".py"]), + (importlib.machinery.SourcelessFileLoader, [".pyc"]), + ), + ] + sys.path_hooks + + yield + + sys.meta_path = old_sys_meta_path + sys.path_hooks = old_sys_path_hooks + + +def use_traditional_import_machinery(f): + @functools.wraps(f) + def wrapped(*a, **k): + with traditional_import_machinery(): + return f(*a, **k) + + return wrapped + + +@use_traditional_import_machinery def run_as_pex(): - prepare_import_machinery() g = {} f = runpy.run_path("./pex", init_globals=g) del sys.argv[1] @@ -97,21 +120,21 @@ def run_as_pex(): sys.exit(0) +@use_traditional_import_machinery def run_as_venv(): - prepare_import_machinery() index = sys.argv.index("-m") import venv - venv.__file__ = "EXTREMELY/FLAH/BLAH" + venv.__file__ = "SOMETHING/THAT/IS/NOT/NONE" sys.argv[1:] = sys.argv[index + 2 :] runpy.run_module("venv") sys.exit(0) +@use_traditional_import_machinery def run_as_dash_c(): - prepare_import_machinery() if "PEX" in os.environ: # Get pex into the modules cache pex = runpy.run_path(os.environ["PEX"]) @@ -122,8 +145,8 @@ def run_as_dash_c(): sys.exit(0) +@use_traditional_import_machinery def run_pex_venv(): - prepare_import_machinery() path_to_run = sys.argv[2] site.PREFIXES = [os.path.dirname(path_to_run)] site.addsitepackages(set()) From 94582dc48c6532e495a499756c2d782e9fc57ba8 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 11 Aug 2022 12:11:10 -0700 Subject: [PATCH 31/45] tidy up comments; remove logging infrastructure # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/ox.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index 7cf651ea76a..ed6ed0a4524 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -40,14 +40,6 @@ def bootstrap_pyoxidizer() -> None: def pex_main() -> bool: - with open("/Users/chrisjrn/src/pants/oxidized-invocations.txt", "a") as f: - import datetime - import site - - f.write( - f"{datetime.datetime.now()}: {sys.argv=} {sys.path=} {site.PREFIXES=} {site.getsitepackages()=}\n" - ) - if sys.argv[1] == "./pex": run_as_pex() return True @@ -115,7 +107,6 @@ def run_as_pex(): g = {} f = runpy.run_path("./pex", init_globals=g) del sys.argv[1] - # sys.argv += ["-v"] * 3 f["bootstrap_pex"]("./pex") sys.exit(0) @@ -126,6 +117,10 @@ def run_as_venv(): import venv + # `venv` is supplied by the oxidized importer (the stdlib is embedded in the rust binary) + # but uses the `__file__` attribute to find the location of `activate` scripts. These scripts + # are not needed by pex, so we're setting the value to something bogus just to prevent + # subsequent exceptions. venv.__file__ = "SOMETHING/THAT/IS/NOT/NONE" sys.argv[1:] = sys.argv[index + 2 :] From 17ecac0538e9165f93f3fdca1f783a017a3d0e54 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 11 Aug 2022 12:18:15 -0700 Subject: [PATCH 32/45] Revert "Revert "Replace `pkgutil.get_data` with new `read_resource` API (#16379)" (#16433)" This reverts commit 3f165962afac243218fa12a6b73e3e64a0be8fd8. --- src/python/pants/BUILD | 6 ++-- src/python/pants/_version/BUILD | 12 +++++++ src/python/pants/_version/VERSION | 1 + src/python/pants/_version/__init__.py | 0 .../backend/codegen/protobuf/scala/rules.py | 4 +-- .../docker/subsystems/dockerfile_parser.py | 4 +-- .../backend/go/go_sources/load_go_binary.py | 4 +-- .../dependency_inference/kotlin_parser.py | 4 +-- .../backend/python/dependency_inference/BUILD | 6 +++- .../parse_python_dependencies.py | 4 +-- .../python/dependency_inference/scripts/BUILD | 15 ++++++-- .../dependency_inference/scripts/__init__.py | 0 ...endency_parser.py => dependency_parser_py} | 0 .../dependency_inference/scala_parser.py | 4 +-- .../backend/terraform/dependency_inference.py | 4 +-- src/python/pants/util/resources.py | 36 +++++++++++++++++++ src/python/pants/version.py | 10 ++++-- 17 files changed, 93 insertions(+), 21 deletions(-) create mode 100644 src/python/pants/_version/BUILD create mode 120000 src/python/pants/_version/VERSION create mode 100644 src/python/pants/_version/__init__.py create mode 100644 src/python/pants/backend/python/dependency_inference/scripts/__init__.py rename src/python/pants/backend/python/dependency_inference/scripts/{dependency_parser.py => dependency_parser_py} (100%) create mode 100644 src/python/pants/util/resources.py diff --git a/src/python/pants/BUILD b/src/python/pants/BUILD index e09db9f10dd..fe6ee11f410 100644 --- a/src/python/pants/BUILD +++ b/src/python/pants/BUILD @@ -5,7 +5,6 @@ python_sources( overrides={ # Enable `python -m pants ...` style execution ala `json.tool` or `venv`. "__main__.py": {"dependencies": ["src/python/pants/bin:pants_loader"]}, - "version.py": {"dependencies": ["./VERSION:resources"]}, }, ) @@ -13,7 +12,10 @@ python_test_utils(name="test_utils") python_distribution( name="pants-packaged", - dependencies=["./__main__.py", ":resources"], + dependencies=[ + "./__main__.py", + ":resources", + ], # Because we have native code, this will cause the wheel to use whatever the ABI is for the # interpreter used to run setup.py, e.g. `cp36m-macosx_10_15_x86_64`. sdist=False, diff --git a/src/python/pants/_version/BUILD b/src/python/pants/_version/BUILD new file mode 100644 index 00000000000..dd4c37fe926 --- /dev/null +++ b/src/python/pants/_version/BUILD @@ -0,0 +1,12 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# This package is used to provide a non-namespaced Python package from which +# `importlib.resources.read_binary` can read our `VERSION` file. The `VERSION` file is a +# symlink to the concrete `VERSION` file in `src/python/pants`. This creates a minimal package +# that can be imported by `pants.version` during tests, and inferred by Pants as a dependency. +# Future versions of `importlib.resources` will be able to read resources from namespace +# packages, at which point, this package will no longer need to exist. + +python_sources(dependencies=["./VERSION:resources"]) +resources(name="resources", sources=["VERSION"]) diff --git a/src/python/pants/_version/VERSION b/src/python/pants/_version/VERSION new file mode 120000 index 00000000000..6ff19de4b80 --- /dev/null +++ b/src/python/pants/_version/VERSION @@ -0,0 +1 @@ +../VERSION \ No newline at end of file diff --git a/src/python/pants/_version/__init__.py b/src/python/pants/_version/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/codegen/protobuf/scala/rules.py b/src/python/pants/backend/codegen/protobuf/scala/rules.py index 8591fa40c1c..65058f8b1b1 100644 --- a/src/python/pants/backend/codegen/protobuf/scala/rules.py +++ b/src/python/pants/backend/codegen/protobuf/scala/rules.py @@ -3,7 +3,6 @@ from __future__ import annotations import os -import pkgutil from dataclasses import dataclass from pants.backend.codegen import export_codegen_goal @@ -55,6 +54,7 @@ from pants.source.source_root import SourceRoot, SourceRootRequest from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource class GenerateScalaFromProtobufRequest(GenerateSourcesRequest): @@ -245,7 +245,7 @@ async def setup_scalapb_shim_classfiles( ) -> ScalaPBShimCompiledClassfiles: dest_dir = "classfiles" - scalapb_shim_content = pkgutil.get_data( + scalapb_shim_content = read_resource( "pants.backend.codegen.protobuf.scala", "ScalaPBShim.scala" ) if not scalapb_shim_content: diff --git a/src/python/pants/backend/docker/subsystems/dockerfile_parser.py b/src/python/pants/backend/docker/subsystems/dockerfile_parser.py index e5fecdef6ab..0b3f82acbab 100644 --- a/src/python/pants/backend/docker/subsystems/dockerfile_parser.py +++ b/src/python/pants/backend/docker/subsystems/dockerfile_parser.py @@ -4,7 +4,6 @@ from __future__ import annotations import json -import pkgutil from dataclasses import dataclass from pathlib import PurePath @@ -35,6 +34,7 @@ from pants.engine.unions import UnionRule from pants.util.docutil import git_url from pants.util.logging import LogLevel +from pants.util.resources import read_resource _DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py" _DOCKERFILE_PACKAGE = "pants.backend.docker.subsystems" @@ -77,7 +77,7 @@ class ParserSetup: @rule async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup: - parser_script_content = pkgutil.get_data(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL) + parser_script_content = read_resource(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL) if not parser_script_content: raise ValueError( f"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}." diff --git a/src/python/pants/backend/go/go_sources/load_go_binary.py b/src/python/pants/backend/go/go_sources/load_go_binary.py index 68c65369bd0..ca84f78e742 100644 --- a/src/python/pants/backend/go/go_sources/load_go_binary.py +++ b/src/python/pants/backend/go/go_sources/load_go_binary.py @@ -3,7 +3,6 @@ from __future__ import annotations -import pkgutil from dataclasses import dataclass from pants.backend.go.util_rules.build_pkg import BuildGoPackageRequest, BuiltGoPackage @@ -12,6 +11,7 @@ from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.util.resources import read_resource @dataclass(frozen=True) @@ -31,7 +31,7 @@ def debug_hint(self) -> str: def setup_files(dir_name: str, file_names: tuple[str, ...]) -> tuple[FileContent, ...]: def get_file(file_name: str) -> bytes: - content = pkgutil.get_data(f"pants.backend.go.go_sources.{dir_name}", file_name) + content = read_resource(f"pants.backend.go.go_sources.{dir_name}", file_name) if not content: raise AssertionError(f"Unable to find resource for `{file_name}`.") return content diff --git a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py index 9dbb7f1fb23..c5107122692 100644 --- a/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py +++ b/src/python/pants/backend/kotlin/dependency_inference/kotlin_parser.py @@ -4,7 +4,6 @@ import json import os -import pkgutil from dataclasses import dataclass from typing import Any, Iterator @@ -25,6 +24,7 @@ from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource _PARSER_KOTLIN_VERSION = "1.6.20" @@ -230,7 +230,7 @@ async def resolve_fallible_result_to_analysis( async def setup_kotlin_parser_classfiles(jdk: InternalJdk) -> KotlinParserCompiledClassfiles: dest_dir = "classfiles" - parser_source_content = pkgutil.get_data( + parser_source_content = read_resource( "pants.backend.kotlin.dependency_inference", "KotlinParser.kt" ) if not parser_source_content: diff --git a/src/python/pants/backend/python/dependency_inference/BUILD b/src/python/pants/backend/python/dependency_inference/BUILD index 0290569f3ec..a7935536dbc 100644 --- a/src/python/pants/backend/python/dependency_inference/BUILD +++ b/src/python/pants/backend/python/dependency_inference/BUILD @@ -1,7 +1,11 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -python_sources() +python_sources( + dependencies=[ + "./scripts:dependency_parser", + ], +) python_tests( name="tests", diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py index 183ded7992f..96b897397b9 100644 --- a/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py +++ b/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import json -import pkgutil from dataclasses import dataclass from pants.backend.python.target_types import PythonSourceField @@ -16,6 +15,7 @@ from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel +from pants.util.resources import read_resource @dataclass(frozen=True) @@ -61,7 +61,7 @@ class ParserScript: @rule async def parser_script() -> ParserScript: - script = pkgutil.get_data(__name__, "scripts/dependency_parser.py") + script = read_resource(__name__, "scripts/dependency_parser_py") assert script is not None return ParserScript( await Get(Digest, CreateDigest([FileContent("__parse_python_dependencies.py", script)])) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/BUILD b/src/python/pants/backend/python/dependency_inference/scripts/BUILD index e142aded947..41c5d4acb49 100644 --- a/src/python/pants/backend/python/dependency_inference/scripts/BUILD +++ b/src/python/pants/backend/python/dependency_inference/scripts/BUILD @@ -1,11 +1,22 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -resource(name="dependency_parser", source="dependency_parser.py") +resource( + name="dependency_parser", + source="dependency_parser_py", + dependencies=[ + ":init_py", + ], +) +resource( + name="init_py", + source="__init__.py", +) + # Also expose scripts as python sources so they get formatted/linted/checked. python_source( name="dependency_parser_source", - source="dependency_parser.py", + source="dependency_parser_py", # This is run with Python 2.7 and 3.5+, so we shouldn't be running pyupgrade. # skip_pyupgrade=True, ) diff --git a/src/python/pants/backend/python/dependency_inference/scripts/__init__.py b/src/python/pants/backend/python/dependency_inference/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/python/dependency_inference/scripts/dependency_parser.py b/src/python/pants/backend/python/dependency_inference/scripts/dependency_parser_py similarity index 100% rename from src/python/pants/backend/python/dependency_inference/scripts/dependency_parser.py rename to src/python/pants/backend/python/dependency_inference/scripts/dependency_parser_py diff --git a/src/python/pants/backend/scala/dependency_inference/scala_parser.py b/src/python/pants/backend/scala/dependency_inference/scala_parser.py index f119a6f78fa..09107b82813 100644 --- a/src/python/pants/backend/scala/dependency_inference/scala_parser.py +++ b/src/python/pants/backend/scala/dependency_inference/scala_parser.py @@ -5,7 +5,6 @@ import json import logging import os -import pkgutil from dataclasses import dataclass from typing import Any, Iterator, Mapping @@ -39,6 +38,7 @@ from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet +from pants.util.resources import read_resource logger = logging.getLogger(__name__) @@ -316,7 +316,7 @@ async def resolve_fallible_result_to_analysis( async def setup_scala_parser_classfiles(jdk: InternalJdk) -> ScalaParserCompiledClassfiles: dest_dir = "classfiles" - parser_source_content = pkgutil.get_data( + parser_source_content = read_resource( "pants.backend.scala.dependency_inference", "ScalaParser.scala" ) if not parser_source_content: diff --git a/src/python/pants/backend/terraform/dependency_inference.py b/src/python/pants/backend/terraform/dependency_inference.py index 49dd76c0d7c..c6d7a015c9d 100644 --- a/src/python/pants/backend/terraform/dependency_inference.py +++ b/src/python/pants/backend/terraform/dependency_inference.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations -import pkgutil from dataclasses import dataclass from pathlib import PurePath @@ -36,6 +35,7 @@ from pants.util.docutil import git_url from pants.util.logging import LogLevel from pants.util.ordered_set import OrderedSet +from pants.util.resources import read_resource class TerraformHcl2Parser(PythonToolRequirementsBase): @@ -75,7 +75,7 @@ class ParserSetup: @rule async def setup_parser(hcl2_parser: TerraformHcl2Parser) -> ParserSetup: - parser_script_content = pkgutil.get_data("pants.backend.terraform", "hcl2_parser.py") + parser_script_content = read_resource("pants.backend.terraform", "hcl2_parser.py") if not parser_script_content: raise ValueError("Unable to find source to hcl2_parser.py wrapper script.") diff --git a/src/python/pants/util/resources.py b/src/python/pants/util/resources.py new file mode 100644 index 00000000000..761e02aadde --- /dev/null +++ b/src/python/pants/util/resources.py @@ -0,0 +1,36 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +import importlib +from importlib import resources +from itertools import chain + + +def read_resource(package_or_module: str, resource: str) -> bytes: + """Reads a resource file from within the Pants package itself. + + This helper function is designed for compatibility with `pkgutil.get_data()` wherever possible, + but also allows compability with PEP302 pluggable importers such as included with PyOxidizer. + This requires that resources are loaded from a valid Python package (i.e. must have an + `__init__.py` file in the directory). + """ + + a = importlib.import_module(package_or_module) + package_ = a.__package__ + + if package_ is None: + raise ValueError( + "`read_resource` can only help find resources for packages or modules that live in " + "a package." + ) + + resource_parts = resource.split("/") + + if len(resource_parts) == 1: + package = package_ + else: + package = ".".join(chain((package_,), resource_parts[:-1])) + resource = resource_parts[-1] + + return resources.read_binary(package, resource) diff --git a/src/python/pants/version.py b/src/python/pants/version.py index adee488c8b5..c896838311c 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -2,10 +2,13 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os -import pkgutil from packaging.version import Version +# Generate a inferrable dependency on the `pants._version` package and its associated resources. +import pants._version +from pants.util.resources import read_resource + # Set this env var to override the version pants reports. Useful for testing. _PANTS_VERSION_OVERRIDE = "_PANTS_VERSION_OVERRIDE" @@ -14,7 +17,10 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - pkgutil.get_data(__name__, "VERSION").decode().strip() # type: ignore[union-attr] + # `pants._version` is a non-namespace package that can be loaded by `importlib.resources` in + # pre-3.11 versions of Python. Once we can read `pants.VERSION` using the resource loader, + # we can remove the `pants._version` package. See #16379. + read_resource(pants._version.__name__, "VERSION").decode().strip() ) PANTS_SEMVER = Version(VERSION) From 1e77a5246572f881ce088a35927ce6a209588817 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 11 Aug 2022 14:22:46 -0700 Subject: [PATCH 33/45] Approaches the `read_resource` api with an approach that doesn't require a symlink (see #13679) # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/__init__.py | 6 +++++ src/python/pants/_version/BUILD | 12 --------- src/python/pants/_version/VERSION | 1 - src/python/pants/_version/__init__.py | 0 src/python/pants/conftest.py | 37 ++------------------------- src/python/pants/version.py | 7 ++--- 6 files changed, 10 insertions(+), 53 deletions(-) create mode 100644 src/python/pants/__init__.py delete mode 100644 src/python/pants/_version/BUILD delete mode 120000 src/python/pants/_version/VERSION delete mode 100644 src/python/pants/_version/__init__.py diff --git a/src/python/pants/__init__.py b/src/python/pants/__init__.py new file mode 100644 index 00000000000..f7dbf514547 --- /dev/null +++ b/src/python/pants/__init__.py @@ -0,0 +1,6 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/src/python/pants/_version/BUILD b/src/python/pants/_version/BUILD deleted file mode 100644 index dd4c37fe926..00000000000 --- a/src/python/pants/_version/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -# This package is used to provide a non-namespaced Python package from which -# `importlib.resources.read_binary` can read our `VERSION` file. The `VERSION` file is a -# symlink to the concrete `VERSION` file in `src/python/pants`. This creates a minimal package -# that can be imported by `pants.version` during tests, and inferred by Pants as a dependency. -# Future versions of `importlib.resources` will be able to read resources from namespace -# packages, at which point, this package will no longer need to exist. - -python_sources(dependencies=["./VERSION:resources"]) -resources(name="resources", sources=["VERSION"]) diff --git a/src/python/pants/_version/VERSION b/src/python/pants/_version/VERSION deleted file mode 120000 index 6ff19de4b80..00000000000 --- a/src/python/pants/_version/VERSION +++ /dev/null @@ -1 +0,0 @@ -../VERSION \ No newline at end of file diff --git a/src/python/pants/_version/__init__.py b/src/python/pants/_version/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/python/pants/conftest.py b/src/python/pants/conftest.py index 192d83e3787..a63ece2644c 100644 --- a/src/python/pants/conftest.py +++ b/src/python/pants/conftest.py @@ -1,49 +1,16 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pathlib import Path import pytest -# The top-level `pants` module must be a namespace package, because we build two dists from it -# (pantsbuild.pants, and pantsbuild.pants.testutil) and consumers of these dists need to be -# able to import from both. -# -# In fact it is an *implicit* namespace package - that is, it has no __init__.py file. -# See https://packaging.python.org/guides/packaging-namespace-packages/ . -# -# Unfortunately, the presence or absence of __init__.py affects how pytest determines the -# module names for test code. For details see -# https://docs.pytest.org/en/stable/goodpractices.html#test-package-name . -# -# Usually this doesn't matter, as tests don't typically care about their own module name. -# But we have tests (notably those in src/python/pants/engine) that create @rules and -# expect them to have certain names. And @rule names are generated from the name of the module -# containing the rule function... -# -# To allow those tests to work naturally (with expected module names relative to `src/python`) -# we artificially create `src/python/pants/__init__.py` in the test sandbox, to force -# pytest to determine module names relative to `src/python` (instead of `src/python/pants`). -# -# Note that while this makes the (implicit) namespace package into a regular package, -# that is fine at test time. We don't consume testutil from a dist but from source, in the same -# source root (src/python). We only need `pants` to be a namespace package in the dists we create. -namespace_init_path = Path("src/python/pants/__init__.py") - def pytest_sessionstart(session) -> None: - if namespace_init_path.exists(): - raise Exception( - f"In order for `pants` to be a namespace package, {namespace_init_path} must not " - f"exist on disk. See the explanation in {__file__}." - ) - namespace_init_path.touch() + pass def pytest_sessionfinish(session) -> None: - # Technically unecessary, but nice if people are running tests directly from repo - # (not using pants). - namespace_init_path.unlink() + pass @pytest.fixture(autouse=True, scope="session") diff --git a/src/python/pants/version.py b/src/python/pants/version.py index c896838311c..6805de0d0d1 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -6,7 +6,7 @@ from packaging.version import Version # Generate a inferrable dependency on the `pants._version` package and its associated resources. -import pants._version +import pants from pants.util.resources import read_resource # Set this env var to override the version pants reports. Useful for testing. @@ -17,10 +17,7 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - # `pants._version` is a non-namespace package that can be loaded by `importlib.resources` in - # pre-3.11 versions of Python. Once we can read `pants.VERSION` using the resource loader, - # we can remove the `pants._version` package. See #16379. - read_resource(pants._version.__name__, "VERSION").decode().strip() + read_resource(pants.__name__, "VERSION").decode().strip() ) PANTS_SEMVER = Version(VERSION) From 1f76dec9569229ecec1dc2bfc00c20d06d9fa3cd Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 11 Aug 2022 14:23:53 -0700 Subject: [PATCH 34/45] Remove Welcome To Pants log # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/bin/pants_loader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 03b3209f0fa..1c4b77c05ed 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -120,5 +120,4 @@ def main() -> None: if __name__ == "__main__": - print(f"welcome to pants! {sys.argv=} {sys.executable=} {os.getcwd()=}", file=sys.stderr) main() From 36453fc0f06bfbda18d3f74e4d3e3d1c119fd87a Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 11 Aug 2022 14:24:26 -0700 Subject: [PATCH 35/45] Lets Pants start up with no arguments # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/ox.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index ed6ed0a4524..61086bad82e 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -40,6 +40,9 @@ def bootstrap_pyoxidizer() -> None: def pex_main() -> bool: + if len(sys.argv) < 2: + return False + if sys.argv[1] == "./pex": run_as_pex() return True From 78a840fc089dfd9c9e63ef3bb25f5a5570fecfce Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 11 Aug 2022 14:25:22 -0700 Subject: [PATCH 36/45] Remove extraneous logging # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/init/plugin_resolver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python/pants/init/plugin_resolver.py b/src/python/pants/init/plugin_resolver.py index 54023e676b3..66a04c72ad8 100644 --- a/src/python/pants/init/plugin_resolver.py +++ b/src/python/pants/init/plugin_resolver.py @@ -134,7 +134,6 @@ def resolve( for resolved_plugin_location in self._resolve_plugins( options_bootstrapper, env, self._request ): - logger.warning(f"{resolved_plugin_location=}") site.addsitedir( resolved_plugin_location ) # Activate any .pth files plugin wheels may have. From 684ae677fd4eea852226fb63b7c780f2641a95d5 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 11 Aug 2022 17:28:03 -0700 Subject: [PATCH 37/45] Swap from `importlib.resources` to `importlib_resources` backport # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] (cherry picked from commit 80dd94d66960af1087a260d2860db57c359986fd) --- 3rdparty/python/requirements.txt | 1 + 3rdparty/python/user_reqs.lock | 125 ++++++++++++++++++----------- src/python/pants/util/resources.py | 3 +- 3 files changed, 82 insertions(+), 47 deletions(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 5307da630d7..ee05e23c3d3 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -13,6 +13,7 @@ freezegun==1.2.1 # anonymity promise we make here: https://www.pantsbuild.org/docs/anonymous-telemetry humbug==0.2.7 +importlib_resources==5.0.* ijson==3.1.4 packaging==21.3 pex==2.1.103 diff --git a/3rdparty/python/user_reqs.lock b/3rdparty/python/user_reqs.lock index 5fd5e95cefc..47f25a96991 100644 --- a/3rdparty/python/user_reqs.lock +++ b/3rdparty/python/user_reqs.lock @@ -4,7 +4,7 @@ // // --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- // { -// "version": 2, +// "version": 3, // "valid_for_interpreter_constraints": [ // "CPython<3.10,>=3.7" // ], @@ -19,6 +19,7 @@ // "freezegun==1.2.1", // "humbug==0.2.7", // "ijson==3.1.4", +// "importlib_resources==5.0.*", // "mypy-typing-asserts==0.1.1", // "packaging==21.3", // "pex==2.1.103", @@ -40,7 +41,8 @@ // "types-toml==0.10.8", // "typing-extensions==4.3.0", // "uvicorn[standard]==0.17.6" -// ] +// ], +// "requirement_constraints": [] // } // --- END PANTS LOCKFILE METADATA --- @@ -793,6 +795,36 @@ "requires_python": ">=3.7", "version": "4.12" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "2238159eb743bd85304a16e0536048b3e991c531d1cd51c4a834d1ccf2829057", + "url": "https://files.pythonhosted.org/packages/46/10/7cc167fe072037c3cd2a15a92bb963b86f2bab8ac0995fab95fb7a152b80/importlib_resources-5.0.7-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "4df460394562b4581bb4e4087ad9447bd433148fba44241754ec3152499f1d1b", + "url": "https://files.pythonhosted.org/packages/4a/d5/22aa0454c06788e59f406a2b0e569fac835c6c45e5ad6ed968804920f0ac/importlib_resources-5.0.7.tar.gz" + } + ], + "project_name": "importlib-resources", + "requires_dists": [ + "jaraco.packaging>=8.2; extra == \"docs\"", + "pytest!=3.7.3,>=3.5; extra == \"testing\"", + "pytest-black>=0.3.7; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "pytest-checkdocs>=1.2.3; extra == \"testing\"", + "pytest-cov; extra == \"testing\"", + "pytest-enabler; extra == \"testing\"", + "pytest-flake8; extra == \"testing\"", + "pytest-mypy; platform_python_implementation != \"PyPy\" and extra == \"testing\"", + "rst.linker>=1.9; extra == \"docs\"", + "sphinx; extra == \"docs\"", + "zipp>=0.4; python_version < \"3.8\"" + ], + "requires_python": ">=3.6", + "version": "5.0.7" + }, { "artifacts": [ { @@ -980,98 +1012,98 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "4988c0f13c42bfa9ddd2fe2f569c9d54646ce84adc5de84228cfe83396f3bd58", - "url": "https://files.pythonhosted.org/packages/f3/88/78666bfe38d3a8aee75fbd2410ac6e26dfdd64585323c07648f387817c76/pydantic-1.9.1-py3-none-any.whl" + "hash": "78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e", + "url": "https://files.pythonhosted.org/packages/fe/27/0de772dcd0517770b265dbc3998ed3ee3aa2ba25ba67e3685116cbbbccc6/pydantic-1.9.2-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "72ccb318bf0c9ab97fc04c10c37683d9eea952ed526707fabf9ac5ae59b701fd", - "url": "https://files.pythonhosted.org/packages/2e/6b/7ae3031fc86b974296ef3a91221aec46fdf66a8dd6ba1d300151a812c5b3/pydantic-1.9.1-cp39-cp39-musllinux_1_1_x86_64.whl" + "hash": "7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb", + "url": "https://files.pythonhosted.org/packages/04/bc/5231387df42b199f38dd3f29eb10338bc0a272e24020aff5c4cd64d3270d/pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "02eefd7087268b711a3ff4db528e9916ac9aa18616da7bca69c1871d0b7a091f", - "url": "https://files.pythonhosted.org/packages/5f/76/11635fe2d808c0062f4f23a6ac6453e6110d3446fb196e3b6dfc1d15f979/pydantic-1.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8", + "url": "https://files.pythonhosted.org/packages/18/2f/228fe5d1dbf7c36bd252fb304b015d02a50f696e659a0bb370a5628d00f4/pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "494f7c8537f0c02b740c229af4cb47c0d39840b829ecdcfc93d91dcbb0779892", - "url": "https://files.pythonhosted.org/packages/71/54/de851f0ab24e6044be1df7186ba000defefe36945f10b86f70f6b4300284/pydantic-1.9.1-cp37-cp37m-macosx_10_9_x86_64.whl" + "hash": "f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801", + "url": "https://files.pythonhosted.org/packages/6f/1e/4dca34af2a7e8effb5226ac2fec3664e99c8e95c97e8ebae9ff47fb3bbef/pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "9bcf8b6e011be08fb729d110f3e22e654a50f8a826b0575c7196616780683380", - "url": "https://files.pythonhosted.org/packages/80/30/553e846151cd7dbd2405fb128272c6cea5ea3e42f9538e2962838dcf755b/pydantic-1.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl" + "hash": "a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f", + "url": "https://files.pythonhosted.org/packages/75/44/e3c3c72ddbf7f6c987e39cc09f21f61f21cffeebddb75b9019f952624942/pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "4ce9ae9e91f46c344bec3b03d6ee9612802682c1551aaf627ad24045ce090761", - "url": "https://files.pythonhosted.org/packages/81/48/b3c8cb4eb7106b612bbfc263d557b75e27d2720a0cf2c442ea67950d6d43/pydantic-1.9.1-cp39-cp39-musllinux_1_1_i686.whl" + "hash": "e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa", + "url": "https://files.pythonhosted.org/packages/84/cf/b2514b857196fb8484209c6bf365a164b684f6eef3d1feaa4f9ce2447389/pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "a9af62e9b5b9bc67b2a195ebc2c2662fdf498a822d62f902bf27cccb52dbbf49", - "url": "https://files.pythonhosted.org/packages/8e/d8/56552e89a644b77e51fd8c2c6cc4982ec83b5f6221c10c00bf1406255fec/pydantic-1.9.1-cp38-cp38-musllinux_1_1_i686.whl" + "hash": "91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3", + "url": "https://files.pythonhosted.org/packages/86/f8/c2effc693180e16b3ec886bc9d080f937afa7964823a7c204d5c9df55264/pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "9ce157d979f742a915b75f792dbd6aa63b8eccaf46a1005ba03aa8a986bde34a", - "url": "https://files.pythonhosted.org/packages/97/7b/229158bcd271f02ec2e92c6740ce47236797fcef2d36cbe949c0ea9f3a68/pydantic-1.9.1-cp38-cp38-macosx_10_9_x86_64.whl" + "hash": "e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747", + "url": "https://files.pythonhosted.org/packages/88/83/42a71762ec2f127ba8141a0608dea0ee2a8aa2dd6fcc0d2cda375aee61eb/pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "b83ba3825bc91dfa989d4eed76865e71aea3a6ca1388b59fc801ee04c4d8d0d6", - "url": "https://files.pythonhosted.org/packages/9a/a2/585b1a747b7bbb22392fe9c875b8adfaaee9a7be506fbb66a749157f9099/pydantic-1.9.1-cp39-cp39-macosx_10_9_x86_64.whl" + "hash": "4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56", + "url": "https://files.pythonhosted.org/packages/95/5b/5d1d8d5e6e2d9a1ec3a94b75b14fe5a2e6efd13fa96a3e53144db9de9d48/pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "969dd06110cb780da01336b281f53e2e7eb3a482831df441fb65dd30403f4608", - "url": "https://files.pythonhosted.org/packages/b9/01/fad5d5a724780873e0fac2b1ca9365636179921fc371ac6a7ec6b9706d8e/pydantic-1.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4", + "url": "https://files.pythonhosted.org/packages/a0/cb/672a6e3a9fa78c9a21f274dbdef7f20633969527f07ac8f882263844f4c1/pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "0bf07cab5b279859c253d26a9194a8906e6f4a210063b84b433cf90a569de0c1", - "url": "https://files.pythonhosted.org/packages/c1/3b/e3f31676b694ab293ba4b7c5a3146fa8ec72c7a4d71c69a883aad1210517/pydantic-1.9.1-cp38-cp38-macosx_11_0_arm64.whl" + "hash": "1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44", + "url": "https://files.pythonhosted.org/packages/bb/9c/7ded003135342ea07fcac5581790634a2d70340175c1e7cb2f0affcb1962/pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "7eb57ba90929bac0b6cc2af2373893d80ac559adda6933e562dcfb375029acee", - "url": "https://files.pythonhosted.org/packages/c4/d2/6118efdb9fdaf3d4dfecc0276d4d47a1ef9aaf9903fa49f1ece765d917cb/pydantic-1.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0", + "url": "https://files.pythonhosted.org/packages/be/72/841dcb62c23d8955b82784dd3bb73770d1ce8aa562e5bd47c1f52230ca12/pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "fe4670cb32ea98ffbf5a1262f14c3e102cccd92b1869df3bb09538158ba90fe6", - "url": "https://files.pythonhosted.org/packages/c7/26/f5d5c0e9d3bca2c720a70b3280299dd76861b1938862cd4425d7d7e88396/pydantic-1.9.1-cp38-cp38-musllinux_1_1_x86_64.whl" + "hash": "4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8", + "url": "https://files.pythonhosted.org/packages/d3/4b/6f539c1f26c6a8ed942fa751981909ab86336ce5ead28b6c92590ee6bc1b/pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "1ed987c3ff29fff7fd8c3ea3a3ea877ad310aae2ef9889a119e22d3f2db0691a", - "url": "https://files.pythonhosted.org/packages/d0/a5/e4a25a0becf35530a3d90459a88855743e942f2e502da49ca5b10aa78568/pydantic-1.9.1.tar.gz" + "hash": "cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979", + "url": "https://files.pythonhosted.org/packages/e0/0f/a8adcc49e58994f6da6b96dac42dedbedd250c3130d59a664d8130c8019d/pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl" }, { "algorithm": "sha256", - "hash": "5d93d4e95eacd313d2c765ebe40d49ca9dd2ed90e5b37d0d421c597af830c195", - "url": "https://files.pythonhosted.org/packages/de/0a/bd6eb16c07eba6aad5e543985995e2cbdc4925ccffc2f9491482f9101c0e/pydantic-1.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453", + "url": "https://files.pythonhosted.org/packages/e9/64/3395d45a05adcebb6d1025702c28d1ed188703397f38999295c52687f87e/pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "1542636a39c4892c4f4fa6270696902acb186a9aaeac6f6cf92ce6ae2e88564b", - "url": "https://files.pythonhosted.org/packages/df/c9/7ea0a065d96a200f008021f28149f16b6d99c30d04461c562061d6a73ce8/pydantic-1.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d", + "url": "https://files.pythonhosted.org/packages/ed/c9/ffe44727dadb0930783a1ffb60facf8ead7dffbb67db9ae2fa28dacabcf1/pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "177071dfc0df6248fd22b43036f936cfe2508077a72af0933d0c1fa269b18537", - "url": "https://files.pythonhosted.org/packages/ea/3a/6b451f0ddb374e6a23642554ee3a357029d718083e084b3c9e6ea9b8119a/pydantic-1.9.1-cp37-cp37m-musllinux_1_1_i686.whl" + "hash": "4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3", + "url": "https://files.pythonhosted.org/packages/f6/63/b412252dbbdc712500ad73fe2e591c3220781e63a8c135d26b7d60fcb99c/pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "1dd8fecbad028cd89d04a46688d2fcc14423e8a196d5b0a5c65105664901f810", - "url": "https://files.pythonhosted.org/packages/ed/30/cc6081090e0653b8bfeac45b5973027771050813c4ac167a277f4f355242/pydantic-1.9.1-cp39-cp39-macosx_11_0_arm64.whl" + "hash": "7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15", + "url": "https://files.pythonhosted.org/packages/f7/76/4a98738c31e520c78a80e9575b655b5c3ae96313102478bff4d643abc2e9/pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "f0f047e11febe5c3198ed346b507e1d010330d56ad615a7e0a89fae604065a0e", - "url": "https://files.pythonhosted.org/packages/f1/ea/1b879b2b30c4e041bce20f3baf785a640f93ae4146d5258dff5661d0bebf/pydantic-1.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d", + "url": "https://files.pythonhosted.org/packages/fd/8f/3f7e88b507dbdfec8f1f914294aa8831edffb03d668799c65b4b46331c8a/pydantic-1.9.2.tar.gz" } ], "project_name": "pydantic", @@ -1082,7 +1114,7 @@ "typing-extensions>=3.7.4.3" ], "requires_python": ">=3.6.1", - "version": "1.9.1" + "version": "1.9.2" }, { "artifacts": [ @@ -1437,13 +1469,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "dc2662692f47d99cb8ae15a784529adeed535bcd7c277fee0beccf961522baf6", - "url": "https://files.pythonhosted.org/packages/90/2e/109766d7f3cb854a083dd66bfc0bf2bf9e40f373829efedb4b5e0d104aa3/setuptools-63.4.1-py3-none-any.whl" + "hash": "7f61f7e82647f77d4118eeaf43d64cbcd4d87e38af9611694d4866eb070cd10d", + "url": "https://files.pythonhosted.org/packages/2a/a3/49c29680d6118273b992b40ebe881e8e899b8e26a4e951f37f223da8f862/setuptools-63.4.3-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "7c7854ee1429a240090297628dc9f75b35318d193537968e2dc14010ee2f5bca", - "url": "https://files.pythonhosted.org/packages/63/2e/e1f3e1b02d7ff0baa356413e93ad8a88706accd6b3538e18204a8a00c42d/setuptools-63.4.1.tar.gz" + "hash": "521c833d1e5e1ef0869940e7f486a83de7773b9f029010ad0c2fe35453a9dad9", + "url": "https://files.pythonhosted.org/packages/5b/ff/69fd395c5237da934753752b71c38e95e137bd0603d5640df70ddaea8038/setuptools-63.4.3.tar.gz" } ], "project_name": "setuptools", @@ -1494,7 +1526,7 @@ "wheel; extra == \"testing-integration\"" ], "requires_python": ">=3.7", - "version": "63.4.1" + "version": "63.4.3" }, { "artifacts": [ @@ -2326,6 +2358,7 @@ "freezegun==1.2.1", "humbug==0.2.7", "ijson==3.1.4", + "importlib_resources==5.0.*", "mypy-typing-asserts==0.1.1", "packaging==21.3", "pex==2.1.103", diff --git a/src/python/pants/util/resources.py b/src/python/pants/util/resources.py index 761e02aadde..f56fe618f16 100644 --- a/src/python/pants/util/resources.py +++ b/src/python/pants/util/resources.py @@ -3,9 +3,10 @@ import importlib -from importlib import resources from itertools import chain +import importlib_resources as resources + def read_resource(package_or_module: str, resource: str) -> bytes: """Reads a resource file from within the Pants package itself. From c51b2a00bc07079a15888e39b1e5f43e84af62d2 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 12 Aug 2022 09:41:23 -0700 Subject: [PATCH 38/45] See if things work without the `__init__.py` in place # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] (cherry picked from commit 438ade88f42212a162e5e50d0544ea60ff50ab18) --- src/python/pants/__init__.py | 6 ------ src/python/pants/version.py | 3 +-- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 src/python/pants/__init__.py diff --git a/src/python/pants/__init__.py b/src/python/pants/__init__.py deleted file mode 100644 index f7dbf514547..00000000000 --- a/src/python/pants/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/src/python/pants/version.py b/src/python/pants/version.py index 6805de0d0d1..57569a30faf 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -6,7 +6,6 @@ from packaging.version import Version # Generate a inferrable dependency on the `pants._version` package and its associated resources. -import pants from pants.util.resources import read_resource # Set this env var to override the version pants reports. Useful for testing. @@ -17,7 +16,7 @@ os.environ.get(_PANTS_VERSION_OVERRIDE) or # NB: We expect VERSION to always have an entry and want a runtime failure if this is false. - read_resource(pants.__name__, "VERSION").decode().strip() + read_resource(__name__, "VERSION").decode().strip() ) PANTS_SEMVER = Version(VERSION) From 087e108289d6fa516fe6aaab64bb6b7ae8745733 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 12 Aug 2022 10:51:26 -0700 Subject: [PATCH 39/45] WHEEEE # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- src/python/pants/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/pants/BUILD b/src/python/pants/BUILD index fe6ee11f410..f075cd0ea59 100644 --- a/src/python/pants/BUILD +++ b/src/python/pants/BUILD @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). python_sources( + dependencies=[":resources"], overrides={ # Enable `python -m pants ...` style execution ala `json.tool` or `venv`. "__main__.py": {"dependencies": ["src/python/pants/bin:pants_loader"]}, From dc51dec9db3f69189f72f1d37de977ec3b99324f Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 25 Aug 2022 14:18:26 -0700 Subject: [PATCH 40/45] update lockfile # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- 3rdparty/python/user_reqs.lock | 38 +++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/3rdparty/python/user_reqs.lock b/3rdparty/python/user_reqs.lock index 76100862369..ecf5aa005b4 100644 --- a/3rdparty/python/user_reqs.lock +++ b/3rdparty/python/user_reqs.lock @@ -42,7 +42,10 @@ // "typing-extensions==4.3.0", // "uvicorn[standard]==0.17.6" // ], -// "requirement_constraints": [] +// "manylinux": "manylinux2014", +// "requirement_constraints": [], +// "only_binary": [], +// "no_binary": [] // } // --- END PANTS LOCKFILE METADATA --- @@ -247,13 +250,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", - "url": "https://files.pythonhosted.org/packages/94/69/64b11e8c2fb21f08634468caef885112e682b0ebe2908e74d3616eb1c113/charset_normalizer-2.1.0-py3-none-any.whl" + "hash": "83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f", + "url": "https://files.pythonhosted.org/packages/db/51/a507c856293ab05cdc1db77ff4bc1268ddd39f29e7dc4919aa497f0adbec/charset_normalizer-2.1.1-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413", - "url": "https://files.pythonhosted.org/packages/93/1d/d9392056df6670ae2a29fcb04cfa5cee9f6fbde7311a1bb511d4115e9b7a/charset-normalizer-2.1.0.tar.gz" + "hash": "5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", + "url": "https://files.pythonhosted.org/packages/a1/34/44964211e5410b051e4b8d2869c470ae8a68ae274953b1c7de6d98bbcf94/charset-normalizer-2.1.1.tar.gz" } ], "project_name": "charset-normalizer", @@ -261,7 +264,7 @@ "unicodedata2; extra == \"unicode_backport\"" ], "requires_python": ">=3.6.0", - "version": "2.1" + "version": "2.1.1" }, { "artifacts": [ @@ -1785,19 +1788,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "09a8783e1002472e8d1e1f3792d4c5cca1fffebb9b48ee1512aae6d16fe186bc", - "url": "https://files.pythonhosted.org/packages/19/8c/b9f75c5b52f79402baeabbb065067f72e922f96a114fe471ce4069b0cb69/types_urllib3-1.26.22-py3-none-any.whl" + "hash": "333e675b188a1c1fd980b4b352f9e40572413a4c1ac689c23cd546e96310070a", + "url": "https://files.pythonhosted.org/packages/2f/e5/988bff1d30df1f32d402125fad52235c3214c6827c2b0b3a658ee93a921a/types_urllib3-1.26.23-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "b05af90e73889e688094008a97ca95788db8bf3736e2776fd43fb6b171485d94", - "url": "https://files.pythonhosted.org/packages/d2/d8/764ca957d9601d956a87a311b107c9b6171ed5bb0c269e8ae60d91fc0fbf/types-urllib3-1.26.22.tar.gz" + "hash": "b78e819f0e350221d0689a5666162e467ba3910737bafda14b5c2c85e9bb1e56", + "url": "https://files.pythonhosted.org/packages/4a/21/ddf7064ff4d10849ded229b2dabc9bfc5f19a7d4531ebf5c3b431267e603/types-urllib3-1.26.23.tar.gz" } ], "project_name": "types-urllib3", "requires_dists": [], "requires_python": null, - "version": "1.26.22" + "version": "1.26.23" }, { "artifacts": [ @@ -1989,13 +1992,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", - "url": "https://files.pythonhosted.org/packages/d1/cb/4783c8f1a90f89e260dbf72ebbcf25931f3a28f8f80e2e90f8a589941b19/urllib3-1.26.11-py2.py3-none-any.whl" + "hash": "b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997", + "url": "https://files.pythonhosted.org/packages/6f/de/5be2e3eed8426f871b170663333a0f627fc2924cc386cd41be065e7ea870/urllib3-1.26.12-py2.py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a", - "url": "https://files.pythonhosted.org/packages/6d/d5/e8258b334c9eb8eb78e31be92ea0d5da83ddd9385dc967dd92737604d239/urllib3-1.26.11.tar.gz" + "hash": "3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", + "url": "https://files.pythonhosted.org/packages/b2/56/d87d6d3c4121c0bcec116919350ca05dc3afd2eeb7dc88d07e8083f8ea94/urllib3-1.26.12.tar.gz" } ], "project_name": "urllib3", @@ -2008,10 +2011,11 @@ "cryptography>=1.3.4; extra == \"secure\"", "idna>=2.0.0; extra == \"secure\"", "ipaddress; python_version == \"2.7\" and extra == \"secure\"", - "pyOpenSSL>=0.14; extra == \"secure\"" + "pyOpenSSL>=0.14; extra == \"secure\"", + "urllib3-secure-extra; extra == \"secure\"" ], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,<4,>=2.7", - "version": "1.26.11" + "version": "1.26.12" }, { "artifacts": [ From fd1952454a21223ad757a4f190894c572c725323 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 25 Aug 2022 14:20:26 -0700 Subject: [PATCH 41/45] Load backend modules with standard Python import machinery # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/init/extension_loader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python/pants/init/extension_loader.py b/src/python/pants/init/extension_loader.py index 2fecae4ba86..720ece781c9 100644 --- a/src/python/pants/init/extension_loader.py +++ b/src/python/pants/init/extension_loader.py @@ -8,6 +8,7 @@ from pkg_resources import Requirement, WorkingSet +from pants import ox from pants.base.exceptions import BackendConfigurationError from pants.build_graph.build_configuration import BuildConfiguration from pants.goal.builtins import register_builtin_goals @@ -138,7 +139,8 @@ def load_backend(build_configuration: BuildConfiguration.Builder, backend_packag """ backend_module = backend_package + ".register" try: - module = importlib.import_module(backend_module) + with ox.traditional_import_machinery(): + module = importlib.import_module(backend_module) except ImportError as ex: traceback.print_exc() raise BackendConfigurationError(f"Failed to load the {backend_module} backend: {ex!r}") From 2a8616860632b6b605ba618ecc1e7ab8a8fab19a Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Thu, 25 Aug 2022 14:39:42 -0700 Subject: [PATCH 42/45] updates instructions # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/bin/pyoxidizer-howto.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/bin/pyoxidizer-howto.md b/src/python/pants/bin/pyoxidizer-howto.md index b25dc8a75e7..63dbed55c2a 100644 --- a/src/python/pants/bin/pyoxidizer-howto.md +++ b/src/python/pants/bin/pyoxidizer-howto.md @@ -1,6 +1,6 @@ NOTES: -Run `./pants --no-pantsd --no-remote-cache-write package --pyoxidizer-args="--target-triple=aarch64-apple-darwin" --pyoxidizer-interpreter-constraints="['CPython==3.9.*']" src/python/pants/bin:pants_oxidized_experimental` +Run `./pants package --pyoxidizer-interpreter-constraints="['CPython==3.9.*']" src/python/pants/bin:pants_oxidized_experimental` The binary will be `dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental` -- this will not work on the pants repo itself (yet?) From 5203dbd239ab38902471b0905d04b71ae12afb9c Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 26 Aug 2022 10:28:00 -0700 Subject: [PATCH 43/45] Fix merge snafus # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- 3rdparty/python/user_reqs.lock | 41 +++++++++++++++------------------- src/python/pants/conftest.py | 37 ++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/3rdparty/python/user_reqs.lock b/3rdparty/python/user_reqs.lock index ecf5aa005b4..799441b8aba 100644 --- a/3rdparty/python/user_reqs.lock +++ b/3rdparty/python/user_reqs.lock @@ -4,7 +4,7 @@ // // --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE --- // { -// "version": 3, +// "version": 2, // "valid_for_interpreter_constraints": [ // "CPython<3.10,>=3.7" // ], @@ -41,11 +41,7 @@ // "types-toml==0.10.8", // "typing-extensions==4.3.0", // "uvicorn[standard]==0.17.6" -// ], -// "manylinux": "manylinux2014", -// "requirement_constraints": [], -// "only_binary": [], -// "no_binary": [] +// ] // } // --- END PANTS LOCKFILE METADATA --- @@ -250,13 +246,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f", - "url": "https://files.pythonhosted.org/packages/db/51/a507c856293ab05cdc1db77ff4bc1268ddd39f29e7dc4919aa497f0adbec/charset_normalizer-2.1.1-py3-none-any.whl" + "hash": "5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "url": "https://files.pythonhosted.org/packages/94/69/64b11e8c2fb21f08634468caef885112e682b0ebe2908e74d3616eb1c113/charset_normalizer-2.1.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", - "url": "https://files.pythonhosted.org/packages/a1/34/44964211e5410b051e4b8d2869c470ae8a68ae274953b1c7de6d98bbcf94/charset-normalizer-2.1.1.tar.gz" + "hash": "575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413", + "url": "https://files.pythonhosted.org/packages/93/1d/d9392056df6670ae2a29fcb04cfa5cee9f6fbde7311a1bb511d4115e9b7a/charset-normalizer-2.1.0.tar.gz" } ], "project_name": "charset-normalizer", @@ -264,7 +260,7 @@ "unicodedata2; extra == \"unicode_backport\"" ], "requires_python": ">=3.6.0", - "version": "2.1.1" + "version": "2.1" }, { "artifacts": [ @@ -1788,19 +1784,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "333e675b188a1c1fd980b4b352f9e40572413a4c1ac689c23cd546e96310070a", - "url": "https://files.pythonhosted.org/packages/2f/e5/988bff1d30df1f32d402125fad52235c3214c6827c2b0b3a658ee93a921a/types_urllib3-1.26.23-py3-none-any.whl" + "hash": "09a8783e1002472e8d1e1f3792d4c5cca1fffebb9b48ee1512aae6d16fe186bc", + "url": "https://files.pythonhosted.org/packages/19/8c/b9f75c5b52f79402baeabbb065067f72e922f96a114fe471ce4069b0cb69/types_urllib3-1.26.22-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "b78e819f0e350221d0689a5666162e467ba3910737bafda14b5c2c85e9bb1e56", - "url": "https://files.pythonhosted.org/packages/4a/21/ddf7064ff4d10849ded229b2dabc9bfc5f19a7d4531ebf5c3b431267e603/types-urllib3-1.26.23.tar.gz" + "hash": "b05af90e73889e688094008a97ca95788db8bf3736e2776fd43fb6b171485d94", + "url": "https://files.pythonhosted.org/packages/d2/d8/764ca957d9601d956a87a311b107c9b6171ed5bb0c269e8ae60d91fc0fbf/types-urllib3-1.26.22.tar.gz" } ], "project_name": "types-urllib3", "requires_dists": [], "requires_python": null, - "version": "1.26.23" + "version": "1.26.22" }, { "artifacts": [ @@ -1992,13 +1988,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997", - "url": "https://files.pythonhosted.org/packages/6f/de/5be2e3eed8426f871b170663333a0f627fc2924cc386cd41be065e7ea870/urllib3-1.26.12-py2.py3-none-any.whl" + "hash": "c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", + "url": "https://files.pythonhosted.org/packages/d1/cb/4783c8f1a90f89e260dbf72ebbcf25931f3a28f8f80e2e90f8a589941b19/urllib3-1.26.11-py2.py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", - "url": "https://files.pythonhosted.org/packages/b2/56/d87d6d3c4121c0bcec116919350ca05dc3afd2eeb7dc88d07e8083f8ea94/urllib3-1.26.12.tar.gz" + "hash": "ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a", + "url": "https://files.pythonhosted.org/packages/6d/d5/e8258b334c9eb8eb78e31be92ea0d5da83ddd9385dc967dd92737604d239/urllib3-1.26.11.tar.gz" } ], "project_name": "urllib3", @@ -2011,11 +2007,10 @@ "cryptography>=1.3.4; extra == \"secure\"", "idna>=2.0.0; extra == \"secure\"", "ipaddress; python_version == \"2.7\" and extra == \"secure\"", - "pyOpenSSL>=0.14; extra == \"secure\"", - "urllib3-secure-extra; extra == \"secure\"" + "pyOpenSSL>=0.14; extra == \"secure\"" ], "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,<4,>=2.7", - "version": "1.26.12" + "version": "1.26.11" }, { "artifacts": [ diff --git a/src/python/pants/conftest.py b/src/python/pants/conftest.py index a63ece2644c..192d83e3787 100644 --- a/src/python/pants/conftest.py +++ b/src/python/pants/conftest.py @@ -1,16 +1,49 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from pathlib import Path import pytest +# The top-level `pants` module must be a namespace package, because we build two dists from it +# (pantsbuild.pants, and pantsbuild.pants.testutil) and consumers of these dists need to be +# able to import from both. +# +# In fact it is an *implicit* namespace package - that is, it has no __init__.py file. +# See https://packaging.python.org/guides/packaging-namespace-packages/ . +# +# Unfortunately, the presence or absence of __init__.py affects how pytest determines the +# module names for test code. For details see +# https://docs.pytest.org/en/stable/goodpractices.html#test-package-name . +# +# Usually this doesn't matter, as tests don't typically care about their own module name. +# But we have tests (notably those in src/python/pants/engine) that create @rules and +# expect them to have certain names. And @rule names are generated from the name of the module +# containing the rule function... +# +# To allow those tests to work naturally (with expected module names relative to `src/python`) +# we artificially create `src/python/pants/__init__.py` in the test sandbox, to force +# pytest to determine module names relative to `src/python` (instead of `src/python/pants`). +# +# Note that while this makes the (implicit) namespace package into a regular package, +# that is fine at test time. We don't consume testutil from a dist but from source, in the same +# source root (src/python). We only need `pants` to be a namespace package in the dists we create. +namespace_init_path = Path("src/python/pants/__init__.py") + def pytest_sessionstart(session) -> None: - pass + if namespace_init_path.exists(): + raise Exception( + f"In order for `pants` to be a namespace package, {namespace_init_path} must not " + f"exist on disk. See the explanation in {__file__}." + ) + namespace_init_path.touch() def pytest_sessionfinish(session) -> None: - pass + # Technically unecessary, but nice if people are running tests directly from repo + # (not using pants). + namespace_init_path.unlink() @pytest.fixture(autouse=True, scope="session") From 727ed34817bccd57f29f0faa2c824fa65e5c2b5d Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Fri, 26 Aug 2022 10:29:40 -0700 Subject: [PATCH 44/45] Renames the `run_as` methods to begin with an underscore. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/ox.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index 61086bad82e..14e96715975 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -39,24 +39,31 @@ def bootstrap_pyoxidizer() -> None: def pex_main() -> bool: + """Detect whether some external process is trying to invoke this binary as Pex. + + When bootstrapping a plugin venv, Pants will reinvoke Pex in the binary that Pants was loaded + with. Pex will subsequently run a bunch of Python processes in order to run pip, venv, and + `python -c` invocations. This looks for certain command line switches, and attempt to invoke the + relevant modules that those switches indicate. + """ if len(sys.argv) < 2: return False if sys.argv[1] == "./pex": - run_as_pex() + _run_as_pex() return True if sys.argv[1] == "-sE" and sys.argv[2].endswith("/pex"): - run_pex_venv() + _run_pex_venv() return True if len(sys.argv) == 4 and sys.argv[1:3] == ["-s", "-c"]: - run_as_dash_c() + _run_as_dash_c() return True if ("-m", "venv") in zip(sys.argv, sys.argv[1:]): - run_as_venv() + _run_as_venv() return True return False @@ -106,7 +113,7 @@ def wrapped(*a, **k): @use_traditional_import_machinery -def run_as_pex(): +def _run_as_pex(): g = {} f = runpy.run_path("./pex", init_globals=g) del sys.argv[1] @@ -115,7 +122,7 @@ def run_as_pex(): @use_traditional_import_machinery -def run_as_venv(): +def _run_as_venv(): index = sys.argv.index("-m") import venv @@ -132,7 +139,7 @@ def run_as_venv(): @use_traditional_import_machinery -def run_as_dash_c(): +def _run_as_dash_c(): if "PEX" in os.environ: # Get pex into the modules cache pex = runpy.run_path(os.environ["PEX"]) @@ -144,7 +151,7 @@ def run_as_dash_c(): @use_traditional_import_machinery -def run_pex_venv(): +def _run_pex_venv(): path_to_run = sys.argv[2] site.PREFIXES = [os.path.dirname(path_to_run)] site.addsitepackages(set()) From 8984bde081c6d358faf6fd751c5c44afd0d2f756 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Mon, 29 Aug 2022 08:58:02 -0700 Subject: [PATCH 45/45] Address code review feedback # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/bin/pants_loader.py | 4 ++++ src/python/pants/init/plugin_resolver.py | 2 +- src/python/pants/ox.py | 26 +++++++++++++++--------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/python/pants/bin/pants_loader.py b/src/python/pants/bin/pants_loader.py index 1c4b77c05ed..86271d76990 100644 --- a/src/python/pants/bin/pants_loader.py +++ b/src/python/pants/bin/pants_loader.py @@ -113,6 +113,10 @@ def main(cls) -> None: def main() -> None: ox.bootstrap_pyoxidizer() + # In our `PyOxidizer`-generated binary, sometimes we'll be attempting to load this + # like a Python interpreter in order to run `pex` and its child processes. If we + # detect such an invocation, we'll try to run it, and then quit this process before + # getting into Pants itself. if ox.is_oxidized and ox.pex_main(): return diff --git a/src/python/pants/init/plugin_resolver.py b/src/python/pants/init/plugin_resolver.py index 66a04c72ad8..4c27b32cece 100644 --- a/src/python/pants/init/plugin_resolver.py +++ b/src/python/pants/init/plugin_resolver.py @@ -12,6 +12,7 @@ from pkg_resources import Requirement, WorkingSet from pkg_resources import working_set as global_working_set +from pants import ox from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess from pants.backend.python.util_rules.pex_environment import PythonExecutable @@ -128,7 +129,6 @@ def resolve( env: CompleteEnvironment, ) -> WorkingSet: """Resolves any configured plugins and adds them to the working_set.""" - from pants import ox with ox.traditional_import_machinery(): for resolved_plugin_location in self._resolve_plugins( diff --git a/src/python/pants/ox.py b/src/python/pants/ox.py index 14e96715975..d6c75b347f9 100644 --- a/src/python/pants/ox.py +++ b/src/python/pants/ox.py @@ -39,18 +39,23 @@ def bootstrap_pyoxidizer() -> None: def pex_main() -> bool: - """Detect whether some external process is trying to invoke this binary as Pex. - - When bootstrapping a plugin venv, Pants will reinvoke Pex in the binary that Pants was loaded - with. Pex will subsequently run a bunch of Python processes in order to run pip, venv, and - `python -c` invocations. This looks for certain command line switches, and attempt to invoke the - relevant modules that those switches indicate. + """Detect whether some process is trying to invoke this binary as (or as a Child of) Pex: + + In order to fetch 3rd-party plugins that will be compatible with Pants' environment, Pants will + invoke Pex using the same binary that Pants was loaded with. In a non-PyOxidized environment, + this is a working Python interpreter. In PyOxidized environments, it's the binary itself. Pex + subsequently runs a bunch of Python processes in order to invoke `pip`, `venv`, and `python -c`. + This looks for certain command line switches, and attempt to invoke the relevant modules that + those switches indicate. Each of the functions below will get the import machinery and + `sys.modules` content into a state where the relevant invocations will run in a more-or-less + "normal" manner (at least as far as `pex` is concerned). """ if len(sys.argv) < 2: return False if sys.argv[1] == "./pex": + # `pex` itself is being called to assemble a pex package _run_as_pex() return True @@ -97,10 +102,11 @@ def traditional_import_machinery(): ), ] + sys.path_hooks - yield - - sys.meta_path = old_sys_meta_path - sys.path_hooks = old_sys_path_hooks + try: + yield + finally: + sys.meta_path = old_sys_meta_path + sys.path_hooks = old_sys_path_hooks def use_traditional_import_machinery(f):