[experimental] adds pants_oxidized_experimental target, creating a standalone binary distribution for Pants

pants.toml

@@ -7,6 +7,7 @@ backend_packages.add = [
   "pants.backend.build_files.fmt.black",
   "pants.backend.python",
   "pants.backend.experimental.python.lint.autoflake",
+  "pants.backend.experimental.python.packaging.pyoxidizer",
   "pants.backend.explorer",
   "pants.backend.python.lint.black",
   "pants.backend.python.lint.docformatter",

src/python/pants/BUILD

@@ -2,10 +2,10 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 python_sources(
+    dependencies=[":resources"],
     overrides={
         # Enable `python -m pants ...` style execution ala `json.tool` or `venv`.
         "__main__.py": {"dependencies": ["src/python/pants/bin:pants_loader"]},
-        "version.py": {"dependencies": ["./VERSION:resources"]},
     },
 )
 

src/python/pants/backend/python/packaging/pyoxidizer/config.py

@@ -11,12 +11,14 @@
 def make_exe():
     dist = default_python_distribution()
     policy = dist.make_python_packaging_policy()
+    policy.extension_module_filter = "no-copyleft"
 
     # Note: Adding this for pydanic and libs that have the "unable to load from memory" error
     # https://github.com/indygreg/PyOxidizer/issues/438
     policy.resources_location_fallback = "filesystem-relative:lib"
 
     python_config = dist.make_python_interpreter_config()
+
     $RUN_MODULE
 
     exe = dist.to_python_executable(

src/python/pants/bin/BUILD

@@ -80,4 +80,11 @@ pex_binary(
     strip_pex_env=False,
 )
 
+
+pyoxidizer_binary(
+    name="pants_oxidized_experimental",
+    dependencies=["src/python/pants:pants-packaged"],
+    entry_point="pants.bin.pants_loader",
+)
+
 python_tests(name="tests")

src/python/pants/bin/pants_loader.py

@@ -1,6 +1,7 @@
 # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+
 import importlib
 import locale
 import os
@@ -9,6 +10,7 @@
 import warnings
 from textwrap import dedent
 
+from pants import ox
 from pants.base.exiter import PANTS_FAILED_EXIT_CODE
 from pants.bin.pants_env_vars import (
     DAEMON_ENTRYPOINT,
@@ -109,6 +111,11 @@ def main(cls) -> None:
 
 
 def main() -> None:
+    ox.bootstrap_pyoxidizer()
+
+    if ox.is_oxidized and ox.pex_main():
+        return
+
     PantsLoader.main()
 
 

src/python/pants/bin/pyoxidizer-howto.md

@@ -0,0 +1,17 @@
+NOTES:
+
+Run `./pants package --pyoxidizer-interpreter-constraints="['CPython==3.9.*']" src/python/pants/bin:pants_oxidized_experimental`
+
+The binary will be `dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/pants_oxidized_experimental` -- this will not work on the pants repo itself (yet?)
+
+
+# Code signing errors?
+
+
+Obtain a self-signed certificate using info at at https://developer.apple.com/library/archive/documentation/Security/Conceptual/CodeSigningGuide/Procedures/Procedures.html
+
+I've called mine "pyox-test".
+
+Get native engine to build first (i.e. run `./pants` and wait for rust to finish doing its thing).
+
+Run `codesign -s pyox-test /Users/chrisjrn/src/pants/dist/src.python.pants.bin/pants_oxidized_experimental/aarch64-apple-darwin/debug/install/lib/pants/engine/internals/native_engine.so`

src/python/pants/engine/rules.py

@@ -383,6 +383,7 @@ def collect_rules(*namespaces: Union[ModuleType, Mapping[str, Any]]) -> Iterable
 
     If no namespaces are given, collects all the @rules in the caller's module namespace.
     """
+
     if not namespaces:
         currentframe = inspect.currentframe()
         assert isinstance(currentframe, FrameType)

src/python/pants/init/extension_loader.py

@@ -8,6 +8,7 @@
 
 from pkg_resources import Requirement, WorkingSet
 
+from pants import ox
 from pants.base.exceptions import BackendConfigurationError
 from pants.build_graph.build_configuration import BuildConfiguration
 from pants.goal.builtins import register_builtin_goals
@@ -138,7 +139,8 @@ def load_backend(build_configuration: BuildConfiguration.Builder, backend_packag
     """
     backend_module = backend_package + ".register"
     try:
-        module = importlib.import_module(backend_module)
+        with ox.traditional_import_machinery():
+            module = importlib.import_module(backend_module)
     except ImportError as ex:
         traceback.print_exc()
         raise BackendConfigurationError(f"Failed to load the {backend_module} backend: {ex!r}")

src/python/pants/init/plugin_resolver.py

@@ -128,13 +128,16 @@ def resolve(
         env: CompleteEnvironment,
     ) -> WorkingSet:
         """Resolves any configured plugins and adds them to the working_set."""
-        for resolved_plugin_location in self._resolve_plugins(
-            options_bootstrapper, env, self._request
-        ):
-            site.addsitedir(
-                resolved_plugin_location
-            )  # Activate any .pth files plugin wheels may have.
-            self._working_set.add_entry(resolved_plugin_location)
+        from pants import ox
+
+        with ox.traditional_import_machinery():
+            for resolved_plugin_location in self._resolve_plugins(
+                options_bootstrapper, env, self._request
+            ):
+                site.addsitedir(
+                    resolved_plugin_location
+                )  # Activate any .pth files plugin wheels may have.
+                self._working_set.add_entry(resolved_plugin_location)
         return self._working_set
 
     def _resolve_plugins(

src/python/pants/option/subsystem.py

@@ -9,6 +9,7 @@
 from abc import ABCMeta
 from typing import Any, ClassVar, TypeVar
 
+from pants import ox
 from pants.engine.internals.selectors import AwaitableConstraints, Get
 from pants.option.errors import OptionsError
 from pants.option.option_types import collect_options_info
@@ -55,7 +56,12 @@ def signature(cls):
         partial_construct_subsystem.__name__ = name
         partial_construct_subsystem.__module__ = cls.__module__
         partial_construct_subsystem.__doc__ = cls.help
-        _, class_definition_lineno = inspect.getsourcelines(cls)
+
+        # `inspect.getsourcelines` does not work under oxidation
+        if not ox.is_oxidized:
+            _, class_definition_lineno = inspect.getsourcelines(cls)
+        else:
+            class_definition_lineno = 0  # `inspect.getsourcelines` returns 0 when undefined.
         partial_construct_subsystem.__line_number__ = class_definition_lineno
 
         return dict(

src/python/pants/ox.py

@@ -0,0 +1,160 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import contextlib
+import functools
+import importlib.machinery
+import logging
+import os
+import runpy
+import site
+import sys
+import zipimport
+
+_logger = logging.getLogger(__name__)
+
+
+def bootstrap_pyoxidizer() -> None:
+    if is_oxidized:
+        _logger.info("Pants is running as a PyOxidizer binary.")
+
+
+# Provide the `is_oxidized` symbol, to allow for workarounds in Pants code where we use things
+# that don't work under PyOxidizer's custom importer. `oxidized_importer` is only accessible
+# in Pants under PyOxidizer, so an import failure will occur if we're not oxidized.
+try:
+    import oxidized_importer  # type: ignore # pants: no-infer-dep # noqa: F401
+
+    is_oxidized = True
+except ModuleNotFoundError:
+    is_oxidized = False
+
+
+if is_oxidized and not sys.argv[0]:
+    # A not insignificant amount of Pants code relies on `sys.argv[0]`, which is modified in an
+    # invalid way by python's `pymain_run_module` support. For our purposes, the executable
+    # distribution is the correct `argv[0]`.
+    # See https://github.com/indygreg/PyOxidizer/issues/307
+    sys.argv[0] = sys.executable
+
+
+def pex_main() -> bool:
+    """Detect whether some external process is trying to invoke this binary as Pex.
+
+    When bootstrapping a plugin venv, Pants will reinvoke Pex in the binary that Pants was loaded
+    with. Pex will subsequently run a bunch of Python processes in order to run pip, venv, and
+    `python -c` invocations. This looks for certain command line switches, and attempt to invoke the
+    relevant modules that those switches indicate.
+    """
+
+    if len(sys.argv) < 2:
+        return False
+
+    if sys.argv[1] == "./pex":
+        _run_as_pex()
+        return True
+
+    if sys.argv[1] == "-sE" and sys.argv[2].endswith("/pex"):
+        _run_pex_venv()
+        return True
+
+    if len(sys.argv) == 4 and sys.argv[1:3] == ["-s", "-c"]:
+        _run_as_dash_c()
+        return True
+
+    if ("-m", "venv") in zip(sys.argv, sys.argv[1:]):
+        _run_as_venv()
+        return True
+
+    return False
+
+
+@contextlib.contextmanager
+def traditional_import_machinery():
+    # pex relies heavily on `__file__`, which the Oxidized importer does not
+    # believe in. This reinstates the default Python import machinery before
+    # loading and running `pex`, but keeps the PyOxidizer machinery at lowest
+    # priority, so we can still load interned `.py` sources (e.g. the stdlib)
+
+    old_sys_meta_path = sys.meta_path
+    old_sys_path_hooks = sys.path_hooks
+
+    if is_oxidized:
+        sys.meta_path = [
+            importlib.machinery.BuiltinImporter,
+            importlib.machinery.FrozenImporter,
+            importlib.machinery.PathFinder,
+        ] + sys.meta_path
+        sys.path_hooks = [
+            zipimport.zipimporter,
+            importlib.machinery.FileFinder.path_hook(
+                (
+                    importlib.machinery.ExtensionFileLoader,
+                    [".cpython-39-darwin.so", ".abi3.so", ".so"],
+                ),
+                (importlib.machinery.SourceFileLoader, [".py"]),
+                (importlib.machinery.SourcelessFileLoader, [".pyc"]),
+            ),
+        ] + sys.path_hooks
+
+    yield
+
+    sys.meta_path = old_sys_meta_path
+    sys.path_hooks = old_sys_path_hooks
+
+
+def use_traditional_import_machinery(f):
+    @functools.wraps(f)
+    def wrapped(*a, **k):
+        with traditional_import_machinery():
+            return f(*a, **k)
+
+    return wrapped
+
+
+@use_traditional_import_machinery
+def _run_as_pex():
+    g = {}
+    f = runpy.run_path("./pex", init_globals=g)
+    del sys.argv[1]
+    f["bootstrap_pex"]("./pex")
+    sys.exit(0)
+
+
+@use_traditional_import_machinery
+def _run_as_venv():
+    index = sys.argv.index("-m")
+
+    import venv
+
+    # `venv` is supplied by the oxidized importer (the stdlib is embedded in the rust binary)
+    # but uses the `__file__` attribute to find the location of `activate` scripts. These scripts
+    # are not needed by pex, so we're setting the value to something bogus just to prevent
+    # subsequent exceptions.
+    venv.__file__ = "SOMETHING/THAT/IS/NOT/NONE"
+
+    sys.argv[1:] = sys.argv[index + 2 :]
+    runpy.run_module("venv")
+    sys.exit(0)
+
+
+@use_traditional_import_machinery
+def _run_as_dash_c():
+    if "PEX" in os.environ:
+        # Get pex into the modules cache
+        pex = runpy.run_path(os.environ["PEX"])
+        pex["bootstrap_pex"](os.environ["PEX"], execute=False)
+
+    index = sys.argv.index("-c")
+    exec(sys.argv[index + 1], {}, {})
+    sys.exit(0)
+
+
+@use_traditional_import_machinery
+def _run_pex_venv():
+    path_to_run = sys.argv[2]
+    site.PREFIXES = [os.path.dirname(path_to_run)]
+    site.addsitepackages(set())
+    del sys.argv[1:3]
+    runpy.run_path(path_to_run, run_name="__main__")
+    sys.exit(0)