From 3033d2b001da68815b11701c02459b8bcf24f99d Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 25 Jan 2025 16:14:42 -0800 Subject: [PATCH 01/17] wip: provide way to not use dangling symlinks --- CHANGELOG.md | 4 ++ .../python/config_settings/index.md | 22 +++++++ docs/environment-variables.md | 13 ++++ examples/bzlmod/.bazelversion | 2 +- examples/bzlmod/BUILD.bazel | 35 +++++++++++ examples/bzlmod/MODULE.bazel | 2 + python/config_settings/BUILD.bazel | 8 +++ python/private/flags.bzl | 15 +++++ python/private/py_executable.bzl | 33 +++++++++-- python/private/stage1_bootstrap_template.sh | 59 +++++++++++++++++-- 10 files changed, 181 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f8da580a1..4871a2f28f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,10 @@ Unreleased changes template. The related issue is [#908](https://github.com/bazelbuild/rules_python/issue/908). * (sphinxdocs) Do not crash when `tag_class` does not have a populated `doc` value. Fixes ([#2579](https://github.com/bazelbuild/rules_python/issues/2579)). +* (binaries/tests) Fix packaging when using `--bootstrap_impl=script`: set + {obj}`--relative_venv_symlinks=no` to have it avoid creating symlinks at + build time. + Fixes ([#2489](https://github.com/bazelbuild/rules_python/issues/2489) {#v0-0-0-added} ### Added diff --git a/docs/api/rules_python/python/config_settings/index.md b/docs/api/rules_python/python/config_settings/index.md index 793f6e08fd..fc99f46759 100644 --- a/docs/api/rules_python/python/config_settings/index.md +++ b/docs/api/rules_python/python/config_settings/index.md @@ -212,6 +212,28 @@ Values: ::: :::: +::::{bzl:flag} relative_venv_symlinks + +Determines if relative symlinks are created using `declare_symlink()` at build +time. + +This is only intended to work around +[#2489](https://github.com/bazelbuild/rules_python/issues/2489), where some +packaging rules don't support `declare_symlink()` artifacts. + +Values: +* `yes`: Use `declare_symlink()` and create relative symlinks at build time. +* `no`: Do not use `declare_symlink()`. Instead, the venv will be created at + runtime. + +:::{seealso} +{envvar}`RULES_PYTHON_VENVS_ROOT` for customizing where the runtime venv +is created. +::: + +:::{versionadded} VERSION_NEXT_PATCH +::: + ::::{bzl:flag} bootstrap_impl Determine how programs implement their startup process. diff --git a/docs/environment-variables.md b/docs/environment-variables.md index fb9971597b..09c924899d 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -68,3 +68,16 @@ When `1`, debug information about coverage behavior is printed to stderr. When `1`, debug information from gazelle is printed to stderr. ::: + +:::{envvar} RULES_PYTHON_VENVS_ROOT + +Directory to use as the root for creating venvs for binaries. Only applicable +when {obj}`--relative_venvs_symlinks=no` is used. A binary will attempt to +find a unique, reusable, location for itself within this directory. When set, +the created venv is not deleted upon program exit; it is the responsibility of +the caller to manage cleanup. + +If not set, then a temporary directory will be created and deleted upon program +exit. + +::: diff --git a/examples/bzlmod/.bazelversion b/examples/bzlmod/.bazelversion index 35907cd9ca..c6b7980b68 100644 --- a/examples/bzlmod/.bazelversion +++ b/examples/bzlmod/.bazelversion @@ -1 +1 @@ -7.x +8.x diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index df07385690..6423c547bf 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -9,6 +9,8 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@pip//:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements", "requirement") load("@python_3_9//:defs.bzl", py_test_with_transition = "py_test") load("@python_versions//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements") +load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "pkg_mklink") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_library.bzl", "py_library") load("@rules_python//python:py_test.bzl", "py_test") @@ -50,6 +52,39 @@ py_binary( ], ) +pkg_tar( + name = "mytar", + srcs = [ + ":myfiles", + ":myinter", + ], + allow_duplicates_with_different_content = True, + ### + ##srcs = [":bzlmod"], + ##include_runfiles = True, + ##remap_paths = { + ## "bzlmod.runfiles/_main/_bzlmod.venv/bin/python3": "blah/whatever", + ##}, + ##symlinks = { + ## "bzlmod.runfiles/_main/_bzlmod.venv/bin/python3": "../../interpreter", + ##}, +) + +pkg_files( + name = "myfiles", + srcs = [":bzlmod"], + excludes = [ + "asdf", + ], + include_runfiles = True, +) + +pkg_mklink( + name = "myinter", + link_name = "bzlmod.runfiles/_main/_bzlmod.venv/bin/python3", + target = "mytarget", +) + # see https://bazel.build/reference/be/python#py_test py_test( name = "test", diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index d8535a0115..3beb76723c 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -7,6 +7,8 @@ module( bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.4") bazel_dep(name = "rules_python", version = "0.0.0") +bazel_dep(name = "rules_pkg", version = "1.0.1") + local_path_override( module_name = "rules_python", path = "../..", diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index fcebcd76dc..798e07bae5 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -9,6 +9,7 @@ load( "LibcFlag", "PrecompileFlag", "PrecompileSourceRetentionFlag", + "RelativeVenvSymlinksFlag", ) load( "//python/private/pypi:flags.bzl", @@ -121,6 +122,13 @@ config_setting( visibility = ["//visibility:public"], ) +string_flag( + name = "relative_venv_symlinks", + build_setting_default = RelativeVenvSymlinksFlag.YES, + values = RelativeVenvSymlinksFlag.flag_values(), + visibility = ["//visibility:public"], +) + # pip.parse related flags string_flag( diff --git a/python/private/flags.bzl b/python/private/flags.bzl index 9070f113ac..917a60f8ed 100644 --- a/python/private/flags.bzl +++ b/python/private/flags.bzl @@ -123,6 +123,21 @@ PrecompileSourceRetentionFlag = enum( get_effective_value = _precompile_source_retention_flag_get_effective_value, ) +def _relative_venv_symlinks_flag_get_value(ctx): + return ctx.attr._relative_venv_symlinks_flag[BuildSettingInfo].value + +# Decides if the venv created by bootstrap=script uses declare_file() to +# create relative symlinks. Workaround for #2489 (packaging rules not supporting +# declare_link() files). +# buildifier: disable=name-conventions +RelativeVenvSymlinksFlag = FlagEnum( + # Use declare_file() and relative symlinks in the venv + YES = "yes", + # Do not use declare_file() and relative symlinks in the venv + NO = "no", + get_value = _relative_venv_symlinks_flag_get_value, +) + # Used for matching freethreaded toolchains and would have to be used in wheels # as well. # buildifier: disable=name-conventions diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 1e437f57e1..e08d646971 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -51,7 +51,7 @@ load( "target_platform_has_any_constraint", "union_attrs", ) -load(":flags.bzl", "BootstrapImplFlag") +load(":flags.bzl", "BootstrapImplFlag", "RelativeVenvSymlinksFlag") load(":precompile.bzl", "maybe_precompile") load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") load(":py_executable_info.bzl", "PyExecutableInfo") @@ -195,6 +195,10 @@ accepting arbitrary Python versions. "_python_version_flag": attr.label( default = "//python/config_settings:python_version", ), + "_relative_venv_symlinks_flag": attr.label( + default = "//python/config_settings:relative_venv_symlinks", + providers = [BuildSettingInfo], + ), "_windows_constraints": attr.label_list( default = [ "@platforms//os:windows", @@ -512,7 +516,25 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): ctx.actions.write(pyvenv_cfg, "") runtime = runtime_details.effective_runtime - if runtime.interpreter: + relative_venv_symlinks_enabled = ( + RelativeVenvSymlinksFlag.get_value(ctx) == RelativeVenvSymlinksFlag.YES + ) + + if not relative_venv_symlinks_enabled: + if runtime.interpreter: + interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) + else: + interpreter_actual_path = runtime.interpreter_path + + py_exe_basename = paths.basename(interpreter_actual_path) + + # When the venv symlinks are disabled, the $venv/bin/python3 file isn't + # needed or used at runtime. However, the zip code uses the interpreter + # File object to figure out some paths. + interpreter = ctx.actions.declare_file("{}/bin/{}".format(venv, py_exe_basename)) + ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path)) + + elif runtime.interpreter: py_exe_basename = paths.basename(runtime.interpreter.short_path) # Even though ctx.actions.symlink() is used, using @@ -571,6 +593,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): return struct( interpreter = interpreter, + recreate_venv_at_runtime = not relative_venv_symlinks_enabled, # Runfiles root relative path or absolute path interpreter_actual_path = interpreter_actual_path, files_without_interpreter = [pyvenv_cfg, pth, site_init], @@ -657,15 +680,13 @@ def _create_stage1_bootstrap( else: python_binary_path = runtime_details.executable_interpreter_path - if is_for_zip and venv: - python_binary_actual = venv.interpreter_actual_path - else: - python_binary_actual = "" + python_binary_actual = venv.interpreter_actual_path if venv else "" subs = { "%is_zipfile%": "1" if is_for_zip else "0", "%python_binary%": python_binary_path, "%python_binary_actual%": python_binary_actual, + "%recreate_venv_at_runtime%": str(int(venv.recreate_venv_at_runtime)) if venv else "0", "%target%": str(ctx.label), "%workspace_name%": ctx.workspace_name, } diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index b05b4a54cd..c5408f157f 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -9,15 +9,17 @@ fi # runfiles-relative path STAGE2_BOOTSTRAP="%stage2_bootstrap%" -# runfiles-relative path +# runfiles-relative path to python interpreter to use PYTHON_BINARY='%python_binary%' # The path that PYTHON_BINARY should symlink to. # runfiles-relative path, absolute path, or single word. -# Only applicable for zip files. +# Only applicable for zip files or when venv is recreated at runtime. PYTHON_BINARY_ACTUAL="%python_binary_actual%" # 0 or 1 IS_ZIPFILE="%is_zipfile%" +# 0 or 1 +RECREATE_VENV_AT_RUNTIME="%recreate_venv_at_runtime%" if [[ "$IS_ZIPFILE" == "1" ]]; then # NOTE: Macs have an old version of mktemp, so we must use only the @@ -104,6 +106,7 @@ python_exe=$(find_python_interpreter $RUNFILES_DIR $PYTHON_BINARY) # Zip files have to re-create the venv bin/python3 symlink because they # don't contain it already. if [[ "$IS_ZIPFILE" == "1" ]]; then + use_exec=0 # It should always be under runfiles, but double check this. We don't # want to accidentally create symlinks elsewhere. if [[ "$python_exe" != $RUNFILES_DIR/* ]]; then @@ -121,13 +124,60 @@ if [[ "$IS_ZIPFILE" == "1" ]]; then symlink_to=$(which $PYTHON_BINARY_ACTUAL) # Guard against trying to symlink to an empty value if [[ $? -ne 0 ]]; then - echo >&2 "ERROR: Python to use found on PATH: $PYTHON_BINARY_ACTUAL" + echo >&2 "ERROR: Python to use not found on PATH: $PYTHON_BINARY_ACTUAL" exit 1 fi fi # The bin/ directory may not exist if it is empty. mkdir -p "$(dirname $python_exe)" ln -s "$symlink_to" "$python_exe" +elif [[ "$RECREATE_VENV_AT_RUNTIME" == "1" ]]; then + runfiles_venv="$RUNFILES_DIR/$(dirname $(dirname $PYTHON_BINARY))" + if [[ -n "$RULES_PYTHON_VENVS_ROOT" ]]; then + use_exec=1 + # Use our runfiles path as a unique, reusable, location for the + # binary-specific venv being created. + venv="$RULES_PYTHON_VENVS_ROOT/$(dirname $(dirname $PYTHON_BINARY))" + mkdir -p $RULES_PYTHON_VENVS_ROOT + else + # Re-exec'ing can't be used because we have to clean up the temporary + # venv directory that is created. + use_exec=0 + venv=$(mktemp -d) + if [[ -n "$venv" && -z "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then + trap 'rm -fr "$venv"' EXIT + fi + fi + + if [[ "$PYTHON_BINARY_ACTUAL" == /* ]]; then + # An absolute path, i.e. platform runtime, e.g. /usr/bin/python3 + symlink_to=$PYTHON_BINARY_ACTUAL + elif [[ "$PYTHON_BINARY_ACTUAL" == */* ]]; then + # A runfiles-relative path + symlink_to="$RUNFILES_DIR/$PYTHON_BINARY_ACTUAL" + else + # A plain word, e.g. "python3". Symlink to where PATH leads + symlink_to=$(which $PYTHON_BINARY_ACTUAL) + # Guard against trying to symlink to an empty value + if [[ $? -ne 0 ]]; then + echo >&2 "ERROR: Python to use not found on PATH: $PYTHON_BINARY_ACTUAL" + exit 1 + fi + fi + mkdir -p "$venv/bin" + # Match the basename; some tools, e.g. pyvenv key off the executable name + python_exe="$venv/bin/$(basename $PYTHON_BINARY_ACTUAL)" + if [[ ! -e "$python_exe" ]]; then + ln -s "$symlink_to" "$python_exe" + fi + if [[ ! -e "$venv/pyvenv.cfg" ]]; then + ln -s "$runfiles_venv/pyvenv.cfg" "$venv/pyvenv.cfg" + fi + if [[ ! -e "$venv/lib" ]]; then + ln -s "$runfiles_venv/lib" "$venv/lib" + fi +else + use_exec=1 fi # At this point, we should have a valid reference to the interpreter. @@ -165,7 +215,6 @@ if [[ "$IS_ZIPFILE" == "1" ]]; then interpreter_args+=("-XRULES_PYTHON_ZIP_DIR=$zip_dir") fi - export RUNFILES_DIR command=( @@ -186,7 +235,7 @@ command=( # # However, when running a zip file, we need to clean up the workspace after the # process finishes so control must return here. -if [[ "$IS_ZIPFILE" == "1" ]]; then +if [[ "$use_exec" == "0" ]]; then "${command[@]}" exit $? else From f7ef53fadb5959cc38b96a1b229c3d544bba4a77 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 28 Jan 2025 22:26:05 -0800 Subject: [PATCH 02/17] revert debug changes --- examples/bzlmod/.bazelversion | 2 +- examples/bzlmod/BUILD.bazel | 35 ----------------------------------- examples/bzlmod/MODULE.bazel | 2 -- 3 files changed, 1 insertion(+), 38 deletions(-) diff --git a/examples/bzlmod/.bazelversion b/examples/bzlmod/.bazelversion index c6b7980b68..35907cd9ca 100644 --- a/examples/bzlmod/.bazelversion +++ b/examples/bzlmod/.bazelversion @@ -1 +1 @@ -8.x +7.x diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index 6423c547bf..df07385690 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -9,8 +9,6 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@pip//:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements", "requirement") load("@python_3_9//:defs.bzl", py_test_with_transition = "py_test") load("@python_versions//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements") -load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "pkg_mklink") -load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_library.bzl", "py_library") load("@rules_python//python:py_test.bzl", "py_test") @@ -52,39 +50,6 @@ py_binary( ], ) -pkg_tar( - name = "mytar", - srcs = [ - ":myfiles", - ":myinter", - ], - allow_duplicates_with_different_content = True, - ### - ##srcs = [":bzlmod"], - ##include_runfiles = True, - ##remap_paths = { - ## "bzlmod.runfiles/_main/_bzlmod.venv/bin/python3": "blah/whatever", - ##}, - ##symlinks = { - ## "bzlmod.runfiles/_main/_bzlmod.venv/bin/python3": "../../interpreter", - ##}, -) - -pkg_files( - name = "myfiles", - srcs = [":bzlmod"], - excludes = [ - "asdf", - ], - include_runfiles = True, -) - -pkg_mklink( - name = "myinter", - link_name = "bzlmod.runfiles/_main/_bzlmod.venv/bin/python3", - target = "mytarget", -) - # see https://bazel.build/reference/be/python#py_test py_test( name = "test", diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 3beb76723c..d8535a0115 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -7,8 +7,6 @@ module( bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.4") bazel_dep(name = "rules_python", version = "0.0.0") -bazel_dep(name = "rules_pkg", version = "1.0.1") - local_path_override( module_name = "rules_python", path = "../..", From 41a4131c4f9be117536cf1ad42dd915ab0c8417c Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 28 Jan 2025 22:45:14 -0800 Subject: [PATCH 03/17] add basic test --- tests/bootstrap_impls/BUILD.bazel | 9 +++ tests/bootstrap_impls/bin.py | 1 + ...n_binary_relative_venv_symlinks_no_test.sh | 56 +++++++++++++++++++ tests/support/sh_py_run_test.bzl | 5 ++ 4 files changed, 71 insertions(+) create mode 100755 tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh diff --git a/tests/bootstrap_impls/BUILD.bazel b/tests/bootstrap_impls/BUILD.bazel index 8e50f34cfa..de01b7a04e 100644 --- a/tests/bootstrap_impls/BUILD.bazel +++ b/tests/bootstrap_impls/BUILD.bazel @@ -29,6 +29,15 @@ sh_py_run_test( sh_src = "run_binary_zip_yes_test.sh", ) +sh_py_run_test( + name = "run_binary_relative_venv_symlinks_no_test", + bootstrap_impl = "script", + py_src = "bin.py", + relative_venv_symlinks = "no", + sh_src = "run_binary_relative_venv_symlinks_no_test.sh", + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, +) + sh_py_run_test( name = "run_binary_bootstrap_script_zip_yes_test", bootstrap_impl = "script", diff --git a/tests/bootstrap_impls/bin.py b/tests/bootstrap_impls/bin.py index c46e43adc8..1176107384 100644 --- a/tests/bootstrap_impls/bin.py +++ b/tests/bootstrap_impls/bin.py @@ -22,3 +22,4 @@ print("PYTHONSAFEPATH:", os.environ.get("PYTHONSAFEPATH", "UNSET") or "EMPTY") print("sys.flags.safe_path:", sys.flags.safe_path) print("file:", __file__) +print("sys.executable:", sys.executable) diff --git a/tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh b/tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh new file mode 100755 index 0000000000..4c8342567e --- /dev/null +++ b/tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh @@ -0,0 +1,56 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- +set +e + +bin=$(rlocation $BIN_RLOCATION) +if [[ -z "$bin" ]]; then + echo "Unable to locate test binary: $BIN_RLOCATION" + exit 1 +fi +actual=$($bin) + +function expect_match() { + local expected_pattern=$1 + local actual=$2 + if ! (echo "$actual" | grep "$expected_pattern" ) >/dev/null; then + echo "expected to match: $expected_pattern" + echo "===== actual START =====" + echo "$actual" + echo "===== actual END =====" + echo + touch EXPECTATION_FAILED + return 1 + fi +} + +expect_match "sys.executable:.*tmp.*python3" "$actual" + +venvs_root=$(mkdir -d) + +actual=$(RULES_PYTHON_VENVS_ROOT=$venvs_root $bin) +expect_match "sys.executable:.*$venvs_root" "$actual" + +# Exit if any of the expects failed +[[ ! -e EXPECTATION_FAILED ]] diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 9bf0a7402e..5632b4c995 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -33,6 +33,8 @@ def _perform_transition_impl(input_settings, attr): settings["//command_line_option:extra_toolchains"] = attr.extra_toolchains if attr.python_version: settings["//python/config_settings:python_version"] = attr.python_version + if attr.relative_venv_symlinks: + settings["//python/config_settings:relative_venv_symlinks"] = attr.relative_venv_symlinks return settings _perform_transition = transition( @@ -41,12 +43,14 @@ _perform_transition = transition( "//python/config_settings:bootstrap_impl", "//command_line_option:extra_toolchains", "//python/config_settings:python_version", + "//python/config_settings:relative_venv_symlinks", ], outputs = [ "//command_line_option:build_python_zip", "//command_line_option:extra_toolchains", "//python/config_settings:bootstrap_impl", "//python/config_settings:python_version", + "//python/config_settings:relative_venv_symlinks", VISIBLE_FOR_TESTING, ], ) @@ -93,6 +97,7 @@ def _py_reconfig_impl(ctx): def _make_reconfig_rule(**kwargs): attrs = { "bootstrap_impl": attr.string(), + "relative_venv_symlinks": attr.string(), "build_python_zip": attr.string(default = "auto"), "extra_toolchains": attr.string_list( doc = """ From 2f2af0e3c3becfa403f4ac5f71bc430453c8cba8 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 28 Jan 2025 23:08:00 -0800 Subject: [PATCH 04/17] add packaging test --- MODULE.bazel | 1 + tests/packaging/BUILD.bazel | 41 ++++++++++++++++++++++++++++++++ tests/packaging/bin.py | 1 + tests/support/sh_py_run_test.bzl | 15 ++++++++---- 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 tests/packaging/BUILD.bazel create mode 100644 tests/packaging/bin.py diff --git a/MODULE.bazel b/MODULE.bazel index 7034357f61..89f1cd7961 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -84,6 +84,7 @@ bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True) bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True) bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True) +bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True) # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests. # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides. diff --git a/tests/packaging/BUILD.bazel b/tests/packaging/BUILD.bazel new file mode 100644 index 0000000000..8a79c2ca0a --- /dev/null +++ b/tests/packaging/BUILD.bazel @@ -0,0 +1,41 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") +load("//python:py_binary.bzl", "py_binary") +load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") + +build_test( + name = "bzl_libraries_build_test", + targets = [ + # keep sorted + ":bin_tar", + ], +) + +py_reconfig_test( + name = "bin_bootstrap_script", + srcs = ["bin.py"], + bootstrap_impl = "script", + main = "bin.py", + relative_venv_symlinks = "no", +) + +pkg_tar( + name = "bin_tar", + testonly = True, + srcs = [":bin"], + include_runfiles = True, +) diff --git a/tests/packaging/bin.py b/tests/packaging/bin.py new file mode 100644 index 0000000000..2f9a147db1 --- /dev/null +++ b/tests/packaging/bin.py @@ -0,0 +1 @@ +print("Hello") diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 5632b4c995..03d1b55059 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -132,10 +132,16 @@ def py_reconfig_test(*, name, **kwargs): name: str, name of teset target. **kwargs: kwargs to pass along to _py_reconfig_test and py_test. """ - reconfig_kwargs = {} - reconfig_kwargs["bootstrap_impl"] = kwargs.pop("bootstrap_impl", None) - reconfig_kwargs["extra_toolchains"] = kwargs.pop("extra_toolchains", None) - reconfig_kwargs["python_version"] = kwargs.pop("python_version", None) + reconfig_only_kwarg_names = [ + "bootstrap_impl", + "extra_toolchains", + "python_version", + "relative_venv_symlinks", + ] + reconfig_kwargs = { + key: kwargs.pop(key, None) + for key in reconfig_only_kwarg_names + } reconfig_kwargs["target_compatible_with"] = kwargs.get("target_compatible_with") inner_name = "_{}_inner".format(name) @@ -144,6 +150,7 @@ def py_reconfig_test(*, name, **kwargs): target = inner_name, **reconfig_kwargs ) + py_test( name = inner_name, tags = ["manual"], From 93848e5cd65e6c1c35fe8fe907638c6b0430a1a2 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 28 Jan 2025 23:11:17 -0800 Subject: [PATCH 05/17] fix bin name --- tests/packaging/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/packaging/BUILD.bazel b/tests/packaging/BUILD.bazel index 8a79c2ca0a..3f69743075 100644 --- a/tests/packaging/BUILD.bazel +++ b/tests/packaging/BUILD.bazel @@ -26,7 +26,7 @@ build_test( ) py_reconfig_test( - name = "bin_bootstrap_script", + name = "bin", srcs = ["bin.py"], bootstrap_impl = "script", main = "bin.py", From e3eadf43228f71e141dd4621d0728dc286524aff Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 29 Jan 2025 18:35:01 -0800 Subject: [PATCH 06/17] update comment about use_exec triggering --- python/private/stage1_bootstrap_template.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index c5408f157f..7f5fef899a 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -233,8 +233,9 @@ command=( # See https://github.com/bazelbuild/rules_python/issues/2043#issuecomment-2215469971 # for more information. # -# However, when running a zip file, we need to clean up the workspace after the -# process finishes so control must return here. +# However, we can't use exec when there is cleanup to do afterwards. Control +# must return to this process so it can run the trap handlers. Such cases +# occur when zip mode or recreate_venv_at_runtime creates temporary files. if [[ "$use_exec" == "0" ]]; then "${command[@]}" exit $? From 37ea44293a4023ef834fae10e44bf436418273c1 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 29 Jan 2025 18:38:15 -0800 Subject: [PATCH 07/17] buildifier --- tests/packaging/BUILD.bazel | 1 - tests/support/sh_py_run_test.bzl | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/packaging/BUILD.bazel b/tests/packaging/BUILD.bazel index 3f69743075..5a9550a740 100644 --- a/tests/packaging/BUILD.bazel +++ b/tests/packaging/BUILD.bazel @@ -14,7 +14,6 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_pkg//pkg:tar.bzl", "pkg_tar") -load("//python:py_binary.bzl", "py_binary") load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") build_test( diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 03d1b55059..b891b57e85 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -97,7 +97,6 @@ def _py_reconfig_impl(ctx): def _make_reconfig_rule(**kwargs): attrs = { "bootstrap_impl": attr.string(), - "relative_venv_symlinks": attr.string(), "build_python_zip": attr.string(default = "auto"), "extra_toolchains": attr.string_list( doc = """ @@ -109,6 +108,7 @@ toolchain. """, ), "python_version": attr.string(), + "relative_venv_symlinks": attr.string(), "target": attr.label(executable = True, cfg = "target"), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", From ee640e80b80b041d5addb5e3a6096f08d92accb3 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 29 Jan 2025 18:42:08 -0800 Subject: [PATCH 08/17] note why relative venv symlinks is used on test --- tests/packaging/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/packaging/BUILD.bazel b/tests/packaging/BUILD.bazel index 5a9550a740..680ec13680 100644 --- a/tests/packaging/BUILD.bazel +++ b/tests/packaging/BUILD.bazel @@ -29,6 +29,8 @@ py_reconfig_test( srcs = ["bin.py"], bootstrap_impl = "script", main = "bin.py", + # Needed until https://github.com/bazelbuild/rules_pkg/issues/929 is fixed + # See: https://github.com/bazelbuild/rules_python/issues/2489 relative_venv_symlinks = "no", ) From c0d20a234c817c12b852594735422c82cc25a3c8 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 29 Jan 2025 18:44:53 -0800 Subject: [PATCH 09/17] skip on windows --- tests/packaging/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/packaging/BUILD.bazel b/tests/packaging/BUILD.bazel index 680ec13680..cbc450f3c8 100644 --- a/tests/packaging/BUILD.bazel +++ b/tests/packaging/BUILD.bazel @@ -15,6 +15,7 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") +load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT") build_test( name = "bzl_libraries_build_test", @@ -32,6 +33,7 @@ py_reconfig_test( # Needed until https://github.com/bazelbuild/rules_pkg/issues/929 is fixed # See: https://github.com/bazelbuild/rules_python/issues/2489 relative_venv_symlinks = "no", + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, ) pkg_tar( From d876f13bcd772d5274f019e65fea6f14010f824a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 2 Feb 2025 10:24:16 -0800 Subject: [PATCH 10/17] fix bug after merge --- tests/support/sh_py_run_test.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index f6f37c719e..3aaf6557db 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -128,7 +128,9 @@ _py_reconfig_test = _make_reconfig_rule(test = True) def _py_reconfig_executable(*, name, py_reconfig_rule, py_inner_rule, **kwargs): reconfig_only_kwarg_names = [ + # keep sorted "bootstrap_impl", + "build_python_zip", "extra_toolchains", "python_version", "relative_venv_symlinks", From 0737858c344a719c156fd3311d9c6a5d093eb9a2 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 2 Feb 2025 11:03:22 -0800 Subject: [PATCH 11/17] address review comments --- docs/environment-variables.md | 1 - python/private/stage1_bootstrap_template.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 09c924899d..0349f36ac6 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -79,5 +79,4 @@ the caller to manage cleanup. If not set, then a temporary directory will be created and deleted upon program exit. - ::: diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index 7f5fef899a..173f4d84ec 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -132,7 +132,6 @@ if [[ "$IS_ZIPFILE" == "1" ]]; then mkdir -p "$(dirname $python_exe)" ln -s "$symlink_to" "$python_exe" elif [[ "$RECREATE_VENV_AT_RUNTIME" == "1" ]]; then - runfiles_venv="$RUNFILES_DIR/$(dirname $(dirname $PYTHON_BINARY))" if [[ -n "$RULES_PYTHON_VENVS_ROOT" ]]; then use_exec=1 # Use our runfiles path as a unique, reusable, location for the @@ -170,6 +169,7 @@ elif [[ "$RECREATE_VENV_AT_RUNTIME" == "1" ]]; then if [[ ! -e "$python_exe" ]]; then ln -s "$symlink_to" "$python_exe" fi + runfiles_venv="$RUNFILES_DIR/$(dirname $(dirname $PYTHON_BINARY))" if [[ ! -e "$venv/pyvenv.cfg" ]]; then ln -s "$runfiles_venv/pyvenv.cfg" "$venv/pyvenv.cfg" fi From 3869174a50d97c711567267df03a07393d9a054c Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 2 Feb 2025 15:01:55 -0800 Subject: [PATCH 12/17] rename to rules_python_extract_root --- .../python/config_settings/index.md | 2 +- docs/environment-variables.md | 21 +++++++++++++------ python/private/stage1_bootstrap_template.sh | 6 +++--- ...n_binary_relative_venv_symlinks_no_test.sh | 2 +- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/api/rules_python/python/config_settings/index.md b/docs/api/rules_python/python/config_settings/index.md index fc99f46759..7bb8c1cacb 100644 --- a/docs/api/rules_python/python/config_settings/index.md +++ b/docs/api/rules_python/python/config_settings/index.md @@ -227,7 +227,7 @@ Values: runtime. :::{seealso} -{envvar}`RULES_PYTHON_VENVS_ROOT` for customizing where the runtime venv +{envvar}`RULES_PYTHON_EXTRACT_ROOT` for customizing where the runtime venv is created. ::: diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 0349f36ac6..efdbc4d1a4 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -69,13 +69,22 @@ When `1`, debug information about coverage behavior is printed to stderr. When `1`, debug information from gazelle is printed to stderr. ::: -:::{envvar} RULES_PYTHON_VENVS_ROOT +:::{envvar} RULES_PYTHON_EXTRACT_ROOT -Directory to use as the root for creating venvs for binaries. Only applicable -when {obj}`--relative_venvs_symlinks=no` is used. A binary will attempt to -find a unique, reusable, location for itself within this directory. When set, -the created venv is not deleted upon program exit; it is the responsibility of -the caller to manage cleanup. +Directory to use as the root for creating files necessary for bootstrapping so +that a binary can run. + +Only applicable when {obj}`--venvs_use_declare_symlink=no` is used. + +When set, a binary will attempt to find a unique, reusable, location within this +directory for the files it needs to create to aid startup. The files may not be +deleted upon program exit; it is the responsibility of the caller to ensure +cleanup. + +Manually specifying the directory is useful to lower the overhead of +extracting/creating files on every program execution. By using a location +outside /tmp, longer lived programs don't have to worry about files in /tmp +being cleaned up by the OS. If not set, then a temporary directory will be created and deleted upon program exit. diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index 173f4d84ec..d760f1bfa2 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -132,12 +132,12 @@ if [[ "$IS_ZIPFILE" == "1" ]]; then mkdir -p "$(dirname $python_exe)" ln -s "$symlink_to" "$python_exe" elif [[ "$RECREATE_VENV_AT_RUNTIME" == "1" ]]; then - if [[ -n "$RULES_PYTHON_VENVS_ROOT" ]]; then + if [[ -n "$RULES_PYTHON_EXTRACT_ROOTT" ]]; then use_exec=1 # Use our runfiles path as a unique, reusable, location for the # binary-specific venv being created. - venv="$RULES_PYTHON_VENVS_ROOT/$(dirname $(dirname $PYTHON_BINARY))" - mkdir -p $RULES_PYTHON_VENVS_ROOT + venv="$RULES_PYTHON_EXTRACT_ROOTT/$(dirname $(dirname $PYTHON_BINARY))" + mkdir -p $RULES_PYTHON_EXTRACT_ROOTT else # Re-exec'ing can't be used because we have to clean up the temporary # venv directory that is created. diff --git a/tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh b/tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh index 4c8342567e..8477e97a3a 100755 --- a/tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh +++ b/tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh @@ -49,7 +49,7 @@ expect_match "sys.executable:.*tmp.*python3" "$actual" venvs_root=$(mkdir -d) -actual=$(RULES_PYTHON_VENVS_ROOT=$venvs_root $bin) +actual=$(RULES_PYTHON_EXTRACT_ROOT=$venvs_root $bin) expect_match "sys.executable:.*$venvs_root" "$actual" # Exit if any of the expects failed From f0857dae3f377fc00999f4041f1513c6f18356c9 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 2 Feb 2025 15:10:17 -0800 Subject: [PATCH 13/17] rename to venvs_use_declare_symlink --- CHANGELOG.md | 2 +- .../python/config_settings/index.md | 2 +- docs/environment-variables.md | 95 +++++++++---------- python/config_settings/BUILD.bazel | 8 +- python/private/flags.bzl | 8 +- python/private/py_executable.bzl | 14 +-- tests/bootstrap_impls/BUILD.bazel | 6 +- ...nary_venvs_use_declare_symlink_no_test.sh} | 0 tests/packaging/BUILD.bazel | 4 +- tests/support/sh_py_run_test.bzl | 12 +-- 10 files changed, 75 insertions(+), 76 deletions(-) rename tests/bootstrap_impls/{run_binary_relative_venv_symlinks_no_test.sh => run_binary_venvs_use_declare_symlink_no_test.sh} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ec04ae57c..1c51e967b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,7 +78,7 @@ Unreleased changes template. * (sphinxdocs) Do not crash when `tag_class` does not have a populated `doc` value. Fixes ([#2579](https://github.com/bazelbuild/rules_python/issues/2579)). * (binaries/tests) Fix packaging when using `--bootstrap_impl=script`: set - {obj}`--relative_venv_symlinks=no` to have it avoid creating symlinks at + {obj}`--venvs_use_declare_symlink=no` to have it avoid creating symlinks at build time. Fixes ([#2489](https://github.com/bazelbuild/rules_python/issues/2489) diff --git a/docs/api/rules_python/python/config_settings/index.md b/docs/api/rules_python/python/config_settings/index.md index 7bb8c1cacb..32f17445b7 100644 --- a/docs/api/rules_python/python/config_settings/index.md +++ b/docs/api/rules_python/python/config_settings/index.md @@ -212,7 +212,7 @@ Values: ::: :::: -::::{bzl:flag} relative_venv_symlinks +::::{bzl:flag} venvs_use_declare_symlink Determines if relative symlinks are created using `declare_symlink()` at build time. diff --git a/docs/environment-variables.md b/docs/environment-variables.md index efdbc4d1a4..0d227f9e81 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -1,37 +1,9 @@ # Environment Variables -:::{envvar} RULES_PYTHON_REPO_DEBUG - -When `1`, repository rules will print debug information about what they're -doing. This is mostly useful for development to debug errors. -::: - -:::{envvar} RULES_PYTHON_REPO_DEBUG_VERBOSITY - -Determines the verbosity of logging output for repo rules. Valid values: - -* `DEBUG` -* `INFO` -* `TRACE` -::: - -:::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH - -Determines the python interpreter platform to be used for a particular -interpreter `(version, os, arch)` triple to be used in repository rules. -Replace the `VERSION_OS_ARCH` part with actual values when using, e.g. -`3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the -os, arch values are the same as the ones mentioned in the -`//python:versions.bzl` file. -::: - -:::{envvar} RULES_PYTHON_PIP_ISOLATED - -Determines if `--isolated` is used with pip. +:::{envvar} RULES_PYTHON_BOOTSTRAP_VERBOSE -Valid values: -* `0` and `false` mean to not use isolated mode -* Other non-empty values mean to use isolated mode. +When `1`, debug information about bootstrapping of a program is printed to +stderr. ::: :::{envvar} RULES_PYTHON_BZLMOD_DEBUG @@ -52,23 +24,6 @@ When `1`, the rules_python Starlark implementation of the core rules is used instead of the Bazel-builtin rules. Note this requires Bazel 7+. ::: -:::{envvar} RULES_PYTHON_BOOTSTRAP_VERBOSE - -When `1`, debug information about bootstrapping of a program is printed to -stderr. -::: - -:::{envvar} VERBOSE_COVERAGE - -When `1`, debug information about coverage behavior is printed to stderr. -::: - - -:::{envvar} RULES_PYTHON_GAZELLE_VERBOSE - -When `1`, debug information from gazelle is printed to stderr. -::: - :::{envvar} RULES_PYTHON_EXTRACT_ROOT Directory to use as the root for creating files necessary for bootstrapping so @@ -89,3 +44,47 @@ being cleaned up by the OS. If not set, then a temporary directory will be created and deleted upon program exit. ::: + +:::{envvar} RULES_PYTHON_GAZELLE_VERBOSE + +When `1`, debug information from gazelle is printed to stderr. +::: + +:::{envvar} RULES_PYTHON_PIP_ISOLATED + +Determines if `--isolated` is used with pip. + +Valid values: +* `0` and `false` mean to not use isolated mode +* Other non-empty values mean to use isolated mode. +::: + +:::{envvar} RULES_PYTHON_REPO_DEBUG + +When `1`, repository rules will print debug information about what they're +doing. This is mostly useful for development to debug errors. +::: + +:::{envvar} RULES_PYTHON_REPO_DEBUG_VERBOSITY + +Determines the verbosity of logging output for repo rules. Valid values: + +* `DEBUG` +* `INFO` +* `TRACE` +::: + +:::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH + +Determines the python interpreter platform to be used for a particular +interpreter `(version, os, arch)` triple to be used in repository rules. +Replace the `VERSION_OS_ARCH` part with actual values when using, e.g. +`3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the +os, arch values are the same as the ones mentioned in the +`//python:versions.bzl` file. +::: + +:::{envvar} VERBOSE_COVERAGE + +When `1`, debug information about coverage behavior is printed to stderr. +::: diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 798e07bae5..796cf0c9c4 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -9,7 +9,7 @@ load( "LibcFlag", "PrecompileFlag", "PrecompileSourceRetentionFlag", - "RelativeVenvSymlinksFlag", + "VenvsUseDeclareSymlinkFlag", ) load( "//python/private/pypi:flags.bzl", @@ -123,9 +123,9 @@ config_setting( ) string_flag( - name = "relative_venv_symlinks", - build_setting_default = RelativeVenvSymlinksFlag.YES, - values = RelativeVenvSymlinksFlag.flag_values(), + name = "venvs_use_declare_symlink", + build_setting_default = VenvsUseDeclareSymlinkFlag.YES, + values = VenvsUseDeclareSymlinkFlag.flag_values(), visibility = ["//visibility:public"], ) diff --git a/python/private/flags.bzl b/python/private/flags.bzl index 917a60f8ed..1019faa8d6 100644 --- a/python/private/flags.bzl +++ b/python/private/flags.bzl @@ -123,19 +123,19 @@ PrecompileSourceRetentionFlag = enum( get_effective_value = _precompile_source_retention_flag_get_effective_value, ) -def _relative_venv_symlinks_flag_get_value(ctx): - return ctx.attr._relative_venv_symlinks_flag[BuildSettingInfo].value +def _venvs_use_declare_symlink_flag_get_value(ctx): + return ctx.attr._venvs_use_declare_symlink_flag[BuildSettingInfo].value # Decides if the venv created by bootstrap=script uses declare_file() to # create relative symlinks. Workaround for #2489 (packaging rules not supporting # declare_link() files). # buildifier: disable=name-conventions -RelativeVenvSymlinksFlag = FlagEnum( +VenvsUseDeclareSymlinkFlag = FlagEnum( # Use declare_file() and relative symlinks in the venv YES = "yes", # Do not use declare_file() and relative symlinks in the venv NO = "no", - get_value = _relative_venv_symlinks_flag_get_value, + get_value = _venvs_use_declare_symlink_flag_get_value, ) # Used for matching freethreaded toolchains and would have to be used in wheels diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index e08d646971..18a7a707fc 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -51,7 +51,7 @@ load( "target_platform_has_any_constraint", "union_attrs", ) -load(":flags.bzl", "BootstrapImplFlag", "RelativeVenvSymlinksFlag") +load(":flags.bzl", "BootstrapImplFlag", "VenvsUseDeclareSymlinkFlag") load(":precompile.bzl", "maybe_precompile") load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") load(":py_executable_info.bzl", "PyExecutableInfo") @@ -195,8 +195,8 @@ accepting arbitrary Python versions. "_python_version_flag": attr.label( default = "//python/config_settings:python_version", ), - "_relative_venv_symlinks_flag": attr.label( - default = "//python/config_settings:relative_venv_symlinks", + "_venvs_use_declare_symlink_flag": attr.label( + default = "//python/config_settings:venvs_use_declare_symlink", providers = [BuildSettingInfo], ), "_windows_constraints": attr.label_list( @@ -516,11 +516,11 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): ctx.actions.write(pyvenv_cfg, "") runtime = runtime_details.effective_runtime - relative_venv_symlinks_enabled = ( - RelativeVenvSymlinksFlag.get_value(ctx) == RelativeVenvSymlinksFlag.YES + venvs_use_declare_symlink_enabled = ( + VenvsUseDeclareSymlinkFlag.get_value(ctx) == VenvsUseDeclareSymlinkFlag.YES ) - if not relative_venv_symlinks_enabled: + if not venvs_use_declare_symlink_enabled: if runtime.interpreter: interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) else: @@ -593,7 +593,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): return struct( interpreter = interpreter, - recreate_venv_at_runtime = not relative_venv_symlinks_enabled, + recreate_venv_at_runtime = not venvs_use_declare_symlink_enabled, # Runfiles root relative path or absolute path interpreter_actual_path = interpreter_actual_path, files_without_interpreter = [pyvenv_cfg, pth, site_init], diff --git a/tests/bootstrap_impls/BUILD.bazel b/tests/bootstrap_impls/BUILD.bazel index fff5fafff0..8a64bf2b5b 100644 --- a/tests/bootstrap_impls/BUILD.bazel +++ b/tests/bootstrap_impls/BUILD.bazel @@ -62,12 +62,12 @@ sh_py_run_test( ) sh_py_run_test( - name = "run_binary_relative_venv_symlinks_no_test", + name = "run_binary_venvs_use_declare_symlink_no_test", bootstrap_impl = "script", py_src = "bin.py", - relative_venv_symlinks = "no", - sh_src = "run_binary_relative_venv_symlinks_no_test.sh", + sh_src = "run_binary_venvs_use_declare_symlink_no_test.sh", target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, + venvs_use_declare_symlink = "no", ) sh_py_run_test( diff --git a/tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh b/tests/bootstrap_impls/run_binary_venvs_use_declare_symlink_no_test.sh similarity index 100% rename from tests/bootstrap_impls/run_binary_relative_venv_symlinks_no_test.sh rename to tests/bootstrap_impls/run_binary_venvs_use_declare_symlink_no_test.sh diff --git a/tests/packaging/BUILD.bazel b/tests/packaging/BUILD.bazel index cbc450f3c8..cc04c05ba9 100644 --- a/tests/packaging/BUILD.bazel +++ b/tests/packaging/BUILD.bazel @@ -30,10 +30,10 @@ py_reconfig_test( srcs = ["bin.py"], bootstrap_impl = "script", main = "bin.py", + target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, # Needed until https://github.com/bazelbuild/rules_pkg/issues/929 is fixed # See: https://github.com/bazelbuild/rules_python/issues/2489 - relative_venv_symlinks = "no", - target_compatible_with = SUPPORTS_BOOTSTRAP_SCRIPT, + venvs_use_declare_symlink = "no", ) pkg_tar( diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 3aaf6557db..4fa53ebd66 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -33,8 +33,8 @@ def _perform_transition_impl(input_settings, attr): settings["//command_line_option:extra_toolchains"] = attr.extra_toolchains if attr.python_version: settings["//python/config_settings:python_version"] = attr.python_version - if attr.relative_venv_symlinks: - settings["//python/config_settings:relative_venv_symlinks"] = attr.relative_venv_symlinks + if attr.venvs_use_declare_symlink: + settings["//python/config_settings:venvs_use_declare_symlink"] = attr.venvs_use_declare_symlink return settings _perform_transition = transition( @@ -43,14 +43,14 @@ _perform_transition = transition( "//python/config_settings:bootstrap_impl", "//command_line_option:extra_toolchains", "//python/config_settings:python_version", - "//python/config_settings:relative_venv_symlinks", + "//python/config_settings:venvs_use_declare_symlink", ], outputs = [ "//command_line_option:build_python_zip", "//command_line_option:extra_toolchains", "//python/config_settings:bootstrap_impl", "//python/config_settings:python_version", - "//python/config_settings:relative_venv_symlinks", + "//python/config_settings:venvs_use_declare_symlink", VISIBLE_FOR_TESTING, ], ) @@ -109,8 +109,8 @@ toolchain. """, ), "python_version": attr.string(), - "relative_venv_symlinks": attr.string(), "target": attr.label(executable = True, cfg = "target"), + "venvs_use_declare_symlink": attr.string(), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), @@ -133,7 +133,7 @@ def _py_reconfig_executable(*, name, py_reconfig_rule, py_inner_rule, **kwargs): "build_python_zip", "extra_toolchains", "python_version", - "relative_venv_symlinks", + "venvs_use_declare_symlink", ] reconfig_kwargs = { key: kwargs.pop(key, None) From fdbf2a6a5b8b928cdaad7996970cc585a0f971d6 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 2 Feb 2025 18:59:39 -0800 Subject: [PATCH 14/17] fix typo; fix broken test --- python/private/stage1_bootstrap_template.sh | 6 +++--- .../run_binary_venvs_use_declare_symlink_no_test.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index d760f1bfa2..19ff763094 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -132,12 +132,12 @@ if [[ "$IS_ZIPFILE" == "1" ]]; then mkdir -p "$(dirname $python_exe)" ln -s "$symlink_to" "$python_exe" elif [[ "$RECREATE_VENV_AT_RUNTIME" == "1" ]]; then - if [[ -n "$RULES_PYTHON_EXTRACT_ROOTT" ]]; then + if [[ -n "$RULES_PYTHON_EXTRACT_ROOT" ]]; then use_exec=1 # Use our runfiles path as a unique, reusable, location for the # binary-specific venv being created. - venv="$RULES_PYTHON_EXTRACT_ROOTT/$(dirname $(dirname $PYTHON_BINARY))" - mkdir -p $RULES_PYTHON_EXTRACT_ROOTT + venv="$RULES_PYTHON_EXTRACT_ROOT/$(dirname $(dirname $PYTHON_BINARY))" + mkdir -p $RULES_PYTHON_EXTRACT_ROOT else # Re-exec'ing can't be used because we have to clean up the temporary # venv directory that is created. diff --git a/tests/bootstrap_impls/run_binary_venvs_use_declare_symlink_no_test.sh b/tests/bootstrap_impls/run_binary_venvs_use_declare_symlink_no_test.sh index 8477e97a3a..d4840116f9 100755 --- a/tests/bootstrap_impls/run_binary_venvs_use_declare_symlink_no_test.sh +++ b/tests/bootstrap_impls/run_binary_venvs_use_declare_symlink_no_test.sh @@ -47,8 +47,8 @@ function expect_match() { expect_match "sys.executable:.*tmp.*python3" "$actual" -venvs_root=$(mkdir -d) - +# Now test that using a custom location for the bootstrap files works +venvs_root=$(mktemp -d) actual=$(RULES_PYTHON_EXTRACT_ROOT=$venvs_root $bin) expect_match "sys.executable:.*$venvs_root" "$actual" From f64ed4bd3d816ef050e4c654afb196e5f9d50399 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 2 Feb 2025 19:33:43 -0800 Subject: [PATCH 15/17] add versionadded, small doc updates --- CHANGELOG.md | 4 +- .../python/config_settings/index.md | 44 ++++++++++--------- docs/environment-variables.md | 5 ++- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c51e967b5..4001ac4678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,8 +78,8 @@ Unreleased changes template. * (sphinxdocs) Do not crash when `tag_class` does not have a populated `doc` value. Fixes ([#2579](https://github.com/bazelbuild/rules_python/issues/2579)). * (binaries/tests) Fix packaging when using `--bootstrap_impl=script`: set - {obj}`--venvs_use_declare_symlink=no` to have it avoid creating symlinks at - build time. + {obj}`--venvs_use_declare_symlink=no` to have it not create symlinks at + build time (they will be created at runtime instead). Fixes ([#2489](https://github.com/bazelbuild/rules_python/issues/2489) {#v0-0-0-added} diff --git a/docs/api/rules_python/python/config_settings/index.md b/docs/api/rules_python/python/config_settings/index.md index 32f17445b7..b2163233ca 100644 --- a/docs/api/rules_python/python/config_settings/index.md +++ b/docs/api/rules_python/python/config_settings/index.md @@ -212,27 +212,6 @@ Values: ::: :::: -::::{bzl:flag} venvs_use_declare_symlink - -Determines if relative symlinks are created using `declare_symlink()` at build -time. - -This is only intended to work around -[#2489](https://github.com/bazelbuild/rules_python/issues/2489), where some -packaging rules don't support `declare_symlink()` artifacts. - -Values: -* `yes`: Use `declare_symlink()` and create relative symlinks at build time. -* `no`: Do not use `declare_symlink()`. Instead, the venv will be created at - runtime. - -:::{seealso} -{envvar}`RULES_PYTHON_EXTRACT_ROOT` for customizing where the runtime venv -is created. -::: - -:::{versionadded} VERSION_NEXT_PATCH -::: ::::{bzl:flag} bootstrap_impl Determine how programs implement their startup process. @@ -280,3 +259,26 @@ Values: ::: :::: + +::::{bzl:flag} venvs_use_declare_symlink + +Determines if relative symlinks are created using `declare_symlink()` at build +time. + +This is only intended to work around +[#2489](https://github.com/bazelbuild/rules_python/issues/2489), where some +packaging rules don't support `declare_symlink()` artifacts. + +Values: +* `yes`: Use `declare_symlink()` and create relative symlinks at build time. +* `no`: Do not use `declare_symlink()`. Instead, the venv will be created at + runtime. + +:::{seealso} +{envvar}`RULES_PYTHON_EXTRACT_ROOT` for customizing where the runtime venv +is created. +::: + +:::{versionadded} VERSION_NEXT_PATCH +::: +:::: diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 0d227f9e81..12d9386224 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -24,7 +24,7 @@ When `1`, the rules_python Starlark implementation of the core rules is used instead of the Bazel-builtin rules. Note this requires Bazel 7+. ::: -:::{envvar} RULES_PYTHON_EXTRACT_ROOT +::::{envvar} RULES_PYTHON_EXTRACT_ROOT Directory to use as the root for creating files necessary for bootstrapping so that a binary can run. @@ -43,7 +43,10 @@ being cleaned up by the OS. If not set, then a temporary directory will be created and deleted upon program exit. + +:::{versionadded} VERSION_NEXT_PATCH ::: +:::: :::{envvar} RULES_PYTHON_GAZELLE_VERBOSE From 8ecd60d795b92c2cff39ecfcd365afbd14836496 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 2 Feb 2025 19:54:49 -0800 Subject: [PATCH 16/17] fix flag ref in doc --- docs/environment-variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 12d9386224..dd4a700081 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -29,7 +29,7 @@ instead of the Bazel-builtin rules. Note this requires Bazel 7+. Directory to use as the root for creating files necessary for bootstrapping so that a binary can run. -Only applicable when {obj}`--venvs_use_declare_symlink=no` is used. +Only applicable when {bzl:flag}`--venvs_use_declare_symlink=no` is used. When set, a binary will attempt to find a unique, reusable, location within this directory for the files it needs to create to aid startup. The files may not be From 910b08751672b8bed9167980f777873c4262bc78 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 2 Feb 2025 19:58:17 -0800 Subject: [PATCH 17/17] fix typo in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4001ac4678..61000a1b08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,7 +80,7 @@ Unreleased changes template. * (binaries/tests) Fix packaging when using `--bootstrap_impl=script`: set {obj}`--venvs_use_declare_symlink=no` to have it not create symlinks at build time (they will be created at runtime instead). - Fixes ([#2489](https://github.com/bazelbuild/rules_python/issues/2489) + (Fixes [#2489](https://github.com/bazelbuild/rules_python/issues/2489)) {#v0-0-0-added} ### Added