Merge branch 'main' into kaos/orhpaned-backends-checker

.devcontainer/Dockerfile

@@ -0,0 +1,15 @@
+FROM mcr.microsoft.com/devcontainers/rust:1
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    # Install prerequisites.
+    && apt-get install --no-install-recommends -y build-essential clang libffi-dev libfuse-dev libssl-dev protobuf-compiler python3-dev \
+    # Install `hyperfine` and `dbg` for debugging and benchmarking.
+    # See: https://www.pantsbuild.org/stable/docs/contributions/development/debugging-and-benchmarking
+    && apt-get install --no-install-recommends -y gdb python3-dbg \
+    && curl -Lo /tmp/hyperfine.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb \
+    && sudo dpkg -i /tmp/hyperfine.deb \
+    # Cleanup.
+    && rm -rf /tmp/* \
+    && apt-get autoremove -y \
+    && apt-get clean all \
+    && rm -rf /var/lib/apt/lists/*

.devcontainer/README.md

@@ -0,0 +1,39 @@
+# Dev Container for developing Pants
+
+This project provides a *Development Container* (or [Dev Container](https://containers.dev/) for short) with a full-featured development environment containing all the tools, libraries, and runtimes needed to work with the [Pants](https://github.com/pantsbuild/pants) codebase. Here are some advantages of using dev containers:
+
+- **Faster setup**: Dev containers can be pre-built with all the necessary tools, libraries and runtimes, making setting up a new development environment faster. This is especially useful for new team members joining a project who can quickly get started without spending time setting up their development environment.
+- **Portability**: Dev containers provide portability between different platforms and clouds, allowing developers to write once and run anywhere. This ensures that developers can use the same development environment across different machines and platforms without compatibility issues.
+- **Consistency**: Dev containers provide a consistent development environment for all developers working on a project. This ensures that everyone is using the same tools, libraries and runtimes, reducing the chances of compatibility issues and making it easier to collaborate and reproduce bugs.
+- **Isolation**: Dev containers run in isolation from the host system, which improves security and reduces the chances of conflicts with other software installed on the host system.
+- **Reproducibility**: Dev containers can be version-controlled, making it easy to reproduce the development environment at any point in time. It also allows developers to roll back to an earlier environment version if necessary.
+
+## Features
+
+- [Rust](https://www.rust-lang.org/) engine and common Rust utilities
+- [Python](https://www.python.org/) 3.11
+- [Docker-in-Docker](https://github.com/devcontainers/features/tree/main/src/docker-in-docker) (DinD)
+- [Shell History](https://github.com/stuartleeks/dev-container-features/tree/main/src/shell-history)
+- [Local Git hooks](https://www.pantsbuild.org/stable/docs/contributions/development/setting-up-pants#step-3-set-up-a-pre-push-git-hook)
+- Useful VS Code extensions like `Python`, `Pylance`, `Black Formatter`, `rust-analyser`, `Even Better TOML`, etc.
+- Volumes for Pants [cache directories](https://www.pantsbuild.org/stable/docs/using-pants/using-pants-in-ci#directories-to-cache)
+- `hyperfine`, `py-sy`, `memray` and `dbg` for [debugging and benchmarking](https://www.pantsbuild.org/stable/docs/contributions/development/debugging-and-benchmarking)
+
+## Getting started
+
+You need three (3) things to get started with development containers:
+
+- [VS Code](https://code.visualstudio.com/)
+- [Docker](https://www.docker.com/)
+- [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension for VS Code
+
+More on getting started can be found [here](https://code.visualstudio.com/docs/devcontainers/containers#_getting-started).
+
+## Using the Dev Container
+
+1. After forking and cloning `pantsbuild/pants`, start VS Code and run the `Dev Containers: Open Folder in Container...` command from the *Command Palette...* (`F1`) and select the Pants project folder.
+2. The VS Code window (instance) will reload and start building the dev container. A progress notification provides status updates.
+3. After the build completes, VS Code will automatically connect to the container. You can now work with the repository source code in this independent environment.
+4. Pants sets up its development virtualenv at `~/.cache/pants/pants_dev_deps/<venv_fingerprint>.venv/`. Point VS Code to the `bin/python` file in this folder by running the `Python: Select Interpreter` command from the *Command Palette...* (`F1`), and then `Enter interpreter path...`. You may need to restart your *terminal*. See [Configure your IDE (optional)](https://www.pantsbuild.org/docs/contributor-setup#configure-your-ide-optional).
+
+More on starting a dev container can be found [here](https://code.visualstudio.com/docs/devcontainers/containers#_picking-your-quick-start).

.devcontainer/devcontainer.json

@@ -0,0 +1,76 @@
+{
+    "name": "Pants Dev Container",
+    "build": {
+      "context": ".",
+      "dockerfile": "./Dockerfile"
+    },
+    "features": {
+      "ghcr.io/devcontainers/features/docker-in-docker:2": {},
+      "ghcr.io/devcontainers/features/python:1": {
+        "version": "3.11.10"
+      },
+      "ghcr.io/stuartleeks/dev-container-features/shell-history:0": {}
+    },
+    "customizations": {
+      "vscode": {
+        "extensions": [
+          // VS Code extensions installed by the Python feature.
+          // See: https://github.com/devcontainers/features/tree/main/src/python#vs-code-extensions
+          // "ms-python.python",
+          // "ms-python.vscode-pylance",
+          // "ms-python.autopep8",
+          // Others useful extensions for Python and Rust.
+          "ms-python.black-formatter",
+          "ms-python.flake8",
+          "ms-python.mypy-type-checker",
+          "rust-lang.rust-analyzer",
+          "vadimcn.vscode-lldb",
+          "tamasfe.even-better-toml",
+          "unifiedjs.vscode-mdx"
+        ],
+        "settings": {
+          "[python]": {
+            "editor.defaultFormatter": "ms-python.black-formatter",
+            "editor.formatOnSave": true
+          },
+          "[rust]": {
+            "editor.defaultFormatter": "rust-lang.rust-analyzer",
+            "editor.formatOnSave": true
+          },
+          "python.analysis.extraPaths": ["src/python"],
+          "black-formatter.args": ["--config=pyproject.toml"],
+          "flake8.args": ["--config=build-support/flake8/.flake8"],
+          "flake8.path": ["dist/export/python/virtualenvs/flake8/3.11.10/bin/flake8"],
+          "mypy-type-checker.args": ["--config-file=pyproject.toml"],
+          "mypy-type-checker.path": ["dist/export/python/virtualenvs/mypy/3.11.10/bin/mypy"],
+          "rust-analyzer.linkedProjects": ["src/rust/engine/Cargo.toml"]
+        }
+      }
+    },
+    "remoteUser": "vscode",
+    "mounts": [
+      // DinD feature mounts `/tmp` to `tmpfs` by default, limiting the `/tmp` folder size
+      // to the amount of RAM that is available, which results in `Out of space Exception`
+      // when Pants runs. Temporary fix until issue #471 is resolved.
+      // See: https://github.com/devcontainers/features/issues/471
+      {
+        "target": "/tmp",
+        "type": "volume"
+      },
+      // Mount cache directories used by the Pants launcher binary
+      // and some underlying tools.
+      // See: https://www.pantsbuild.org/stable/docs/using-pants/using-pants-in-ci#directories-to-cache
+      {
+        "source": "pants-named_caches",
+        "target": "/home/vscode/.cache/pants/named_caches",
+        "type": "volume"
+      },
+      {
+        "source": "pants-lmdb_store",
+        "target": "/home/vscode/.cache/pants/lmdb_store",
+        "type": "volume"
+      }
+    ],
+    "postCreateCommand": ".devcontainer/postCreateCommand.sh",
+    "postStartCommand": ".devcontainer/postStartCommand.sh"
+  }

.devcontainer/postCreateCommand.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Since this repository runs VS Code as the non-root "vscode" user,
+# we need to be sure the user can access the folder.
+# See: https://code.visualstudio.com/remote/advancedcontainers/improve-performance#_use-a-targeted-named-volume
+sudo chown -R vscode:vscode /home/vscode/.cache
+
+# Install local git hooks.
+# See: https://www.pantsbuild.org/stable/docs/contributions/development/setting-up-pants#step-3-set-up-a-pre-push-git-hook
+build-support/bin/setup.sh
+
+# Install `py-sy` and `memray` for CPU and Memory profiling.
+# See: https://www.pantsbuild.org/stable/docs/contributions/development/debugging-and-benchmarking
+pip install py-spy memray
+
+# Bootstrap Pants.
+./pants --version

.devcontainer/postStartCommand.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Export resolves used by VS Code extensions.
+resolves=(flake8 mypy)
+for r in ${resolves[@]}; do
+    sha256sum_file=dist/$r-lockfile.sha256
+    if [[ ! $(sha256sum -c $sha256sum_file) ]]; then
+        ./pants export --resolve=$r
+        sha256sum 3rdparty/python/$r.lock > $sha256sum_file
+    fi
+done

.github/FUNDING.yml

@@ -0,0 +1 @@
+Please visit [our sponsorship page](https://www.pantsbuild.org/sponsorship) for information on how to support the Pants Build project.

.github/runs-on.yml

@@ -0,0 +1,6 @@
+images:
+  ubuntu22-full-arm64-python3.7-3.13:
+    platform: "linux"
+    arch: "arm64"
+    owner: "408085265505"
+    ami: "ami-0718bef309678bb83"

.github/workflows/audit.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 10
     - name: Cargo audit (for security vulnerabilities)

.github/workflows/auto-cherry-picker.yaml

@@ -73,7 +73,27 @@ jobs:
           # NB: If this fetch fails, we assume that means the milestone branch hasn't been created yet
           git fetch --depth 1 origin "${{ matrix.milestone }}" || exit 0
           git checkout -b "${{ matrix.branch_name }}" FETCH_HEAD
+
+          set +e
           git cherry-pick "${{ needs.prerequisites.outputs.merge_commit }}"
+          cherry_pick_exit_code="$?"
+          set -e
+          if [ "$cherry_pick_exit_code" != 0 ]; then
+            # Cherry pick failed. See if it was from a release notes file for a subsequent version of Pants,
+            # and, if so, delete the file and continue the `git cherry-pick`.
+            this_minor_version="$(cat ./src/python/pants/VERSION | cut -d. -f2)"
+            for conflicting_file in "$(git status --porcelain=v1 -- docs/notes)" ; do
+              conflicting_minor_version="$(echo $conflicting_file | sed -E -e 's#^.*docs/notes/2\.([0-9]+)\.x\.md$#\1#')"
+              if [[ conflicting_minor_version -gt this_minor_version ]]; then
+                echo "NOTE: Deleting release notes file from subsequent Pants version (v2.{conflicting_minor_version}.x)."
+                git rm "docs/notes/2.${conflicting_minor_version}.x.md"
+              fi
+            done
+
+            # With the "future" release notes removed, attempt to continue the cherry-pick operation.
+            git cherry-pick --continue
+          fi
+
           git push -u origin "${{ matrix.branch_name }}"
 
           # Now we go back to `main` to ensure we're running the latest `make_pr.sh`.

.github/workflows/cache_comparison.yaml

@@ -8,15 +8,25 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 10
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
+    - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.12, 3.13, 3.11
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
-    - name: Expose Pythons
-      uses: pantsbuild/actions/expose-pythons@627a8ce25d972afa03da1641be9261bbbe0e3ffe
+        python-version: '3.7
+
+          3.8
+
+          3.9
+
+          3.10
+
+          3.12
+
+          3.13
+
+          3.11'
     - env:
         BASE_REF: ${{ github.event.inputs.base_ref }}
         BUILD_COMMIT: ${{ github.event.inputs.build_commit }}

.github/workflows/clear_self_hosted_persistent_caches.yaml

@@ -0,0 +1,31 @@
+# GENERATED, DO NOT EDIT!
+# To change, edit `src/python/pants_release/generate_github_workflows.py` and run:
+#   ./pants run src/python/pants_release/generate_github_workflows.py
+
+
+jobs:
+  clean_linux_arm64:
+    runs-on:
+    - self-hosted
+    - runs-on
+    - runner=4cpu-linux-arm64
+    - image=ubuntu22-full-arm64-python3.7-3.13
+    - run-id=${{ github.run_id }}
+    steps:
+    - name: df before
+      run: df -h
+    - name: Deleting ~/Library/Caches
+      run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches || true
+    - name: Deleting ~/.cache
+      run: du -sh ~/.cache || true; rm -rf ~/.cache || true
+    - name: Deleting ~/.nce
+      run: du -sh ~/.nce || true; rm -rf ~/.nce || true
+    - name: Deleting ~/.rustup
+      run: du -sh ~/.rustup || true; rm -rf ~/.rustup || true
+    - name: Deleting ~/.pex
+      run: du -sh ~/.pex || true; rm -rf ~/.pex || true
+    - name: df after
+      run: df -h
+name: Clear persistent caches on long-lived self-hosted runners
+'on':
+  workflow_dispatch: {}