Skip to content

Commit

Permalink
Add free-threaded Python support (#2310)
Browse files Browse the repository at this point in the history
Co-authored-by: Nathan Goldbaum <[email protected]>
messense and ngoldbaum authored Nov 25, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent ceaefd5 commit 65404af
Showing 8 changed files with 184 additions and 55 deletions.
16 changes: 10 additions & 6 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ on:
workflow_dispatch:

concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }}
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.action }}-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
@@ -43,9 +43,10 @@ jobs:
- macos-14
- windows-latest
python-version:
- '3.9'
- '3.13'
- 'pypy3.10'
- "3.9"
- "3.13"
- "3.13t"
- "pypy3.10"
exclude:
# Skip PyPy on macOS M1 runner because they are built for x86_64
- os: macos-14
@@ -81,7 +82,7 @@ jobs:
auto-activate-base: "false"
activate-environment: ""
miniconda-version: "latest"
- uses: actions/setup-python@v5
- uses: Quansight-Labs/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Set PYTHON_VERSION env var
@@ -96,7 +97,10 @@ jobs:
run: |
uv tool install virtualenv
uv tool install twine
uv pip install --system "ziglang>=0.10.0,<0.13.0" uniffi-bindgen==0.28.0
uv tool install uniffi-bindgen==0.28.0
- uses: mlugg/setup-zig@v1
with:
version: 0.12.1
- uses: dtolnay/rust-toolchain@stable
id: rustup
with:
2 changes: 1 addition & 1 deletion src/auditwheel/audit.rs
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ use std::{fmt, io};
use thiserror::Error;

static IS_LIBPYTHON: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^libpython3\.\d+m?u?\.so\.\d+\.\d+$").unwrap());
Lazy::new(|| Regex::new(r"^libpython3\.\d+m?u?t?\.so\.\d+\.\d+$").unwrap());

/// Error raised during auditing an elf file for manylinux/musllinux compatibility
#[derive(Error, Debug)]
38 changes: 30 additions & 8 deletions src/build_options.rs
Original file line number Diff line number Diff line change
@@ -274,6 +274,10 @@ impl BuildOptions {
.get("ABIFLAGS")
.map(ToString::to_string)
.unwrap_or_default();
let gil_disabled = sysconfig_data
.get("Py_GIL_DISABLED")
.map(|x| x == "1")
.unwrap_or_default();
let ext_suffix = sysconfig_data
.get("EXT_SUFFIX")
.context("syconfig didn't define an `EXT_SUFFIX` ಠ_ಠ")?;
@@ -299,6 +303,7 @@ impl BuildOptions {
abiflags,
ext_suffix: ext_suffix.to_string(),
pointer_width: None,
gil_disabled,
},
executable: PathBuf::new(),
platform: None,
@@ -376,6 +381,7 @@ impl BuildOptions {
abiflags: "".to_string(),
ext_suffix: ".pyd".to_string(),
pointer_width: None,
gil_disabled: false,
},
executable: PathBuf::new(),
platform: None,
@@ -402,6 +408,7 @@ impl BuildOptions {
abiflags: "".to_string(),
ext_suffix: ".pyd".to_string(),
pointer_width: None,
gil_disabled: false,
},
executable: PathBuf::new(),
platform: None,
@@ -441,6 +448,7 @@ impl BuildOptions {
abiflags: "".to_string(),
ext_suffix: "".to_string(),
pointer_width: None,
gil_disabled: false,
},
executable: PathBuf::new(),
platform: None,
@@ -1244,26 +1252,35 @@ fn find_interpreter_in_sysconfig(
let mut interpreters = Vec::new();
for interp in interpreter {
let python = interp.display().to_string();
let (python_impl, python_ver) = if let Some(ver) = python.strip_prefix("pypy") {
(InterpreterKind::PyPy, ver.strip_prefix('-').unwrap_or(ver))
} else if let Some(ver) = python.strip_prefix("graalpy") {
let (python_impl, python_ver, abiflags) = if let Some(ver) = python.strip_prefix("pypy") {
(
InterpreterKind::GraalPy,
InterpreterKind::PyPy,
ver.strip_prefix('-').unwrap_or(ver),
"",
)
} else if let Some(ver) = python.strip_prefix("python") {
} else if let Some(ver) = python.strip_prefix("graalpy") {
(
InterpreterKind::CPython,
InterpreterKind::GraalPy,
ver.strip_prefix('-').unwrap_or(ver),
"",
)
} else if let Some(ver) = python.strip_prefix("python") {
// Also accept things like `python3.13t` for free-threaded python
let (ver, abiflags) =
if let Some(ver) = ver.strip_prefix('-').unwrap_or(ver).strip_suffix('t') {
(ver, "t")
} else {
(ver, "")
};
(InterpreterKind::CPython, ver, abiflags)
} else if python
.chars()
.next()
.map(|c| c.is_ascii_digit())
.unwrap_or(false)
{
// Eg: -i 3.9 without interpreter kind, assume it's CPython
(InterpreterKind::CPython, &*python)
(InterpreterKind::CPython, &*python, "")
} else {
// if interpreter not known
if std::path::Path::new(&python).is_file() {
@@ -1284,7 +1301,12 @@ fn find_interpreter_in_sysconfig(
let ver_minor = ver_minor.parse::<usize>().with_context(|| {
format!("Invalid python interpreter minor version '{ver_minor}', expect a digit")
})?;
let sysconfig = InterpreterConfig::lookup_one(target, python_impl, (ver_major, ver_minor))

if (ver_major, ver_minor) < (3, 13) && abiflags == "t" {
bail!("Free-threaded Python interpreter is only supported on 3.13 and later.");
}

let sysconfig = InterpreterConfig::lookup_one(target, python_impl, (ver_major, ver_minor), abiflags)
.with_context(|| {
format!("Failed to find a {python_impl} {ver_major}.{ver_minor} interpreter in known sysconfig")
})?;
1 change: 1 addition & 0 deletions src/cross_compile.rs
Original file line number Diff line number Diff line change
@@ -52,6 +52,7 @@ KEYS = [
"ABIFLAGS",
"EXT_SUFFIX",
"SOABI",
"Py_GIL_DISABLED",
]
for key in KEYS:
print(key, build_time_vars.get(key, ""))
149 changes: 122 additions & 27 deletions src/python_interpreter/config.rs

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions src/python_interpreter/get_interpreter_metadata.py
Original file line number Diff line number Diff line change
@@ -35,6 +35,7 @@
"system": platform.system().lower(),
# This one is for generating a config file for pyo3
"pointer_width": struct.calcsize("P") * 8,
"gil_disabled": sysconfig.get_config_var("Py_GIL_DISABLED") == 1,
}

print(json.dumps(metadata))
31 changes: 19 additions & 12 deletions src/python_interpreter/mod.rs
Original file line number Diff line number Diff line change
@@ -245,7 +245,13 @@ fn find_all_windows(
Ok(interpreter)
}

fn windows_python_info(executable: &Path) -> Result<Option<InterpreterConfig>> {
struct WindowsPythonInfo {
major: usize,
minor: usize,
pointer_width: Option<usize>,
}

fn windows_python_info(executable: &Path) -> Result<Option<WindowsPythonInfo>> {
let python_info = Command::new(executable)
.arg("-c")
.arg("import sys; print(sys.version)")
@@ -276,12 +282,9 @@ fn windows_python_info(executable: &Path) -> Result<Option<InterpreterConfig>> {
} else {
32
};
Ok(Some(InterpreterConfig {
Ok(Some(WindowsPythonInfo {
major,
minor,
interpreter_kind: InterpreterKind::CPython,
abiflags: String::new(),
ext_suffix: String::new(),
pointer_width: Some(pointer_width),
}))
} else {
@@ -353,6 +356,7 @@ struct InterpreterMetadataMessage {
// comes from `platform.system()`
system: String,
soabi: Option<String>,
gil_disabled: bool,
}

/// The location and version of an interpreter
@@ -425,19 +429,19 @@ fn fun_with_abiflags(
if matches!(message.abiflags.as_deref(), Some("") | None) {
Ok("".to_string())
} else {
bail!("A python 3 interpreter on windows does not define abiflags in its sysconfig ಠ_ಠ")
bail!("A python 3 interpreter on Windows does not define abiflags in its sysconfig ಠ_ಠ")
}
} else if let Some(ref abiflags) = message.abiflags {
if message.minor >= 8 {
// for 3.8, "builds with and without pymalloc are ABI compatible" and the flag dropped
Ok(abiflags.to_string())
} else if (abiflags != "m") && (abiflags != "dm") {
bail!("A python 3 interpreter on linux or mac os must have 'm' or 'dm' as abiflags ಠ_ಠ")
bail!("A python 3 interpreter on Linux or macOS must have 'm' or 'dm' as abiflags ಠ_ಠ")
} else {
Ok(abiflags.to_string())
}
} else {
bail!("A python 3 interpreter on linux or mac os must define abiflags in its sysconfig ಠ_ಠ")
bail!("A python 3 interpreter on Linux or macOS must define abiflags in its sysconfig ಠ_ಠ")
}
}

@@ -448,7 +452,8 @@ impl PythonInterpreter {
false
} else {
match self.interpreter_kind {
InterpreterKind::CPython => true,
// Free-threaded python does not have stable api support yet
InterpreterKind::CPython => !self.config.gil_disabled,
InterpreterKind::PyPy | InterpreterKind::GraalPy => false,
}
}
@@ -695,6 +700,7 @@ impl PythonInterpreter {
.ext_suffix
.context("syconfig didn't define an `EXT_SUFFIX` ಠ_ಠ")?,
pointer_width: None,
gil_disabled: message.gil_disabled,
},
executable,
platform,
@@ -989,19 +995,19 @@ mod tests {
let target =
Target::from_target_triple(Some("x86_64-unknown-linux-gnu".to_string())).unwrap();
let pythons = PythonInterpreter::find_by_target(&target, None);
assert_eq!(pythons.len(), 11);
assert_eq!(pythons.len(), 12);

let pythons = PythonInterpreter::find_by_target(
&target,
Some(&VersionSpecifiers::from_str(">=3.8").unwrap()),
);
assert_eq!(pythons.len(), 9);
assert_eq!(pythons.len(), 10);

let pythons = PythonInterpreter::find_by_target(
&target,
Some(&VersionSpecifiers::from_str(">=3.10").unwrap()),
);
assert_eq!(pythons.len(), 5);
assert_eq!(pythons.len(), 6);
}

#[test]
@@ -1010,6 +1016,7 @@ mod tests {
(".cpython-37m-x86_64-linux-gnu.so", Some("cp37m")),
(".cpython-310-x86_64-linux-gnu.so", Some("cp310")),
(".cpython-310-darwin.so", Some("cp310")),
(".cpython-313t-darwin.so", Some("cp313t")),
(".cp310-win_amd64.pyd", Some("cp310")),
(".cp39-mingw_x86_64.pyd", Some("cp39")),
(".cpython-312-wasm32-wasi.so", Some("cp312")),
1 change: 0 additions & 1 deletion tests/common/integration.rs
Original file line number Diff line number Diff line change
@@ -94,7 +94,6 @@ pub fn test_integration(
let file = File::create(venvs_dir.join("cffi-provider.lock"))?;
file.file().lock_exclusive()?;
if !dbg!(venvs_dir.join(cffi_provider)).is_dir() {
println!("CRAETAITROAPTGIIDSOGRKADS");
dbg!(create_virtualenv_name(
cffi_provider,
python_interp.clone().map(PathBuf::from)

0 comments on commit 65404af

Please sign in to comment.