diff --git a/.github/setup-rust-windows/action.yml b/.github/setup-rust-windows/action.yml new file mode 100644 index 00000000000000..1ad913e4e9b921 --- /dev/null +++ b/.github/setup-rust-windows/action.yml @@ -0,0 +1,39 @@ +name: Set up Rust and LLVM for Windows +description: >- + Install the pinned Rust toolchain with Windows cross-compile targets and + a version of LLVM whose libclang is loadable by bindgen. + +inputs: + arch: + description: CPU architecture (x64, Win32, arm64) + required: true + +runs: + using: composite + steps: + - uses: dtolnay/rust-toolchain@f133eefe930d61f0d9371efd474daf0125ed3dd1 # 1.95.0 + with: + toolchain: 1.95.0 + targets: i686-pc-windows-msvc,x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + # LIBCLANG_PATH must be set explicitly so the vcxproj uses the LLVM we + # install here rather than the VS-bundled LLVM whose clang headers have + # AVX intrinsic bugs. The values are hardcoded literals (no attacker- + # controlled input), so the GITHUB_ENV writes are safe. + - name: Install LLVM for bindgen + if: inputs.arch != 'arm64' + shell: pwsh + run: | # zizmor: ignore[github-env] + choco install llvm --allow-downgrade --no-progress --version 21.1.0 + if (!(Test-Path 'C:\Program Files\LLVM\bin\libclang.dll')) { exit 1 } + echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV + # Chocolatey's LLVM package only ships x64 binaries, which an ARM64-native + # cargo process cannot load. Install the official ARM64 build directly. + - name: Install LLVM for bindgen (ARM64) + if: inputs.arch == 'arm64' + shell: pwsh + run: | # zizmor: ignore[github-env] + $installer = Join-Path $env:RUNNER_TEMP 'LLVM-21.1.0-woa64.exe' + Invoke-WebRequest 'https://github.com/llvm/llvm-project/releases/download/llvmorg-21.1.0/LLVM-21.1.0-woa64.exe' -OutFile $installer + Start-Process -Wait -FilePath $installer -ArgumentList '/S','/D=C:\Program Files\LLVM' + if (!(Test-Path 'C:\Program Files\LLVM\bin\libclang.dll')) { exit 1 } + echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07087b836cb597..d487b18f32e65a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -342,6 +342,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - uses: dtolnay/rust-toolchain@f133eefe930d61f0d9371efd474daf0125ed3dd1 # 1.95.0 + with: + toolchain: 1.95.0 + targets: ${{ matrix.arch }}-linux-android - name: Build and test run: python3 Platforms/Android ci --fast-ci ${{ matrix.arch }}-linux-android @@ -363,7 +367,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - + - uses: dtolnay/rust-toolchain@f133eefe930d61f0d9371efd474daf0125ed3dd1 # 1.95.0 + with: + toolchain: 1.95.0 + targets: aarch64-apple-ios-sim - name: Build and test run: >- python3 Platforms/Apple ci iOS --fast-ci diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml new file mode 100644 index 00000000000000..c4f2740993cb86 --- /dev/null +++ b/.github/workflows/cargo.yml @@ -0,0 +1,27 @@ +name: cargo + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + timeout-minutes: 15 + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + with: + toolchain: stable + - run: ./configure + - run: cargo test + - run: cargo fmt --check + - run: cargo clippy diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index d093c86eb2db96..a721dcf5050e3b 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -55,6 +55,10 @@ jobs: uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.x' + - uses: dtolnay/rust-toolchain@f133eefe930d61f0d9371efd474daf0125ed3dd1 # 1.95.0 + with: + toolchain: 1.95.0 + targets: wasm32-unknown-emscripten - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: "Install Emscripten" diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 12dd790897ad39..ca9bf64ea8229d 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -46,6 +46,11 @@ jobs: uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.x' + # wasm32-wasip3 has prebuilt std starting with Rust 1.100. + - uses: dtolnay/rust-toolchain@9522411a9bce8484da960823fc2f46e52f981f71 # 1.100.0 + with: + toolchain: 1.100.0 + targets: wasm32-wasip3 - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: "Configure build Python" diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 4be4bc05fa6d0d..2921aa9bfe886a 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -34,6 +34,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - uses: ./.github/setup-rust-windows + with: + arch: ${{ inputs.arch }} - name: Register MSVC problem matcher if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" diff --git a/.gitignore b/.gitignore index 91659803f5faf2..03ef6e438af5f3 100644 --- a/.gitignore +++ b/.gitignore @@ -195,3 +195,6 @@ CLAUDE.local.md #### main branch only stuff below this line, things to backport go above. #### # main branch only: ABI files are not checked/maintained. Doc/data/python*.abi + +# Rust build artifacts +/target/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000000000..14e6791704c196 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,253 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "_base64" +version = "0.1.0" +dependencies = [ + "cpython-build-helper", + "cpython-sys", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cpython-build-helper" +version = "0.1.0" +dependencies = [ + "shlex", +] + +[[package]] +name = "cpython-rust-staticlib" +version = "0.1.0" +dependencies = [ + "_base64", +] + +[[package]] +name = "cpython-sys" +version = "0.1.0" +dependencies = [ + "bindgen", + "prettyplease", + "shlex", + "syn", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000000000..0eecc3f7643a67 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,10 @@ +[workspace] +resolver = "3" +members = [ + "Modules/_base64", "Modules/cpython-build-helper", "Modules/cpython-rust-staticlib", + "Modules/cpython-sys" +] + +[workspace.package] +# Make sure to also update the Rust toolchains in GitHub Actions whenever this version is modified +rust-version = "1.95" diff --git a/Makefile.pre.in b/Makefile.pre.in index 1d109dac77fa90..011fd392fbbd63 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -66,6 +66,14 @@ DTRACE_HEADERS= @DTRACE_HEADERS@ DTRACE_OBJS= @DTRACE_OBJS@ DSYMUTIL= @DSYMUTIL@ DSYMUTIL_PATH= @DSYMUTIL_PATH@ +CARGO=@CARGO@ +CARGO_TARGET_DIR=@CARGO_TARGET_DIR@ +CARGO_PROFILE=@CARGO_PROFILE@ +CARGO_TARGET=@CARGO_TARGET@ +CARGO_TARGET_LINKER_ENV=@CARGO_TARGET_LINKER_ENV@ +CARGO_DYLIB_SUFFIX=@CARGO_DYLIB_SUFFIX@ +LLVM_TARGET=@LLVM_TARGET@ +RUST_STATICLIB_DEP=@RUST_STATICLIB_DEP@ GNULD= @GNULD@ @@ -188,7 +196,8 @@ CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) SHLIB_SUFFIX= @SHLIB_SUFFIX@ EXT_SUFFIX= @EXT_SUFFIX@ LDSHARED= @LDSHARED@ $(PY_LDFLAGS) -BLDSHARED= @BLDSHARED@ $(PY_CORE_LDFLAGS) +BLDSHARED_EXE= @BLDSHARED_EXE@ +BLDSHARED_ARGS= @BLDSHARED_ARGS@ $(PY_CORE_LDFLAGS) LDCXXSHARED= @LDCXXSHARED@ $(PY_LDFLAGS) DESTSHARED= $(BINLIBDEST)/lib-dynload @@ -1010,7 +1019,7 @@ clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c PLATFORM_MAIN_OBJS= @PLATFORM_MAIN_OBJS@ # Build the interpreter -$(BUILDPYTHON): Programs/python.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_DEPS) +$(BUILDPYTHON): Programs/python.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_DEPS) $(RUST_STATICLIB_DEP) $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt @@ -1043,16 +1052,16 @@ $(LIBRARY): $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) # AIX Linker don't support "-h" option if test "$(MACHDEP)" != "aix"; then \ - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED_EXE) $(BLDSHARED_ARGS) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ else \ - $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED_EXE) $(BLDSHARED_ARGS) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ fi if test $(INSTSONAME) != $@; then \ $(LN) -f $(INSTSONAME) $@; \ fi libpython3.so: libpython$(LDVERSION).so - $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ + $(BLDSHARED_EXE) $(BLDSHARED_ARGS) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) $(CC) -dynamiclib $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ @@ -3400,8 +3409,15 @@ clean-profile: clean-retain-profile clean-bolt clean-jit-stencils: -rm -f $(JIT_TARGETS) $(JIT_GENERATED_STAMP) jit_stencils*.h jit_unwind_info*.h jit_shim*.o +.PHONY: clean-rust +clean-rust: + @if test '$(CARGO)' != 'no'; then \ + echo Running cargo clean...; \ + $(CARGO) clean; \ + fi + .PHONY: clean -clean: clean-profile clean-jit-stencils +clean: clean-profile clean-jit-stencils clean-rust .PHONY: clobber clobber: clean @@ -3509,6 +3525,14 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h ########################################################################## # Module dependencies and platform-specific files +cpython-sys: Modules/cpython-sys/Cargo.toml Modules/cpython-sys/build.rs Modules/cpython-sys/wrapper.h Modules/cpython-sys/parser.h + CARGO_TARGET_DIR=$(abs_builddir)/target PYTHON_BUILD_DIR=$(abs_builddir) PY_CC="$(CC)" PY_CPPFLAGS="$(CPPFLAGS)" PY_CFLAGS="$(CFLAGS)" LLVM_TARGET="$(LLVM_TARGET)" IPHONEOS_DEPLOYMENT_TARGET=$(IPHONEOS_DEPLOYMENT_TARGET) $(CARGO) build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE) $(if $(CARGO_TARGET),--target=$(CARGO_TARGET)) --manifest-path $(srcdir)/Cargo.toml + +RUST_STATICLIB_A= target/$(if $(CARGO_TARGET),$(CARGO_TARGET)/$(CARGO_TARGET_DIR),$(CARGO_TARGET_DIR))/libcpython_rust_staticlib.a + +cpython-rust-staticlib: cpython-sys + CARGO_TARGET_DIR=$(abs_builddir)/target PYTHON_BUILD_DIR=$(abs_builddir) $(CARGO) build --lib --locked --package cpython-rust-staticlib --profile $(CARGO_PROFILE) $(if $(CARGO_TARGET),--target=$(CARGO_TARGET)) --manifest-path $(srcdir)/Cargo.toml + # force rebuild when header file or module build flavor (static/shared) is changed MODULE_DEPS_STATIC=Modules/config.c MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@ diff --git a/Misc/ACKS b/Misc/ACKS index 4178ae3108128a..bc956e70e28cf1 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1209,6 +1209,7 @@ Nick Maclaren Don MacMillen Tomasz Maćkowiak Wolfgang Maier +Paul Mairo Steve Majewski Marek Majkowski Grzegorz Makarewicz diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c index 6c1863c943b353..2af962544dc487 100644 --- a/Misc/platform_triplet.c +++ b/Misc/platform_triplet.c @@ -1,5 +1,12 @@ /* Detect platform triplet from builtin defines * cc -E Misc/platform_triplet.c | grep '^PLATFORM_TRIPLET=' | tr -d ' ' + * + * Also detects the closest Rust target triple for cargo/rustc: + * cc -E Misc/platform_triplet.c | grep '^RUST_TARGET=' | tr -d ' ' + * + * And the LLVM/clang target triple (for bindgen), only when it differs + * from RUST_TARGET: + * cc -E Misc/platform_triplet.c | grep '^LLVM_TARGET=' | tr -d ' ' */ #undef bfin #undef cris @@ -16,12 +23,16 @@ #if defined(__ANDROID__) # if defined(__x86_64__) PLATFORM_TRIPLET=x86_64-linux-android +RUST_TARGET=x86_64-linux-android # elif defined(__i386__) PLATFORM_TRIPLET=i686-linux-android +RUST_TARGET=i686-linux-android # elif defined(__aarch64__) PLATFORM_TRIPLET=aarch64-linux-android +RUST_TARGET=aarch64-linux-android # elif defined(__arm__) PLATFORM_TRIPLET=arm-linux-androideabi +RUST_TARGET=arm-linux-androideabi # else # error unknown Android platform # endif @@ -39,18 +50,25 @@ PLATFORM_TRIPLET=arm-linux-androideabi # elif defined(__GLIBC__) # define LIBC gnu # define LIBC_X32 gnux32 +# define RUST_LIBC gnu +# define RUST_LIBC_X32 gnux32 # if defined(__ARM_PCS_VFP) # define LIBC_ARM gnueabihf +# define RUST_LIBC_ARM gnueabihf # else # define LIBC_ARM gnueabi +# define RUST_LIBC_ARM gnueabi # endif # if defined(__loongarch__) # if defined(__loongarch_soft_float) # define LIBC_LA gnusf +# define RUST_LIBC_LA unknown # elif defined(__loongarch_single_float) # define LIBC_LA gnuf32 +# define RUST_LIBC_LA unknown # elif defined(__loongarch_double_float) # define LIBC_LA gnu +# define RUST_LIBC_LA gnu # else # error unknown loongarch floating-point base abi # endif @@ -59,20 +77,26 @@ PLATFORM_TRIPLET=arm-linux-androideabi # if defined(__mips_hard_float) # if defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define LIBC_MIPS gnu +# define RUST_LIBC_MIPS gnu # elif defined(_ABIN32) && _MIPS_SIM == _ABIN32 # define LIBC_MIPS gnuabin32 +# define RUST_LIBC_MIPS unknown # elif defined(_ABI64) && _MIPS_SIM == _ABI64 # define LIBC_MIPS gnuabi64 +# define RUST_LIBC_MIPS gnuabi64 # else # error unknown mips sim value # endif # else # if defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define LIBC_MIPS gnusf +# define RUST_LIBC_MIPS unknown # elif defined(_ABIN32) && _MIPS_SIM == _ABIN32 # define LIBC_MIPS gnuabin32sf +# define RUST_LIBC_MIPS unknown # elif defined(_ABI64) && _MIPS_SIM == _ABI64 # define LIBC_MIPS gnuabi64sf +# define RUST_LIBC_MIPS unknown # else # error unknown mips sim value # endif @@ -80,8 +104,10 @@ PLATFORM_TRIPLET=arm-linux-androideabi # endif # if defined(__SPE__) # define LIBC_PPC gnuspe +# define RUST_LIBC_PPC gnuspe # else # define LIBC_PPC gnu +# define RUST_LIBC_PPC gnu # endif # else // Heuristic to detect musl libc @@ -89,18 +115,25 @@ PLATFORM_TRIPLET=arm-linux-androideabi # ifdef __DEFINED_va_list # define LIBC musl # define LIBC_X32 muslx32 +# define RUST_LIBC musl +# define RUST_LIBC_X32 unknown # if defined(__ARM_PCS_VFP) # define LIBC_ARM musleabihf +# define RUST_LIBC_ARM musleabihf # else # define LIBC_ARM musleabi +# define RUST_LIBC_ARM musleabi # endif # if defined(__loongarch__) # if defined(__loongarch_soft_float) # define LIBC_LA muslsf +# define RUST_LIBC_LA unknown # elif defined(__loongarch_single_float) # define LIBC_LA muslf32 +# define RUST_LIBC_LA unknown # elif defined(__loongarch_double_float) # define LIBC_LA musl +# define RUST_LIBC_LA musl # else # error unknown loongarch floating-point base abi # endif @@ -109,20 +142,27 @@ PLATFORM_TRIPLET=arm-linux-androideabi # if defined(__mips_hard_float) # if defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define LIBC_MIPS musl +# define RUST_LIBC_MIPS musl # elif defined(_ABIN32) && _MIPS_SIM == _ABIN32 # define LIBC_MIPS musln32 +# define RUST_LIBC_MIPS unknown # elif defined(_ABI64) && _MIPS_SIM == _ABI64 # define LIBC_MIPS musl +# define RUST_LIBC_MIPS muslabi64 +# define LLVM_LIBC_MIPS musl # else # error unknown mips sim value # endif # else # if defined(_ABIO32) && _MIPS_SIM == _ABIO32 # define LIBC_MIPS muslsf +# define RUST_LIBC_MIPS unknown # elif defined(_ABIN32) && _MIPS_SIM == _ABIN32 # define LIBC_MIPS musln32sf +# define RUST_LIBC_MIPS unknown # elif defined(_ABI64) && _MIPS_SIM == _ABI64 # define LIBC_MIPS muslsf +# define RUST_LIBC_MIPS unknown # else # error unknown mips sim value # endif @@ -130,8 +170,10 @@ PLATFORM_TRIPLET=arm-linux-androideabi # endif # if defined(_SOFT_FLOAT) || defined(__NO_FPRS__) # define LIBC_PPC muslsf +# define RUST_LIBC_PPC unknown # else # define LIBC_PPC musl +# define RUST_LIBC_PPC musl # endif # else # error unknown libc @@ -140,85 +182,126 @@ PLATFORM_TRIPLET=arm-linux-androideabi # if defined(__x86_64__) && defined(__LP64__) PLATFORM_TRIPLET=x86_64-linux-LIBC +RUST_TARGET=x86_64-unknown-linux-RUST_LIBC # elif defined(__x86_64__) && defined(__ILP32__) PLATFORM_TRIPLET=x86_64-linux-LIBC_X32 +RUST_TARGET=x86_64-unknown-linux-RUST_LIBC_X32 # elif defined(__i386__) PLATFORM_TRIPLET=i386-linux-LIBC +RUST_TARGET=i686-unknown-linux-RUST_LIBC # elif defined(__aarch64__) && defined(__AARCH64EL__) # if defined(__ILP32__) PLATFORM_TRIPLET=aarch64_ilp32-linux-LIBC +RUST_TARGET=unknown # else PLATFORM_TRIPLET=aarch64-linux-LIBC +RUST_TARGET=aarch64-unknown-linux-RUST_LIBC # endif # elif defined(__aarch64__) && defined(__AARCH64EB__) # if defined(__ILP32__) PLATFORM_TRIPLET=aarch64_be_ilp32-linux-LIBC +RUST_TARGET=unknown # else PLATFORM_TRIPLET=aarch64_be-linux-LIBC +RUST_TARGET=aarch64_be-unknown-linux-RUST_LIBC # endif # elif defined(__alpha__) PLATFORM_TRIPLET=alpha-linux-LIBC +RUST_TARGET=unknown # elif defined(__ARM_EABI__) # if defined(__ARMEL__) PLATFORM_TRIPLET=arm-linux-LIBC_ARM +RUST_TARGET=arm-unknown-linux-RUST_LIBC_ARM # else PLATFORM_TRIPLET=armeb-linux-LIBC_ARM +RUST_TARGET=armeb-unknown-linux-RUST_LIBC_ARM # endif # elif defined(__hppa__) PLATFORM_TRIPLET=hppa-linux-LIBC +RUST_TARGET=unknown # elif defined(__ia64__) PLATFORM_TRIPLET=ia64-linux-LIBC +RUST_TARGET=unknown # elif defined(__loongarch__) && defined(__loongarch_lp64) PLATFORM_TRIPLET=loongarch64-linux-LIBC_LA +RUST_TARGET=loongarch64-unknown-linux-RUST_LIBC_LA # elif defined(__m68k__) && !defined(__mcoldfire__) PLATFORM_TRIPLET=m68k-linux-LIBC +RUST_TARGET=m68k-unknown-linux-RUST_LIBC # elif defined(__mips__) # if defined(__mips_isa_rev) && (__mips_isa_rev >=6) # if defined(_MIPSEL) && defined(__mips64) PLATFORM_TRIPLET=mipsisa64r6el-linux-LIBC_MIPS +RUST_TARGET=mipsisa64r6el-unknown-linux-RUST_LIBC_MIPS # elif defined(_MIPSEL) PLATFORM_TRIPLET=mipsisa32r6el-linux-LIBC_MIPS +RUST_TARGET=mipsisa32r6el-unknown-linux-RUST_LIBC_MIPS # elif defined(__mips64) PLATFORM_TRIPLET=mipsisa64r6-linux-LIBC_MIPS +RUST_TARGET=mipsisa64r6-unknown-linux-RUST_LIBC_MIPS # else PLATFORM_TRIPLET=mipsisa32r6-linux-LIBC_MIPS +RUST_TARGET=mipsisa32r6-unknown-linux-RUST_LIBC_MIPS # endif # else # if defined(_MIPSEL) && defined(__mips64) PLATFORM_TRIPLET=mips64el-linux-LIBC_MIPS +RUST_TARGET=mips64el-unknown-linux-RUST_LIBC_MIPS +# ifdef LLVM_LIBC_MIPS +LLVM_TARGET=mips64el-unknown-linux-LLVM_LIBC_MIPS +# endif # elif defined(_MIPSEL) PLATFORM_TRIPLET=mipsel-linux-LIBC_MIPS +RUST_TARGET=mipsel-unknown-linux-RUST_LIBC_MIPS # elif defined(__mips64) PLATFORM_TRIPLET=mips64-linux-LIBC_MIPS +RUST_TARGET=mips64-unknown-linux-RUST_LIBC_MIPS +# ifdef LLVM_LIBC_MIPS +LLVM_TARGET=mips64-unknown-linux-LLVM_LIBC_MIPS +# endif # else PLATFORM_TRIPLET=mips-linux-LIBC_MIPS +RUST_TARGET=mips-unknown-linux-RUST_LIBC_MIPS # endif # endif # elif defined(__or1k__) PLATFORM_TRIPLET=or1k-linux-LIBC +RUST_TARGET=unknown # elif defined(__powerpc64__) # if defined(__LITTLE_ENDIAN__) PLATFORM_TRIPLET=powerpc64le-linux-LIBC +RUST_TARGET=powerpc64le-unknown-linux-RUST_LIBC # else PLATFORM_TRIPLET=powerpc64-linux-LIBC +RUST_TARGET=powerpc64-unknown-linux-RUST_LIBC # endif # elif defined(__powerpc__) PLATFORM_TRIPLET=powerpc-linux-LIBC_PPC +RUST_TARGET=powerpc-unknown-linux-RUST_LIBC_PPC # elif defined(__s390x__) PLATFORM_TRIPLET=s390x-linux-LIBC +RUST_TARGET=s390x-unknown-linux-RUST_LIBC # elif defined(__s390__) PLATFORM_TRIPLET=s390-linux-LIBC +RUST_TARGET=unknown # elif defined(__sh__) && defined(__LITTLE_ENDIAN__) PLATFORM_TRIPLET=sh4-linux-LIBC +RUST_TARGET=unknown # elif defined(__sparc__) && defined(__arch64__) PLATFORM_TRIPLET=sparc64-linux-LIBC +RUST_TARGET=sparc64-unknown-linux-RUST_LIBC # elif defined(__sparc__) PLATFORM_TRIPLET=sparc-linux-LIBC +RUST_TARGET=sparc-unknown-linux-RUST_LIBC # elif defined(__riscv) # if __riscv_xlen == 32 PLATFORM_TRIPLET=riscv32-linux-LIBC +RUST_TARGET=riscv32gc-unknown-linux-RUST_LIBC +LLVM_TARGET=riscv32-unknown-linux-RUST_LIBC # elif __riscv_xlen == 64 PLATFORM_TRIPLET=riscv64-linux-LIBC +RUST_TARGET=riscv64gc-unknown-linux-RUST_LIBC +LLVM_TARGET=riscv64-unknown-linux-RUST_LIBC # else # error unknown platform triplet # endif @@ -231,16 +314,20 @@ PLATFORM_TRIPLET=riscv64-linux-LIBC #elif defined(__FreeBSD_kernel__) # if defined(__LP64__) PLATFORM_TRIPLET=x86_64-kfreebsd-gnu +RUST_TARGET=unknown # elif defined(__i386__) PLATFORM_TRIPLET=i386-kfreebsd-gnu +RUST_TARGET=unknown # else # error unknown platform triplet # endif #elif defined(__gnu_hurd__) # if defined(__x86_64__) && defined(__LP64__) PLATFORM_TRIPLET=x86_64-gnu +RUST_TARGET=x86_64-unknown-hurd-gnu # elif defined(__i386__) PLATFORM_TRIPLET=i386-gnu +RUST_TARGET=i686-unknown-hurd-gnu # else # error unknown platform triplet # endif @@ -251,11 +338,17 @@ PLATFORM_TRIPLET=i386-gnu # if defined(TARGET_OS_SIMULATOR) && TARGET_OS_SIMULATOR # if __x86_64__ PLATFORM_TRIPLET=x86_64-iphonesimulator +RUST_TARGET=x86_64-apple-ios +LLVM_TARGET=x86_64-apple-ios-simulator # else PLATFORM_TRIPLET=arm64-iphonesimulator +RUST_TARGET=aarch64-apple-ios-sim +LLVM_TARGET=arm64-apple-ios-simulator # endif # else PLATFORM_TRIPLET=arm64-iphoneos +RUST_TARGET=aarch64-apple-ios +LLVM_TARGET=arm64-apple-ios # endif # elif defined(TARGET_OS_TV) && TARGET_OS_TV # if defined(TARGET_OS_SIMULATOR) && TARGET_OS_SIMULATOR @@ -286,19 +379,36 @@ PLATFORM_TRIPLET=arm64-xros // Older macOS SDKs do not define TARGET_OS_OSX # elif !defined(TARGET_OS_OSX) || TARGET_OS_OSX PLATFORM_TRIPLET=darwin +# if defined(__x86_64__) +RUST_TARGET=x86_64-apple-darwin +LLVM_TARGET=x86_64-apple-macosx +# elif defined(__aarch64__) +RUST_TARGET=aarch64-apple-darwin +LLVM_TARGET=arm64-apple-macosx +# else +RUST_TARGET=unknown +# endif # else # error unknown Apple platform # endif #elif defined(__VXWORKS__) PLATFORM_TRIPLET=vxworks +RUST_TARGET=unknown #elif defined(__wasm32__) # if defined(__EMSCRIPTEN__) PLATFORM_TRIPLET=wasm32-emscripten +RUST_TARGET=wasm32-unknown-emscripten # elif defined(__wasi__) -# if defined(_REENTRANT) +# if defined(__wasip3__) +PLATFORM_TRIPLET=wasm32-wasi +RUST_TARGET=wasm32-wasip3 +# elif defined(_REENTRANT) PLATFORM_TRIPLET=wasm32-wasi-threads +RUST_TARGET=wasm32-wasip1-threads +LLVM_TARGET=wasm32-wasi # else PLATFORM_TRIPLET=wasm32-wasi +RUST_TARGET=wasm32-wasip1 # endif # else # error unknown wasm32 platform @@ -306,8 +416,10 @@ PLATFORM_TRIPLET=wasm32-wasi #elif defined(__wasm64__) # if defined(__EMSCRIPTEN__) PLATFORM_TRIPLET=wasm64-emscripten +RUST_TARGET=unknown # elif defined(__wasi__) PLATFORM_TRIPLET=wasm64-wasi +RUST_TARGET=unknown # else # error unknown wasm64 platform # endif diff --git a/Modules/Setup b/Modules/Setup index 0a183ae30f8f0a..d313bd1fedc186 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -182,6 +182,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_codecs_tw cjkcodecs/_codecs_tw.c #_multibytecodec cjkcodecs/multibytecodec.c #unicodedata unicodedata.c +#_base64 _base64/Cargo.toml _base64/src/lib.rs lib_base64.a # Modules with some UNIX dependencies diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index d8cd8de10dfe08..e5300fcb481a08 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -115,6 +115,11 @@ @MODULE__MULTIBYTECODEC_TRUE@_multibytecodec cjkcodecs/multibytecodec.c @MODULE_UNICODEDATA_TRUE@unicodedata unicodedata.c +############################################################################ +# Rust modules +# +@MODULE__BASE64_TRUE@_base64 _base64/Cargo.toml _base64/src/lib.rs + ############################################################################ # Modules with some UNIX dependencies # diff --git a/Modules/_base64/Cargo.toml b/Modules/_base64/Cargo.toml new file mode 100644 index 00000000000000..dbe739fa7a19b4 --- /dev/null +++ b/Modules/_base64/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "_base64" +version = "0.1.0" +edition = "2024" +rust-version.workspace = true + +[dependencies] +cpython-sys ={ path = "../cpython-sys" } + +[build-dependencies] +cpython-build-helper = { path = "../cpython-build-helper" } + +[lib] +name = "_base64" +# For shared builds, we generate a c dynamic library. Static builds use the rlib +crate-type = ["cdylib", "rlib"] diff --git a/Modules/_base64/build.rs b/Modules/_base64/build.rs new file mode 100644 index 00000000000000..8cca32f8f8c3df --- /dev/null +++ b/Modules/_base64/build.rs @@ -0,0 +1,5 @@ +use cpython_build_helper::print_linker_args; + +fn main() { + print_linker_args(); +} diff --git a/Modules/_base64/src/lib.rs b/Modules/_base64/src/lib.rs new file mode 100644 index 00000000000000..64e4d718789372 --- /dev/null +++ b/Modules/_base64/src/lib.rs @@ -0,0 +1,249 @@ +use std::cell::UnsafeCell; +use std::ffi::{c_char, c_int, c_void}; +use std::mem::MaybeUninit; +use std::ptr; +use std::slice; + +use cpython_sys::METH_FASTCALL; +use cpython_sys::Py_DecRef; +use cpython_sys::Py_buffer; +use cpython_sys::Py_ssize_t; +use cpython_sys::PyBuffer_Release; +use cpython_sys::PyBytes_AsString; +use cpython_sys::PyBytes_FromStringAndSize; +use cpython_sys::PyErr_NoMemory; +use cpython_sys::PyErr_SetString; +use cpython_sys::PyExc_TypeError; +use cpython_sys::PyMethodDef; +use cpython_sys::PyMethodDefFuncPointer; +use cpython_sys::PyModuleDef; +use cpython_sys::PyModuleDef_HEAD_INIT; +use cpython_sys::PyModuleDef_Init; +use cpython_sys::PyObject; +use cpython_sys::PyObject_GetBuffer; + +const PYBUF_SIMPLE: c_int = 0; +const PAD_BYTE: u8 = b'='; +const ENCODE_TABLE: [u8; 64] = *b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#[inline] +fn encoded_output_len(input_len: usize) -> Option { + input_len + .checked_add(2) + .map(|n| n / 3) + .and_then(|blocks| blocks.checked_mul(4)) +} + +#[inline] +fn encode_into(input: &[u8], output: &mut [u8]) -> usize { + let mut src_index = 0; + let mut dst_index = 0; + let len = input.len(); + + while src_index + 3 <= len { + let chunk = (u32::from(input[src_index]) << 16) + | (u32::from(input[src_index + 1]) << 8) + | u32::from(input[src_index + 2]); + output[dst_index] = ENCODE_TABLE[((chunk >> 18) & 0x3f) as usize]; + output[dst_index + 1] = ENCODE_TABLE[((chunk >> 12) & 0x3f) as usize]; + output[dst_index + 2] = ENCODE_TABLE[((chunk >> 6) & 0x3f) as usize]; + output[dst_index + 3] = ENCODE_TABLE[(chunk & 0x3f) as usize]; + src_index += 3; + dst_index += 4; + } + + match len - src_index { + 0 => {} + 1 => { + let chunk = u32::from(input[src_index]) << 16; + output[dst_index] = ENCODE_TABLE[((chunk >> 18) & 0x3f) as usize]; + output[dst_index + 1] = ENCODE_TABLE[((chunk >> 12) & 0x3f) as usize]; + output[dst_index + 2] = PAD_BYTE; + output[dst_index + 3] = PAD_BYTE; + dst_index += 4; + } + 2 => { + let chunk = + (u32::from(input[src_index]) << 16) | (u32::from(input[src_index + 1]) << 8); + output[dst_index] = ENCODE_TABLE[((chunk >> 18) & 0x3f) as usize]; + output[dst_index + 1] = ENCODE_TABLE[((chunk >> 12) & 0x3f) as usize]; + output[dst_index + 2] = ENCODE_TABLE[((chunk >> 6) & 0x3f) as usize]; + output[dst_index + 3] = PAD_BYTE; + dst_index += 4; + } + _ => unreachable!("len - src_index cannot exceed 2"), + } + + dst_index +} + +struct BorrowedBuffer { + view: Py_buffer, +} + +impl BorrowedBuffer { + fn from_object(obj: &PyObject) -> Result { + let mut view = MaybeUninit::::uninit(); + let buffer = unsafe { + if PyObject_GetBuffer(obj.as_raw(), view.as_mut_ptr(), PYBUF_SIMPLE) != 0 { + return Err(()); + } + Self { + view: view.assume_init(), + } + }; + Ok(buffer) + } + + fn len(&self) -> Py_ssize_t { + self.view.len + } + + fn as_ptr(&self) -> *const u8 { + self.view.buf.cast::() as *const u8 + } +} + +impl Drop for BorrowedBuffer { + fn drop(&mut self) { + unsafe { + PyBuffer_Release(&mut self.view); + } + } +} + +/// # Safety +/// `module` must be a valid pointer of PyObject representing the module. +/// `args` must be a valid pointer to an array of valid PyObject pointers with length `nargs`. +pub unsafe extern "C" fn standard_b64encode( + _module: *mut PyObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, +) -> *mut PyObject { + if nargs != 1 { + unsafe { + PyErr_SetString( + PyExc_TypeError, + c"standard_b64encode() takes exactly one argument".as_ptr(), + ); + } + return ptr::null_mut(); + } + + let source = unsafe { &**args }; + + // Safe cast by Safety + match standard_b64encode_impl(source) { + Ok(result) => result, + Err(_) => ptr::null_mut(), + } +} + +fn standard_b64encode_impl(source: &PyObject) -> Result<*mut PyObject, ()> { + let buffer = match BorrowedBuffer::from_object(source) { + Ok(buf) => buf, + Err(_) => return Err(()), + }; + + let view_len = buffer.len(); + if view_len < 0 { + unsafe { + PyErr_SetString( + PyExc_TypeError, + c"standard_b64encode() argument has negative length".as_ptr(), + ); + } + return Err(()); + } + + let input_len = view_len as usize; + let input = unsafe { slice::from_raw_parts(buffer.as_ptr(), input_len) }; + + let Some(output_len) = encoded_output_len(input_len) else { + unsafe { + PyErr_NoMemory(); + } + return Err(()); + }; + + if output_len > isize::MAX as usize { + unsafe { + PyErr_NoMemory(); + } + return Err(()); + } + + let result = unsafe { PyBytes_FromStringAndSize(ptr::null(), output_len as Py_ssize_t) }; + if result.is_null() { + return Err(()); + } + + let dest_ptr = unsafe { PyBytes_AsString(result) }; + if dest_ptr.is_null() { + unsafe { + Py_DecRef(result); + } + return Err(()); + } + let dest = unsafe { slice::from_raw_parts_mut(dest_ptr.cast::(), output_len) }; + + let written = encode_into(input, dest); + debug_assert_eq!(written, output_len); + Ok(result) +} + +pub extern "C" fn _base64_clear(_obj: *mut PyObject) -> c_int { + //TODO + 0 +} + +pub extern "C" fn _base64_free(_o: *mut c_void) { + //TODO +} + +pub struct ModuleDef { + ffi: UnsafeCell, +} + +impl ModuleDef { + fn init_multi_phase(&'static self) -> *mut PyObject { + unsafe { PyModuleDef_Init(self.ffi.get()) } + } +} + +unsafe impl Sync for ModuleDef {} + +pub static _BASE64_MODULE_METHODS: [PyMethodDef; 2] = { + [ + PyMethodDef { + ml_name: c"standard_b64encode".as_ptr() as *mut c_char, + ml_meth: PyMethodDefFuncPointer { + PyCFunctionFast: standard_b64encode, + }, + ml_flags: METH_FASTCALL, + ml_doc: c"Demo for the _base64 module".as_ptr() as *mut c_char, + }, + PyMethodDef::zeroed(), + ] +}; + +pub static _BASE64_MODULE: ModuleDef = { + ModuleDef { + ffi: UnsafeCell::new(PyModuleDef { + m_base: PyModuleDef_HEAD_INIT, + m_name: c"_base64".as_ptr() as *mut _, + m_doc: c"A test Rust module".as_ptr() as *mut _, + m_size: 0, + m_methods: &_BASE64_MODULE_METHODS as *const PyMethodDef as *mut _, + m_slots: std::ptr::null_mut(), + m_traverse: None, + m_clear: Some(_base64_clear), + m_free: Some(_base64_free), + }), + } +}; + +#[unsafe(no_mangle)] +pub extern "C" fn PyInit__base64() -> *mut PyObject { + _BASE64_MODULE.init_multi_phase() +} diff --git a/Modules/cpython-build-helper/Cargo.toml b/Modules/cpython-build-helper/Cargo.toml new file mode 100644 index 00000000000000..193808546be285 --- /dev/null +++ b/Modules/cpython-build-helper/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "cpython-build-helper" +version = "0.1.0" +edition = "2024" +rust-version.workspace = true + +[dependencies] +shlex = "1.3" diff --git a/Modules/cpython-build-helper/README.md b/Modules/cpython-build-helper/README.md new file mode 100644 index 00000000000000..f42946cfefc31c --- /dev/null +++ b/Modules/cpython-build-helper/README.md @@ -0,0 +1,4 @@ +# cpython-build-helper + +This crate is used in Rust modules `build.rs` files to help do common tasks such +as passing necessary link arguments. \ No newline at end of file diff --git a/Modules/cpython-build-helper/src/lib.rs b/Modules/cpython-build-helper/src/lib.rs new file mode 100644 index 00000000000000..8bfb07eef55271 --- /dev/null +++ b/Modules/cpython-build-helper/src/lib.rs @@ -0,0 +1,137 @@ +use std::env; + +/// Print necessary link arguments for the library depending on the build +/// configuration (static or shared) +pub fn print_linker_args() { + println!("cargo:rerun-if-env-changed=RUST_SHARED_BUILD"); + println!("cargo:rerun-if-env-changed=BLDSHARED_EXE"); + println!("cargo:rerun-if-env-changed=BLDSHARED_ARGS"); + println!("cargo:rerun-if-env-changed=LIBPYTHON"); + println!("cargo:rerun-if-env-changed=PY_CC"); + println!("cargo:rerun-if-env-changed=PYTHON_BUILD_DIR"); + println!("cargo:rerun-if-env-changed=CARGO_CFG_TARGET_OS"); + + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let shared_build = + env::var("RUST_SHARED_BUILD").unwrap_or_default() == "1" || target_os == "ios"; + + // On Apple platforms (macOS, iOS), Cargo's cdylib produces a Mach-O + // dynamiclib (via -dynamiclib), but CPython's C extensions are built as + // bundles (via -bundle). Unlike bundles, dynamiclibs require all symbols + // to be resolved at link time. Pass -undefined dynamic_lookup so that + // Python C API symbols are resolved at load time by the interpreter. + if target_os == "macos" || target_os == "ios" { + println!("cargo:rustc-cdylib-link-arg=-undefined"); + println!("cargo:rustc-cdylib-link-arg=dynamic_lookup"); + } + + // Rust 1.100 stopped passing --allow-undefined to wasm-ld by default. + // Python C API symbols are provided by the interpreter, so keep them + // undefined in the cdylib. On the component model targets (wasip2, + // wasip3) the linker is wasm-component-ld, which cannot turn a module + // with such imports into a component, so emit a core module instead. + if target_os == "wasi" { + println!("cargo:rustc-cdylib-link-arg=--allow-undefined"); + let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); + if target_env == "p2" || target_env == "p3" { + println!("cargo:rustc-cdylib-link-arg=--skip-wit-component"); + } + } + + // Apple framework builds for iOS encode framework search/link flags in + // BLDSHARED_EXE. Skip the linker executable itself and filter out flags + // that conflict with Cargo's own cdylib invocation. + if shared_build && let Ok(args) = env::var("BLDSHARED_EXE") { + print_link_args(&args, true); + } + + // Pass platform-specific shared link arguments (e.g. PY_CORE_LDFLAGS) + // from the CPython build system. + if let Ok(args) = env::var("BLDSHARED_ARGS") { + print_link_args(&args, false); + } + + // On Android (and Cygwin), extension modules must link against libpython. + // LIBPYTHON is set by the CPython build system on these platforms and + // typically contains "-L. -lpython3.X". The "-L." is relative to the + // make build directory, so resolve it to an absolute path using + // PYTHON_BUILD_DIR. + if let Ok(libpython) = env::var("LIBPYTHON") { + let builddir = env::var("PYTHON_BUILD_DIR").ok(); + for arg in shlex::split(&libpython).expect("Invalid LIBPYTHON") { + if arg == "-L." + && let Some(ref dir) = builddir + { + println!("cargo:rustc-cdylib-link-arg=-L{}", dir); + continue; + } + println!("cargo:rustc-cdylib-link-arg={}", arg); + } + } + + // Static linker configuration is in cpython-rust-staticlib +} + +fn print_link_args(raw: &str, skip_first: bool) { + let mut args = shlex::split(raw).expect("Invalid linker args"); + if skip_first { + args = strip_linker_executable(args, env::var("PY_CC").ok().as_deref()); + } + + let build_dir = env::var("PYTHON_BUILD_DIR").ok(); + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + // -bundle_loader only works with -bundle, not with Cargo's + // cdylib mode (-dynamiclib). + "-bundle_loader" => { + i += 2; + } + // Cargo chooses the Mach-O output type for cdylib already. + "-bundle" | "-dynamiclib" => { + i += 1; + } + // dynamic_lookup is added explicitly for Apple targets above. + "-undefined" if i + 1 < args.len() && args[i + 1] == "dynamic_lookup" => { + i += 2; + } + "-F" if i + 1 < args.len() => { + let value = if args[i + 1] == "." { + build_dir.clone().unwrap_or_else(|| ".".to_string()) + } else { + args[i + 1].clone() + }; + println!("cargo:rustc-cdylib-link-arg=-F"); + println!("cargo:rustc-cdylib-link-arg={value}"); + i += 2; + } + _ => { + println!("cargo:rustc-cdylib-link-arg={}", args[i]); + i += 1; + } + } + } +} + +// BLDSHARED_EXE may start with a wrapper/compiler command such as +// "xcrun --sdk iphoneos clang" or "ccache clang". Strip that prefix so only +// linker flags are forwarded to rustc. +fn strip_linker_executable(args: Vec, compiler: Option<&str>) -> Vec { + if let Some(compiler) = compiler + && let Some(prefix) = compiler_command_prefix(compiler) + && args.as_slice().starts_with(prefix.as_slice()) + { + return args[prefix.len()..].to_vec(); + } + + if args.is_empty() { + return args; + } + args[1..].to_vec() +} + +fn compiler_command_prefix(raw: &str) -> Option> { + let tokens = shlex::split(raw)?; + let last_non_flag = tokens.iter().rposition(|token| !token.starts_with('-'))?; + Some(tokens.into_iter().take(last_non_flag + 1).collect()) +} diff --git a/Modules/cpython-rust-staticlib/Cargo.toml b/Modules/cpython-rust-staticlib/Cargo.toml new file mode 100644 index 00000000000000..2b21ab576a09c3 --- /dev/null +++ b/Modules/cpython-rust-staticlib/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "cpython-rust-staticlib" +version = "0.1.0" +edition = "2024" +rust-version.workspace = true + +[dependencies] +_base64 ={ path = "../_base64" } + +[lib] +name = "cpython_rust_staticlib" +crate-type = ["staticlib"] diff --git a/Modules/cpython-rust-staticlib/src/lib.rs b/Modules/cpython-rust-staticlib/src/lib.rs new file mode 100644 index 00000000000000..a440707833f65d --- /dev/null +++ b/Modules/cpython-rust-staticlib/src/lib.rs @@ -0,0 +1 @@ +pub use _base64::PyInit__base64; diff --git a/Modules/cpython-sys/Cargo.toml b/Modules/cpython-sys/Cargo.toml new file mode 100644 index 00000000000000..4f0dc4cbaca684 --- /dev/null +++ b/Modules/cpython-sys/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "cpython-sys" +version = "0.1.0" +edition = "2024" +rust-version.workspace = true + +[dependencies] + +[build-dependencies] +bindgen = "0.72.1" +prettyplease = "0.2.37" +shlex = "1.3" +syn = { version = "2.0.110", features = ["full", "parsing"] } diff --git a/Modules/cpython-sys/build.rs b/Modules/cpython-sys/build.rs new file mode 100644 index 00000000000000..385f99a466594d --- /dev/null +++ b/Modules/cpython-sys/build.rs @@ -0,0 +1,407 @@ +use std::env; +use std::path::{Path, PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let srcdir = manifest_dir + .parent() + .and_then(Path::parent) + .expect("expected Modules/cpython-sys to live under the source tree"); + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + let builddir = env::var("PYTHON_BUILD_DIR").ok(); + emit_rerun_instructions(builddir.as_deref()); + let gil_disabled = gil_disabled(srcdir, builddir.as_deref()); + if gil_disabled { + println!("cargo:rustc-cfg=py_gil_disabled"); + } + println!("cargo::rustc-check-cfg=cfg(py_gil_disabled)"); + generate_c_api_bindings( + srcdir, + builddir.as_deref(), + out_path.as_path(), + gil_disabled, + ); +} + +// Bindgen depends on build-time env and, on iOS, can also inherit the +// deployment target from the generated Makefile. Declare both so Cargo reruns +// the build script when those inputs change. +fn emit_rerun_instructions(builddir: Option<&str>) { + for var in [ + "IPHONEOS_DEPLOYMENT_TARGET", + "LLVM_TARGET", + "PY_DEBUG", + "PY_GIL_DISABLED", + "PYTHON_BUILD_DIR", + "PY_CC", + "PY_CPPFLAGS", + "PY_CFLAGS", + "TARGET", + "WASI_SDK_PATH", + "EMSDK", + ] { + println!("cargo:rerun-if-env-changed={var}"); + } + + if let Some(builddir) = builddir { + let makefile = Path::new(builddir).join("Makefile"); + println!("cargo:rerun-if-changed={}", makefile.display()); + } +} + +/// Find the newest clang resource directory on the system. +/// +/// Ubuntu 24.04 ships libclang-18 whose built-in headers (stdatomic.h, +/// mmintrin.h) are broken. CI jobs install a newer clang (e.g. clang-20) +/// whose resource directory at /usr/lib/llvm-20/lib/clang/20 has working +/// headers. We pass -resource-dir to bindgen's clang so it picks up those +/// headers instead of the broken libclang-18 ones. +fn newest_clang_resource_dir() -> Option { + // On Windows, derive the resource directory from LIBCLANG_PATH so that + // bindgen uses headers matching the libclang.dll it loads, rather than + // picking up a different, incompatible, system LLVM installation + if cfg!(windows) { + return clang_resource_dir_from_libclang_path(); + } + + let base = Path::new("/usr/lib"); + let mut best: Option<(u32, PathBuf)> = None; + for entry in std::fs::read_dir(base).ok()?.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some(ver_str) = name.strip_prefix("llvm-") + && let Ok(ver) = ver_str.parse::() + { + // Resource dir: /usr/lib/llvm-/lib/clang/ + let resource_dir = entry.path().join("lib").join("clang").join(ver_str); + if resource_dir.join("include").is_dir() + && best.as_ref().map_or(true, |(v, _)| ver > *v) + { + best = Some((ver, resource_dir)); + } + } + } + best.map(|(_, p)| p) +} + +/// Derive the clang resource directory from LIBCLANG_PATH. +/// +/// When LIBCLANG_PATH points to e.g. `...\bin`, the resource directory is +/// at `...\lib\clang\`. We pick the version matching the loaded +/// libclang, since the directory may also contain headers left behind by +/// another LLVM installation. Otherwise, we pick the highest version found. +fn clang_resource_dir_from_libclang_path() -> Option { + let libclang_path = env::var("LIBCLANG_PATH").ok()?; + let bin_dir = Path::new(&libclang_path); + // LIBCLANG_PATH typically points to a `bin` directory; the resource + // dir lives under the sibling `lib/clang/`. + let clang_lib_dir = bin_dir.parent()?.join("lib").join("clang"); + let libclang_major = bindgen::clang_version().parsed.map(|(major, _)| major); + let mut best: Option<(u32, PathBuf)> = None; + for entry in std::fs::read_dir(&clang_lib_dir).ok()?.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + // Version directories can be just a major number (e.g. "18") or + // a full dotted version (e.g. "18.1.3"). Parse the major part. + if let Ok(ver) = name.split('.').next().unwrap_or("").parse::() { + let resource_dir = entry.path(); + if !resource_dir.join("include").is_dir() { + continue; + } + if Some(ver) == libclang_major { + return Some(resource_dir); + } + if best.as_ref().map_or(true, |(v, _)| ver > *v) { + best = Some((ver, resource_dir)); + } + } + } + best.map(|(_, p)| p) +} + +fn gil_disabled(srcdir: &Path, builddir: Option<&str>) -> bool { + if env_var_is_truthy("PY_GIL_DISABLED") { + return true; + } + + let mut candidates = Vec::new(); + if let Some(build) = builddir { + candidates.push(PathBuf::from(build)); + } + candidates.push(srcdir.to_path_buf()); + for base in candidates { + let path = base.join("pyconfig.h"); + if let Ok(contents) = std::fs::read_to_string(&path) + && contents.contains("Py_GIL_DISABLED 1") + { + return true; + } + } + false +} + +fn env_var_is_truthy(name: &str) -> bool { + env::var(name) + .map(|v| matches!(v.to_ascii_lowercase().as_str(), "1" | "true" | "yes")) + .unwrap_or(false) +} + +fn generate_c_api_bindings( + srcdir: &Path, + builddir: Option<&str>, + out_path: &Path, + gil_disabled: bool, +) { + let mut builder = bindgen::Builder::default().header("wrapper.h"); + + // Suppress all clang warnings (deprecation warnings, etc.) + builder = builder.clang_arg("-w"); + + if env_var_is_truthy("PY_DEBUG") { + builder = builder.clang_arg("-D_DEBUG"); + } + if gil_disabled { + builder = builder.clang_arg("-DPy_GIL_DISABLED=1"); + } + + // Use the newest clang resource directory available on the system. + // Bindgen links against whatever libclang it finds (often an older + // system version), but the built-in headers in that version may be + // broken (e.g. libclang-18 on Ubuntu 24.04 has broken stdatomic.h + // and mmintrin.h). Overriding -resource-dir makes clang use a + // newer set of built-in headers without changing which libclang.so + // is loaded. + if let Some(resource_dir) = newest_clang_resource_dir() { + eprintln!( + "cpython-sys: using clang resource dir {}", + resource_dir.display() + ); + builder = builder.clang_arg(format!("-resource-dir={}", resource_dir.display())); + } + + // Tell clang the correct target triple for cross-compilation when we have + // an LLVM-specific triple. Otherwise let bindgen translate Cargo's TARGET + // itself (e.g. aarch64-apple-ios-sim -> arm64-apple-ios-simulator). + let cargo_target = env::var("TARGET").unwrap_or_default(); + let llvm_target = env::var("LLVM_TARGET").unwrap_or_default(); + if !llvm_target.is_empty() && llvm_target != cargo_target { + builder = builder.clang_arg(format!("--target={llvm_target}")); + } + + // Extract cross-compilation flags from the C compiler command (PY_CC), + // preprocessor flags (PY_CPPFLAGS), and compiler flags (PY_CFLAGS). + // These provide the sysroot, include paths, and defines that bindgen's + // clang needs when cross-compiling. + // + // - WASI: sysroot in CC, -D_WASI_EMULATED_SIGNAL in CFLAGS + // - iOS: -isysroot in CPPFLAGS + let mut have_sysroot = false; + for env_name in ["PY_CC", "PY_CPPFLAGS", "PY_CFLAGS"] { + if let Ok(value) = env::var(env_name) + && let Some(flags) = shlex::split(&value) + { + let mut iter = flags.iter().peekable(); + while let Some(flag) = iter.next() { + if flag.starts_with("--sysroot") || flag.starts_with("-isysroot") { + builder = builder.clang_arg(flag); + have_sysroot = true; + // Handle "-isysroot " (space-separated) + if (flag == "-isysroot" || flag == "--sysroot") + && let Some(path) = iter.next() + { + builder = builder.clang_arg(path); + } + } else if flag.starts_with("-I") + || flag.starts_with("-D") + || flag.starts_with("-std=") + || flag.starts_with("-isystem") + { + builder = builder.clang_arg(flag); + } + } + } + } + + // WASI SDK: WASI_SDK_PATH is set by Tools/wasm/wasi/__main__.py. + // The sysroot is at $WASI_SDK_PATH/share/wasi-sysroot. + if !have_sysroot + && cargo_target.contains("wasi") + && let Ok(sdk_path) = env::var("WASI_SDK_PATH") + { + let sysroot = PathBuf::from(&sdk_path).join("share").join("wasi-sysroot"); + if sysroot.is_dir() { + builder = builder.clang_arg(format!("--sysroot={}", sysroot.display())); + have_sysroot = true; + } + } + + if !have_sysroot + && cargo_target.contains("emscripten") + && let Ok(emsdk) = env::var("EMSDK") + { + let sysroot = PathBuf::from(&emsdk) + .join("upstream") + .join("emscripten") + .join("cache") + .join("sysroot"); + if sysroot.is_dir() { + builder = builder.clang_arg(format!("--sysroot={}", sysroot.display())); + have_sysroot = true; + } + } + + // Android NDK: ANDROID_HOME is set by the CI/user environment, and + // Android/android-env.sh sets CC to the NDK clang binary at: + // $ANDROID_HOME/ndk//toolchains/llvm/prebuilt//bin/-clang + // The sysroot is a sibling of bin/: + // .../toolchains/llvm/prebuilt//sysroot + if !have_sysroot + && cargo_target.contains("android") + && let Ok(cc) = env::var("PY_CC") + && let Some(parts) = shlex::split(&cc) + && let Some(binary) = parts.first() + && let Some(bin_dir) = Path::new(binary).parent() + { + let sysroot = bin_dir.with_file_name("sysroot"); + if sysroot.is_dir() { + builder = builder.clang_arg(format!("--sysroot={}", sysroot.display())); + } + } + + // Include the build directory first so that cross-build pyconfig.h + // takes precedence over any pyconfig.h in the source tree (which may + // be from a native build with different settings like LONG_BIT). + let mut include_dirs = Vec::new(); + if let Some(build) = builddir { + include_dirs.push(PathBuf::from(build)); + } + include_dirs.push(srcdir.to_path_buf()); + include_dirs.push(srcdir.join("Include")); + + for dir in include_dirs { + builder = builder.clang_arg(format!("-I{}", dir.display())); + } + + builder = add_target_clang_args(builder, builddir); + + let bindings = builder + .allowlist_function("_?Py.*") + .allowlist_type("_?Py.*") + .allowlist_var("_?Py.*") + .blocklist_type("^PyMethodDef$") + .blocklist_type("PyObject") + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .generate() + .expect("Unable to generate bindings"); + + let dll_name = python_dll_name(srcdir, env_var_is_truthy("PY_DEBUG"), gil_disabled); + let bindings = patch_windows_imported_pointer_globals(bindings.to_string(), &dll_name); + + // Write the bindings to the $OUT_DIR/c_api.rs file. + std::fs::write(out_path.join("c_api.rs"), bindings).expect("Couldn't write bindings!"); +} + +/// Build the Windows DLL base name: `python{major}{minor}[t][_d]`. +fn python_dll_name(srcdir: &Path, debug: bool, gil_disabled: bool) -> String { + let patchlevel = srcdir.join("Include").join("patchlevel.h"); + let contents = + std::fs::read_to_string(&patchlevel).expect("failed to read Include/patchlevel.h"); + + let major = extract_define_int(&contents, "PY_MAJOR_VERSION"); + let minor = extract_define_int(&contents, "PY_MINOR_VERSION"); + + let mut name = format!("python{major}{minor}"); + if gil_disabled { + name.push('t'); + } + if debug { + name.push_str("_d"); + } + name +} + +fn extract_define_int(contents: &str, name: &str) -> u32 { + for line in contents.lines() { + let trimmed = line.trim(); + if let Some(rest) = trimmed.strip_prefix("#define") + && let Some(value) = rest.trim().strip_prefix(name) + && let Ok(n) = value.trim().parse() + { + return n; + } + } + panic!("could not find #define {name} in patchlevel.h"); +} + +fn patch_windows_imported_pointer_globals(bindings: String, dll_name: &str) -> String { + // On Windows/MSVC, exported data is imported through a synthetic + // "__imp_" pointer in the import address table (IAT). A plain + // `extern { pub static X: *mut T; }` linked via import library fails for + // data symbols because the import library only defines `__imp_X`, not `X`. + // + // Using `#[link_name = "__imp_X"]` links successfully but produces a + // single load — returning the *address* of the variable (the IAT slot + // value) rather than the variable's value. C's `__declspec(dllimport)` + // generates two loads to chase the indirection. + // + // The fix: annotate pointer-valued extern statics with `raw-dylib` on + // Windows so Rust generates the import thunk itself and handles the IAT + // indirection correctly — two loads, matching `__declspec(dllimport)`. + let mut file = syn::parse_file(&bindings).expect("bindgen emitted invalid Rust"); + + // Bindgen generates a single extern block per symbol, so we can iterate over all items + // and patch the pointer-valued statics. + for item in &mut file.items { + let syn::Item::ForeignMod(foreign_mod) = item else { + continue; + }; + let [syn::ForeignItem::Static(static_item)] = foreign_mod.items.as_slice() else { + continue; + }; + if !matches!(*static_item.ty, syn::Type::Ptr(_)) { + continue; + } + foreign_mod.attrs.push(syn::parse_quote!( + #[cfg_attr(windows, link(name = #dll_name, kind = "raw-dylib"))] + )); + } + + prettyplease::unparse(&file) +} + +fn add_target_clang_args( + mut builder: bindgen::Builder, + builddir: Option<&str>, +) -> bindgen::Builder { + let target = env::var("TARGET").unwrap_or_default(); + if !target.contains("apple-ios") { + return builder; + } + + // For iOS targets, bindgen may parse headers with an iOS simulator/device + // target but without a deployment minimum, which disables TLS support. + let deployment_target = ios_deployment_target(builddir).unwrap_or_else(|| "13.0".to_string()); + builder = builder.clang_arg(format!("-mios-version-min={deployment_target}")); + builder +} + +fn ios_deployment_target(builddir: Option<&str>) -> Option { + if let Ok(value) = env::var("IPHONEOS_DEPLOYMENT_TARGET") + && !value.is_empty() + { + return Some(value); + } + + let builddir = builddir?; + let makefile = Path::new(builddir).join("Makefile"); + let text = std::fs::read_to_string(makefile).ok()?; + for line in text.lines() { + if let Some(value) = line.strip_prefix("IPHONEOS_DEPLOYMENT_TARGET=") { + let value = value.trim(); + if !value.is_empty() { + return Some(value.to_string()); + } + } + } + None +} diff --git a/Modules/cpython-sys/parser.h b/Modules/cpython-sys/parser.h new file mode 100644 index 00000000000000..3fa5866b5bcdc9 --- /dev/null +++ b/Modules/cpython-sys/parser.h @@ -0,0 +1,9 @@ +/* Private APIs */ +#define Py_BUILD_CORE + +// Parser +#include "Parser/pegen.h" +#include "Parser/string_parser.h" +#include "Parser/lexer/state.h" +#include "Parser/tokenizer/tokenizer.h" +#include "Parser/tokenizer/helpers.h" diff --git a/Modules/cpython-sys/src/lib.rs b/Modules/cpython-sys/src/lib.rs new file mode 100644 index 00000000000000..17924f0e5fb00a --- /dev/null +++ b/Modules/cpython-sys/src/lib.rs @@ -0,0 +1,154 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(unsafe_op_in_unsafe_fn)] +#![allow(unnecessary_transmutes)] +#![allow(clippy::approx_constant)] + +use std::ffi::{c_char, c_int, c_void}; + +include!(concat!(env!("OUT_DIR"), "/c_api.rs")); + +// TODO(emmatyping): include parser bindings (see build.rs) +//include!(concat!(env!("OUT_DIR"), "/parser.rs")); +/* Flag passed to newmethodobject */ +/* #define METH_OLDARGS 0x0000 -- unsupported now */ +pub const METH_VARARGS: c_int = 0x0001; +pub const METH_KEYWORDS: c_int = 0x0002; +/* METH_NOARGS and METH_O must not be combined with the flags above. */ +pub const METH_NOARGS: c_int = 0x0004; +pub const METH_O: c_int = 0x0008; + +/* METH_CLASS and METH_STATIC are a little different; these control +the construction of methods for a class. These cannot be used for +functions in modules. */ +pub const METH_CLASS: c_int = 0x0010; +pub const METH_STATIC: c_int = 0x0020; + +/* METH_COEXIST allows a method to be entered even though a slot has +already filled the entry. When defined, the flag allows a separate +method, "__contains__" for example, to coexist with a defined +slot like sq_contains. */ + +pub const METH_COEXIST: c_int = 0x0040; + +pub const METH_FASTCALL: c_int = 0x0080; + +/* This bit is preserved for Stackless Python +pub const METH_STACKLESS: c_int = 0x0100; +pub const METH_STACKLESS: c_int = 0x0000; +*/ + +/* METH_METHOD means the function stores an + * additional reference to the class that defines it; + * both self and class are passed to it. + * It uses PyCMethodObject instead of PyCFunctionObject. + * May not be combined with METH_NOARGS, METH_O, METH_CLASS or METH_STATIC. + */ + +pub const METH_METHOD: c_int = 0x0200; + +#[cfg(target_pointer_width = "64")] +pub const _Py_STATIC_FLAG_BITS: Py_ssize_t = + (_Py_STATICALLY_ALLOCATED_FLAG | _Py_IMMORTAL_FLAGS) as Py_ssize_t; +#[cfg(target_pointer_width = "64")] +pub const _Py_STATIC_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = + (_Py_IMMORTAL_INITIAL_REFCNT as Py_ssize_t) | (_Py_STATIC_FLAG_BITS << 48); +#[cfg(not(target_pointer_width = "64"))] +pub const _Py_STATIC_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = (7u32 << 28) as Py_ssize_t; + +#[repr(transparent)] +pub struct PyObject(std::cell::UnsafeCell<_object>); + +impl PyObject { + #[inline] + pub fn as_raw(&self) -> *mut Self { + self.0.get() as *mut Self + } +} + +#[repr(C)] +pub union PyMethodDefFuncPointer { + pub PyCFunction: unsafe extern "C" fn(slf: *mut PyObject, args: *mut PyObject) -> *mut PyObject, + pub PyCFunctionFast: unsafe extern "C" fn( + slf: *mut PyObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, + ) -> *mut PyObject, + pub PyCFunctionWithKeywords: unsafe extern "C" fn( + slf: *mut PyObject, + args: *mut PyObject, + kwargs: *mut PyObject, + ) -> *mut PyObject, + pub PyCFunctionFastWithKeywords: unsafe extern "C" fn( + slf: *mut PyObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, + kwargs: *mut PyObject, + ) -> *mut PyObject, + pub PyCMethod: unsafe extern "C" fn( + slf: *mut PyObject, + typ: *mut PyTypeObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, + kwargs: *mut PyObject, + ) -> *mut PyObject, + pub Void: *mut c_void, +} + +#[repr(C)] +pub struct PyMethodDef { + pub ml_name: *mut c_char, + pub ml_meth: PyMethodDefFuncPointer, + pub ml_flags: c_int, + pub ml_doc: *mut c_char, +} + +impl PyMethodDef { + pub const fn zeroed() -> Self { + Self { + ml_name: std::ptr::null_mut(), + ml_meth: PyMethodDefFuncPointer { + Void: std::ptr::null_mut(), + }, + ml_flags: 0, + ml_doc: std::ptr::null_mut(), + } + } +} + +// TODO: this is pretty unsafe, we should probably wrap this in a nicer +// abstraction +unsafe impl Sync for PyMethodDef {} +unsafe impl Send for PyMethodDef {} + +#[cfg(py_gil_disabled)] +pub const PyObject_HEAD_INIT: PyObject = { + let mut obj: _object = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; + obj.ob_flags = _Py_STATICALLY_ALLOCATED_FLAG as _; + PyObject(std::cell::UnsafeCell::new(obj)) +}; + +#[cfg(all(not(py_gil_disabled), target_pointer_width = "64"))] +pub const PyObject_HEAD_INIT: PyObject = PyObject(std::cell::UnsafeCell::new(_object { + __bindgen_anon_1: _object__bindgen_ty_1 { + ob_refcnt_full: _Py_STATIC_IMMORTAL_INITIAL_REFCNT as i64, + }, + ob_type: std::ptr::null_mut(), +})); + +// On 32-bit platforms, the refcount union only has ob_refcnt (no ob_refcnt_split). +#[cfg(all(not(py_gil_disabled), not(target_pointer_width = "64")))] +pub const PyObject_HEAD_INIT: PyObject = PyObject(std::cell::UnsafeCell::new(_object { + __bindgen_anon_1: _object__bindgen_ty_1 { + ob_refcnt: _Py_STATIC_IMMORTAL_INITIAL_REFCNT, + }, + ob_type: std::ptr::null_mut(), +})); + +pub const PyModuleDef_HEAD_INIT: PyModuleDef_Base = PyModuleDef_Base { + ob_base: PyObject_HEAD_INIT, + m_init: None, + m_index: 0, + m_copy: std::ptr::null_mut(), +}; diff --git a/Modules/cpython-sys/wrapper.h b/Modules/cpython-sys/wrapper.h new file mode 100644 index 00000000000000..f79a44f0580633 --- /dev/null +++ b/Modules/cpython-sys/wrapper.h @@ -0,0 +1,177 @@ +/* Public APIs */ +#include "Python.h" + +// Misc + +// OS macros used in C, not necessary in Rust +//#include "osdefs.h" + +// Valgrind / analysis tools macros and functions +#include "dynamic_annotations.h" +// Macros for error codes +// #include "errcode.h" +// Macros to define symbol visibility +// #include "exports.h" +// Includes pyframe.h and cpython/frameobject.h +#include "frameobject.h" +// Includes cpython/marshal.h +#include "marshal.h" +// Macros defining opcodes +#include "opcode.h" +// More macros defining opcodes +#include "opcode_ids.h" +// Dtrace probes +#include "pydtrace.h" +//New code should use descrobject.h +//#include "structmember.h" + +// List of all stdlib names, autogenerated +#include "Python/stdlib_module_names.h" + +/* Private APIs */ +#define Py_BUILD_CORE + +// Internal +#include "internal/pycore_parser.h" +#include "internal/pycore_mimalloc.h" +#include "internal/mimalloc/mimalloc.h" +#include "internal/mimalloc/mimalloc/atomic.h" +#include "internal/mimalloc/mimalloc/internal.h" +#include "internal/mimalloc/mimalloc/prim.h" +#include "internal/mimalloc/mimalloc/track.h" +#include "internal/mimalloc/mimalloc/types.h" +#include "internal/pycore_abstract.h" +#include "internal/pycore_asdl.h" +#include "internal/pycore_ast.h" +#include "internal/pycore_ast_state.h" +#include "internal/pycore_atexit.h" +#include "internal/pycore_audit.h" +#include "internal/pycore_backoff.h" +#include "internal/pycore_bitutils.h" +#include "internal/pycore_blocks_output_buffer.h" +#include "internal/pycore_brc.h" +#include "internal/pycore_bytes_methods.h" +#include "internal/pycore_bytesobject.h" +#include "internal/pycore_call.h" +#include "internal/pycore_capsule.h" +#include "internal/pycore_cell.h" +#include "internal/pycore_ceval.h" +#include "internal/pycore_ceval_state.h" +#include "internal/pycore_code.h" +#include "internal/pycore_codecs.h" +#include "internal/pycore_compile.h" +#include "internal/pycore_complexobject.h" +#include "internal/pycore_condvar.h" +#include "internal/pycore_context.h" +#include "internal/pycore_critical_section.h" +#include "internal/pycore_crossinterp.h" +#include "internal/pycore_debug_offsets.h" +#include "internal/pycore_descrobject.h" +#include "internal/pycore_dict.h" +#include "internal/pycore_dict_state.h" +#include "internal/pycore_dtoa.h" +#include "internal/pycore_exceptions.h" +#include "internal/pycore_faulthandler.h" +#include "internal/pycore_fileutils.h" +#include "internal/pycore_floatobject.h" +#include "internal/pycore_flowgraph.h" +#include "internal/pycore_format.h" +#include "internal/pycore_frame.h" +#include "internal/pycore_freelist.h" +#include "internal/pycore_freelist_state.h" +#include "internal/pycore_function.h" +#include "internal/pycore_gc.h" +#include "internal/pycore_genobject.h" +#include "internal/pycore_getopt.h" +#include "internal/pycore_gil.h" +#include "internal/pycore_global_objects.h" +#include "internal/pycore_global_objects_fini_generated.h" +#include "internal/pycore_global_strings.h" +#include "internal/pycore_hamt.h" +#include "internal/pycore_hashtable.h" +#include "internal/pycore_import.h" +#include "internal/pycore_importdl.h" +#include "internal/pycore_index_pool.h" +#include "internal/pycore_initconfig.h" +#include "internal/pycore_instruments.h" +#include "internal/pycore_instruction_sequence.h" +#include "internal/pycore_interp.h" +#include "internal/pycore_interp_structs.h" +#include "internal/pycore_interpframe.h" +#include "internal/pycore_interpframe_structs.h" +#include "internal/pycore_interpolation.h" +#include "internal/pycore_intrinsics.h" +#include "internal/pycore_jit.h" +#include "internal/pycore_list.h" +#include "internal/pycore_llist.h" +#include "internal/pycore_lock.h" +#include "internal/pycore_long.h" +#include "internal/pycore_memoryobject.h" +#include "internal/pycore_mimalloc.h" +#include "internal/pycore_modsupport.h" +#include "internal/pycore_moduleobject.h" +#include "internal/pycore_namespace.h" +#include "internal/pycore_object.h" +#include "internal/pycore_object_alloc.h" +#include "internal/pycore_object_deferred.h" +#include "internal/pycore_object_stack.h" +#include "internal/pycore_object_state.h" +#include "internal/pycore_obmalloc.h" +#include "internal/pycore_obmalloc_init.h" +#include "internal/pycore_opcode_metadata.h" +#include "internal/pycore_opcode_utils.h" +#include "internal/pycore_optimizer.h" +#include "internal/pycore_parking_lot.h" +#include "internal/pycore_parser.h" +#include "internal/pycore_pathconfig.h" +#include "internal/pycore_pyarena.h" +#include "internal/pycore_pyatomic_ft_wrappers.h" +#include "internal/pycore_pybuffer.h" +#include "internal/pycore_pyerrors.h" +#include "internal/pycore_pyhash.h" +#include "internal/pycore_pylifecycle.h" +#include "internal/pycore_pymath.h" +#include "internal/pycore_pymem.h" +#include "internal/pycore_pymem_init.h" +#include "internal/pycore_pystate.h" +#include "internal/pycore_pystats.h" +#include "internal/pycore_pythonrun.h" +#include "internal/pycore_pythread.h" +#include "internal/pycore_qsbr.h" +#include "internal/pycore_range.h" +#include "internal/pycore_runtime.h" +#include "internal/pycore_runtime_init.h" +#include "internal/pycore_runtime_init_generated.h" +#include "internal/pycore_runtime_structs.h" +#include "internal/pycore_semaphore.h" +#include "internal/pycore_setobject.h" +#include "internal/pycore_signal.h" +#include "internal/pycore_sliceobject.h" +#include "internal/pycore_stats.h" +#include "internal/pycore_strhex.h" +#include "internal/pycore_stackref.h" +#include "internal/pycore_structs.h" +#include "internal/pycore_structseq.h" +#include "internal/pycore_symtable.h" +#include "internal/pycore_sysmodule.h" +#include "internal/pycore_template.h" +#include "internal/pycore_time.h" +#include "internal/pycore_token.h" +#include "internal/pycore_traceback.h" +#include "internal/pycore_tracemalloc.h" +#include "internal/pycore_tstate.h" +#include "internal/pycore_tuple.h" +#include "internal/pycore_typedefs.h" +#include "internal/pycore_typeobject.h" +#include "internal/pycore_typevarobject.h" +#include "internal/pycore_ucnhash.h" +#include "internal/pycore_unicodectype.h" +#include "internal/pycore_unicodeobject.h" +#include "internal/pycore_unicodeobject_generated.h" +#include "internal/pycore_unionobject.h" +#include "internal/pycore_uniqueid.h" +#include "internal/pycore_uop.h" +#include "internal/pycore_uop_ids.h" +#include "internal/pycore_uop_metadata.h" +#include "internal/pycore_warnings.h" +#include "internal/pycore_weakref.h" diff --git a/Modules/makesetup b/Modules/makesetup index 932c0952e4781d..c768fc1ee66935 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -94,6 +94,8 @@ case $makepre in '') makepre=Makefile.pre;; esac +UNAME_SYSTEM=`uname -s 2>/dev/null || echo unknown` + # Newline for sed i and a commands NL='\ ' @@ -162,6 +164,8 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | srcs= cpps= libs= + rust= + manifest= mods= mods_upper= skip= @@ -194,6 +198,8 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | *.c++) srcs="$srcs $arg";; *.cxx) srcs="$srcs $arg";; *.cpp) srcs="$srcs $arg";; + *.rs) srcs="$srcs $arg"; rust="true";; + *.toml) manifest="$arg";; \$\(*_CFLAGS\)) cpps="$cpps $arg";; \$\(*_INCLUDES\)) cpps="$cpps $arg";; \$\(*_LIBS\)) libs="$libs $arg";; @@ -262,74 +268,109 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | yes) continue;; esac objs='' - for src in $srcs - do - case $src in - *.c) obj=`basename $src .c`.o; cc='$(CC)';; - *.cc) obj=`basename $src .cc`.o; cc='$(CXX)';; - *.c++) obj=`basename $src .c++`.o; cc='$(CXX)';; - *.C) obj=`basename $src .C`.o; cc='$(CXX)';; - *.cxx) obj=`basename $src .cxx`.o; cc='$(CXX)';; - *.cpp) obj=`basename $src .cpp`.o; cc='$(CXX)';; - *.m) obj=`basename $src .m`.o; cc='$(CC)';; # Obj-C - *) continue;; - esac - case $src in - */*) obj="$srcdir/`dirname $src`/$obj";; - *) obj="$srcdir/$obj";; - esac - objs="$objs $obj" - case $src in - glmodule.c) ;; - /*) ;; - \$*) ;; - *) src='$(srcdir)/'"$srcdir/$src";; - esac - # custom flags first, PY_STDMODULE_CFLAGS may contain -I with system libmpdec + if test "x$rust" = "x"; then + for src in $srcs + do + case $src in + *.c) obj=`basename $src .c`.o; cc='$(CC)';; + *.cc) obj=`basename $src .cc`.o; cc='$(CXX)';; + *.c++) obj=`basename $src .c++`.o; cc='$(CXX)';; + *.C) obj=`basename $src .C`.o; cc='$(CXX)';; + *.cxx) obj=`basename $src .cxx`.o; cc='$(CXX)';; + *.cpp) obj=`basename $src .cpp`.o; cc='$(CXX)';; + *.m) obj=`basename $src .m`.o; cc='$(CC)';; # Obj-C + *) continue;; + esac + case $src in + */*) obj="$srcdir/`dirname $src`/$obj";; + *) obj="$srcdir/$obj";; + esac + objs="$objs $obj" + case $src in + glmodule.c) ;; + /*) ;; + \$*) ;; + *) src='$(srcdir)/'"$srcdir/$src";; + esac + # custom flags first, PY_STDMODULE_CFLAGS may contain -I with system libmpdec + case $doconfig in + no) + cc="$cc $cpps \$(PY_STDMODULE_CFLAGS) \$(CCSHARED)" + rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(MODULE_DEPS_SHARED) \$(PYTHON_HEADERS)" + rule="$rule; $cc -c $src -o $obj" + ;; + *) + cc="$cc $cpps \$(PY_BUILTIN_MODULE_CFLAGS)" + rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(MODULE_DEPS_STATIC) \$(PYTHON_HEADERS)" + rule="$rule; $cc -c $src -o $obj" + ;; + esac + echo "$rule" >>$rulesf + done case $doconfig in - no) - cc="$cc $cpps \$(PY_STDMODULE_CFLAGS) \$(CCSHARED)" - rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(MODULE_DEPS_SHARED) \$(PYTHON_HEADERS)" - rule="$rule; $cc -c $src -o $obj" - ;; - *) - cc="$cc $cpps \$(PY_BUILTIN_MODULE_CFLAGS)" - rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(MODULE_DEPS_STATIC) \$(PYTHON_HEADERS)" - rule="$rule; $cc -c $src -o $obj" + yes) + OBJS="$OBJS $objs" + case $bootstrap in + yes) BOOTOBJS="$BOOTOBJS $objs";; + esac + # Record each static module's objects and libs by name, so + # the bootstrap handling above can find a definition that + # won over a later line in the bootstrap Setup file. + for mod in $mods + do + eval "STATICMOD_$mod='$mod'" + eval "STATICOBJS_$mod='$objs'" + eval "STATICLIBS_$mod='$libs'" + done ;; esac - echo "$rule" >>$rulesf - done - case $doconfig in - yes) - OBJS="$OBJS $objs" - case $bootstrap in - yes) BOOTOBJS="$BOOTOBJS $objs";; - esac - # Record each static module's objects and libs by name, so - # the bootstrap handling above can find a definition that - # won over a later line in the bootstrap Setup file. for mod in $mods do - eval "STATICMOD_$mod='$mod'" - eval "STATICOBJS_$mod='$objs'" - eval "STATICLIBS_$mod='$libs'" + file="$srcdir/$mod\$(EXT_SUFFIX)" + case $doconfig in + no) + SHAREDMODS="$SHAREDMODS $file" + BUILT_SHARED="$BUILT_SHARED $mod" + ;; + esac + rule="$file: $objs \$(MODULE_${mods_upper}_LDEPS)" + rule="$rule; \$(BLDSHARED_EXE) \$(BLDSHARED_ARGS) $objs $libs \$(MODULE_LDFLAGS_SHARED) -o $file" + echo "$rule" >>$rulesf done - ;; - esac - for mod in $mods - do - file="$srcdir/$mod\$(EXT_SUFFIX)" + else + prefixed_srcs= + for src in $srcs + do + prefixed_srcs="$prefixed_srcs $srcdir/$src" + done + objs= + libs= + case $doconfig in no) - SHAREDMODS="$SHAREDMODS $file" - BUILT_SHARED="$BUILT_SHARED $mod" + # Shared build: build each Rust module as a cdylib (.so) + for mod in $mods + do + rust_shared="target/\$(if \$(CARGO_TARGET),\$(CARGO_TARGET)/\$(CARGO_TARGET_DIR),\$(CARGO_TARGET_DIR))/lib$mod\$(CARGO_DYLIB_SUFFIX)" + file="$srcdir/$mod\$(EXT_SUFFIX)" + SHAREDMODS="$SHAREDMODS $file" + BUILT_SHARED="$BUILT_SHARED $mod" + # depends on the headers through cpython-sys + rule="$rust_shared: cpython-sys \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest $prefixed_srcs \$(PYTHON_HEADERS) \$(MODULE_${mods_upper}_LDEPS) \$(LIBRARY)" + rule="$rule; CARGO_TARGET_DIR=\$(abs_builddir)/target PYTHON_BUILD_DIR=\$(abs_builddir) PY_CC=\"\$(CC)\" PY_CPPFLAGS=\"\$(CPPFLAGS)\" PY_CFLAGS=\"\$(CFLAGS)\" LLVM_TARGET=\"\$(LLVM_TARGET)\" RUST_SHARED_BUILD=\$(PY_ENABLE_SHARED) IPHONEOS_DEPLOYMENT_TARGET=\$(IPHONEOS_DEPLOYMENT_TARGET) BLDSHARED_EXE=\"\$(BLDSHARED_EXE)\" BLDSHARED_ARGS=\"\$(BLDSHARED_ARGS)\" LIBPYTHON=\"\$(MODULE_LDFLAGS_SHARED)\" \$(if \$(CARGO_TARGET_LINKER_ENV),\$(CARGO_TARGET_LINKER_ENV)=\$(CC)) \$(CARGO) build -vvv --lib --locked --package ${mod} --profile \$(CARGO_PROFILE) \$(if \$(CARGO_TARGET),--target=\$(CARGO_TARGET)) --manifest-path \$(srcdir)/Cargo.toml" + echo "$rule" >>$rulesf + echo "$file: $rust_shared; mv $rust_shared $file" >>$rulesf + done + ;; + yes) + # Static build: Rust modules are built as part of cpython-rust-staticlib. + # The dependency on cpython-rust-staticlib is in Makefile.pre.in + # (via RUST_STATICLIB_DEP on the BUILDPYTHON rule). + rust_staticlib="target/\$(if \$(CARGO_TARGET),\$(CARGO_TARGET)/\$(CARGO_TARGET_DIR),\$(CARGO_TARGET_DIR))/libcpython_rust_staticlib.a" + LIBS="$LIBS $rust_staticlib" ;; esac - rule="$file: $objs \$(MODULE_${mods_upper}_LDEPS)" - rule="$rule; \$(BLDSHARED) $objs $libs \$(MODULE_LDFLAGS_SHARED) -o $file" - echo "$rule" >>$rulesf - done + fi done case $SHAREDMODS in diff --git a/PCbuild/_base64.vcxproj b/PCbuild/_base64.vcxproj new file mode 100644 index 00000000000000..916069c1aea82a --- /dev/null +++ b/PCbuild/_base64.vcxproj @@ -0,0 +1,164 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4} + _base64 + MakeFileProj + + + + + Makefile + NotSet + false + + + + $(PyStdlibPydExt) + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + llvm-21.1.4.0 + dev + release + debug + release + i686-pc-windows-msvc + x86_64-pc-windows-msvc + thumbv7a-pc-windows-msvc + aarch64-pc-windows-msvc + $(IntDir)cargo\ + $(CargoTargetDir.TrimEnd(`\`)) + $(BuildPath.TrimEnd(`\`)) + $(PySourcePath)PC + $(PySourcePath)Cargo.toml + $(LIBCLANG_PATH) + $(ExternalsDir)$(BundledLlvmTag)\bin + $(ExternalsDir)$(BundledLlvmTag)\bin + $(LLVMInstallDir)\bin + $(LLVMInstallDir)\bin + $(VCInstallDir)Tools\Llvm\x64\bin + $(VCInstallDir)Tools\Llvm\x64\bin + $(VCInstallDir)Tools\Llvm\bin + $(VCInstallDir)Tools\Llvm\bin + C:\Program Files\LLVM\bin + C:\Program Files\LLVM\bin + $(OutDir) + $(TargetDir)$(TargetName)$(TargetExt) + $(CargoTargetDirNoSlash)\$(CargoTarget)\$(CargoBuildSubdir)\_base64.dll + $(CargoTargetDirNoSlash)\$(CargoTarget)\$(CargoBuildSubdir)\_base64.pdb + $(OutDir)$(TargetName).pdb + setlocal +if "$(Configuration)"=="Debug" (set "PY_DEBUG=1") else (set "PY_DEBUG=") +if "$(DisableGil)"=="true" (set "PY_GIL_DISABLED=1") else (set "PY_GIL_DISABLED=") +where cargo >NUL 2>NUL || (echo WARNING: cargo was not found on PATH, skipping _base64 & exit /b 0) +if not "$(LibClangPath)"=="" set "LIBCLANG_PATH=$(LibClangPath)" +if "$(LibClangPath)"=="" if not defined LIBCLANG_PATH (echo WARNING: libclang was not found, skipping _base64 & exit /b 0) +if defined LIBCLANG_PATH if not exist "%LIBCLANG_PATH%\libclang.dll" if not exist "%LIBCLANG_PATH%\clang.dll" (echo WARNING: libclang.dll not found at %LIBCLANG_PATH%, skipping _base64 & exit /b 0) +if not exist "$(CargoTargetDirNoSlash)" mkdir "$(CargoTargetDirNoSlash)" +set "CARGO_TARGET_DIR=$(CargoTargetDirNoSlash)" +set "PYTHON_BUILD_DIR=$(PythonBuildDir)" +set "LIBPYTHON="/LIBPATH:$(BuildPathNoSlash)" "$(PyDllName).lib"" +cargo build --lib --locked --package _base64 --manifest-path "$(CargoWorkspaceManifest)" --profile $(CargoProfile) --target $(CargoTarget) +if errorlevel 1 exit /b 1 +if not exist "$(RustExtensionPath)" (echo ERROR: cargo did not produce $(RustExtensionPath) & exit /b 1) +copy /Y "$(RustExtensionPath)" "$(TargetPath)" >NUL +if errorlevel 1 exit /b 1 +if exist "$(RustExtensionPdb)" copy /Y "$(RustExtensionPdb)" "$(TargetPdb)" >NUL + if exist "$(TargetPath)" del /F /Q "$(TargetPath)" +if exist "$(TargetPdb)" del /F /Q "$(TargetPdb)" +if exist "$(CargoTargetDirNoSlash)" rmdir /S /Q "$(CargoTargetDirNoSlash)" + $(TargetPath) + + + + + + + + + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} + false + + + + + + + + + + diff --git a/PCbuild/_base64.vcxproj.filters b/PCbuild/_base64.vcxproj.filters new file mode 100644 index 00000000000000..0ba48e32c44518 --- /dev/null +++ b/PCbuild/_base64.vcxproj.filters @@ -0,0 +1,19 @@ + + + + + {86A93473-99A3-4B0B-9B78-4A09FF3A0A6F} + + + + + Rust Sources + + + Rust Sources + + + Rust Sources + + + diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 9d2f032f5a9355..ffa09a1b9de3f1 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -116,7 +116,7 @@ if "%IncludeExternals%"=="" set IncludeExternals=true if "%IncludeCTypes%"=="" set IncludeCTypes=true if "%IncludeSSL%"=="" set IncludeSSL=true if "%IncludeTkinter%"=="" set IncludeTkinter=true -if "%UseJIT%" NEQ "true" set IncludeLLVM=false +if "%IncludeLLVM%"=="" if "%UseJIT%"=="true" (set IncludeLLVM=true) else (set IncludeLLVM=false) if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 53aec0276beaee..fa33fcb6a77754 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -64,7 +64,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index e2cac3158831ee..e45d48b47d12c2 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -23,6 +23,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python", "python.vcxproj", {28B5D777-DDF2-4B6B-B34F-31D938813856} = {28B5D777-DDF2-4B6B-B34F-31D938813856} {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} = {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} {384C224A-7474-476E-A01B-750EA7DE918C} = {384C224A-7474-476E-A01B-750EA7DE918C} + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4} = {D7B7A122-80A0-4AFB-9C06-18C29B3178E4} {447F05A8-F581-4CAC-A466-5AC7936E207E} = {447F05A8-F581-4CAC-A466-5AC7936E207E} {4946ECAC-2E69-4BF8-A90A-F5136F5094DF} = {4946ECAC-2E69-4BF8-A90A-F5136F5094DF} {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} = {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} @@ -125,6 +126,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsol EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_base64", "_base64.vcxproj", "{D7B7A122-80A0-4AFB-9C06-18C29B3178E4}" +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_zoneinfo", "_zoneinfo.vcxproj", "{FCBE1EF2-E0F0-40B1-88B5-00A35D378742}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_queue", "_queue.vcxproj", "{78D80A15-BD8C-44E2-B49E-1F05B0A0A687}" @@ -1281,6 +1284,38 @@ Global {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.ActiveCfg = Release|x64 {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.Build.0 = Release|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|ARM.ActiveCfg = Debug|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|ARM.Build.0 = Debug|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|ARM64.Build.0 = Debug|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|Win32.ActiveCfg = Debug|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|Win32.Build.0 = Debug|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|x64.ActiveCfg = Debug|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Debug|x64.Build.0 = Debug|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|ARM.ActiveCfg = Release|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|ARM.Build.0 = Release|ARM + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|ARM64.ActiveCfg = Release|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|ARM64.Build.0 = Release|ARM64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|Win32.ActiveCfg = Release|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|Win32.Build.0 = Release|Win32 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|x64.ActiveCfg = Release|x64 + {D7B7A122-80A0-4AFB-9C06-18C29B3178E4}.Release|x64.Build.0 = Release|x64 {FCBE1EF2-E0F0-40B1-88B5-00A35D378742}.Debug|ARM.ActiveCfg = Debug|ARM {FCBE1EF2-E0F0-40B1-88B5-00A35D378742}.Debug|ARM.Build.0 = Debug|ARM {FCBE1EF2-E0F0-40B1-88B5-00A35D378742}.Debug|ARM64.ActiveCfg = Debug|ARM64 diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index d54c0ed3e3c633..29f065f05bc767 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -168,6 +168,7 @@ The following sub-projects are for individual modules of the standard library which are implemented in C; each one builds a DLL (renamed to .pyd) of the same name as the project: * _asyncio + * _base64 * _ctypes * _ctypes_test * _decimal diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 565be27b7cebda..45e76b737c6008 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -10,6 +10,7 @@ static const char* _Py_stdlib_module_names[] = { "_ast", "_ast_unparse", "_asyncio", +"_base64", "_bisect", "_blake2", "_bz2", diff --git a/RUST_API.md b/RUST_API.md new file mode 100644 index 00000000000000..869a530fbcad55 --- /dev/null +++ b/RUST_API.md @@ -0,0 +1,1524 @@ +# Rust for CPython: API Design for 3.16 + +## Overview + +This document designs a safe Rust API for in-tree CPython extension modules, +with `zlib` as the first full migration target. + +The wrapper enforces the lifetime, aliasing, and thread-safety rules missing +from raw FFI. It requires CPython support for context enforcement, protection of active thread states, and deferred reference release +without attachment. The appendix lists prerequisite CPython fixes. + +## Glossary + +Standard Rust terms keep their usual meanings; in particular, `&T` is a +shared reference and `&mut T` is an exclusive (mutable) reference. The terms +below have design-specific meanings. + +- **Entry:** a call into Rust from Python or the C API: a module function, + method, or slot callback. Generated **entry glue** wraps the Rust + function; its private **entry guard** keeps resources alive from argument + parsing through cleanup. See [Entry glue](#entry-glue). +- **Execution phase:** the circumstances under which CPython calls Rust: + ordinary execution, finalization cleanup, or GC traversal. Each permits + different operations; see [Execution phases](#execution-phases). +- **Capability:** a value or reference proving permission to perform an + operation: normally a borrow of `ThreadContext`, with restricted + alternatives for GC traversal and reference release. +- **Class payload:** the Rust data stored in a Python instance. See + [Classes](#classes). +- **Leaf operation:** an operation that preserves attachment and cannot + re-enter Python, invoke arbitrary Rust callbacks, run deallocators, or + reach a safepoint. It may still call other C functions. See + [The thread context](#the-thread-context). +- **Safepoint:** a point in a runtime operation where the thread may pause + or service pending work, for example a stop-the-world request. +- **Exclusive context access:** an `&mut ThreadContext` borrow, required by + operations that can run callbacks or change attachment. It is Rust's + exclusive borrow, not a lock on the interpreter or its objects. +- **Available / suspended context:** safe Rust code may use an available + context. Suspension temporarily makes it unavailable around a reentrant + call while keeping the original thread state protected; it does not by + itself detach the thread. See [Scope transitions](#scope-transitions). +- **Interpreter guard:** a PEP 788 `PyInterpreterGuard`, which prevents an + interpreter from finalizing until it is closed. Each ordinary entry owns + one inside its private entry guard. See + [Interpreter guards and entry lifetime](#interpreter-guards-and-entry-lifetime). +- **Admission:** permission to acquire an interpreter guard or submit a + reference for release. Closing admission prevents new ordinary entries + and submissions; finalization callbacks use separate cleanup authorization. +- **Transition check:** runtime enforcement + that rejects a thread-state transition or nested entry that would + invalidate an available Rust context. See + [Transition checks](#transition-checks). +- **Release sink / queue:** a sink buffers references awaiting `Py_DECREF` + for one entry; the queue holds them for an interpreter. **Draining** + performs the deferred decrefs. See + [Deferred reference release](#deferred-reference-release). +- **Foreign code:** C or Rust code outside this API, including other Rust + wrappers of the C API. See + [Detachment and foreign wrappers](#detachment-and-foreign-wrappers). + +## Goals + +In priority order, the API should: + +1. be sound on GIL and free-threaded builds, including subinterpreters with + separate GILs; +2. support every PEP 11 platform; +3. approach equivalent C performance where safety permits; +4. feel familiar to PyO3 and C API users; and +5. use modern APIs such as `PyModExport`. + +It encodes CPython contracts in the type system where practical, preserves +Rust's initialization, aliasing, and concurrency invariants, and keeps +compile times under control. + +**Non-goals:** full C API parity, a stable ABI, and sharing Python objects +between interpreters. Wrappers are added as Rust modules need them. + +### Runtime lifetime + +Before reinitializing Python after `Py_FinalizeEx()`, embedders must ensure +that no old handles, traced slots, interpreter views, or release submissions +can be used or dropped afterward, including those retained in globals or +worker threads. They must dispose of these values and quiesce their producers. + +This safety requirement is unenforced; violating it can cause undefined +behavior. Given the expected rarity of such retention, the design accepts +this risk to avoid generation bookkeeping. Reinitialization is otherwise +permitted. + +### Safety rules + +- Every `unsafe` has a `// SAFETY:` comment stating its proof obligation. +- No panic may escape a non-unwinding C ABI. +- Generated code never expands author-supplied expressions, defaults, or + calls inside an `unsafe` block; it binds them in safe items first. +- A trait whose implementations unsafe code trusts (a type check, type + pointer, or class identity) is `unsafe` or sealed. Otherwise a crate with + `#![forbid(unsafe_code)]` could cause undefined behavior by implementing + it. +- A type parameter carrying an identity that unsafe code relies on (a class + payload, module state, or traced slot's contents) is invariant. + `Payload` is a subtype of `Payload`, so a + covariant parameter would let safe code relabel one as the other. +- A typed handle is created only after a type check, or from a C function + whose result type Python code cannot choose. (A metaclass, for example, + decides what `type(name, bases, dict)` returns.) + +## API design + +1. [Bindings and the FFI boundary](#bindings-and-the-ffi-boundary) +2. [Interpreter identity and lifetime](#interpreter-identity-and-lifetime) +3. [The thread context](#the-thread-context) +4. [Python object handles](#python-object-handles) +5. [Deferred reference release](#deferred-reference-release) +6. [Exceptions](#exceptions) +7. [Concrete objects and conversions](#concrete-objects-and-conversions) +8. [Modules](#modules) +9. [Argument parsing](#argument-parsing) +10. [Functions](#functions) +11. [Classes](#classes) + +### Bindings and the FFI boundary + +`cpython-sys` generates raw bindings at build time from `Python.h`, the +internal headers, and the active `pyconfig.h`, capturing layout that depends +on platform, target, debug, and free-threading configuration. Bindgen +allowlists public and private `Py*`/`_Py*` names; constructs it cannot +translate, such as macros, are reimplemented in Rust. + +Python objects are mutable through shared pointers, so the binding uses +`UnsafeCell`: + +```rust +#[repr(transparent)] +pub struct PyObject(UnsafeCell<_object>); +``` + +All C calls go through a private `c_api` module. Every safe operation that +may touch Python requires a capability (see +[The thread context](#the-thread-context)); an operation without one needs +an audited proof that its whole call path never accesses a thread state. +Until Rust offers a `Drop` that can take a context, `Drop` never obtains a +capability; it only submits a release. + +### Interpreter identity and lifetime + +#### Interpreter IDs + +The API uses `PyInterpreterState_GetID()`: a nonnegative `int64_t` (`i64` +in Rust), with zero identifying the main interpreter. IDs are not reused +within one runtime, but reset on reinitialization. The +[runtime lifetime requirement](#runtime-lifetime) therefore replaces a +separate generation token. PEP 788 views resolve by the same IDs and do +not keep interpreters alive. + +An ID names the interpreter under whose API contract a reference was +acquired, not the one that allocated the object. + +Objects carry no interpreter association. Generated glue reads the ID from +the attached thread state's interpreter, with no allocation, search, or +lock, and conversions preserve it. + +#### Interpreter guards and entry lifetime + +An interpreter ID proves identity, not liveness. Every ordinary entry uses +a [PEP 788 guard](https://peps.python.org/pep-0788/#interpreter-guards) and +`PyThreadState_Ensure(guard)` to establish attachment before exposing a +context or payload borrow. Generated Python callbacks acquire the guard +with `PyInterpreterGuard_FromCurrent()`; native callers use an explicit +interpreter view and `PyInterpreterGuard_FromView()`. + +The private entry guard owns the successful `PyThreadStateToken` and keeps +the interpreter guarded through its release. With a separately acquired +guard, it calls `PyThreadState_Release(token)` before closing the guard; +`PyThreadState_EnsureFromView` manages that guard implicitly. Both paths +finish conversion, unwinding, release processing, and exception reporting +before releasing the token, so the guard covers thread-state cleanup too. + +Acquisition may invoke allocator hooks or fail because admission is closed +or allocation failed. It therefore precedes context availability; failure +follows the [entry-glue policy](#entry-glue), never an unguarded fallback. +Cleanup callbacks use [cleanup authorization](#cleanup-authorization) +without acquiring a guard or an ensure token. Pre-finalization waiting +still admits guarded nested entries. + +Guards cover indirect detachment through Python as well as explicit +`detach` and `interop` calls. This protects: + +- **Ancestor borrows:** if A lends `&S` through scoped TLS to nested entry + B, A's guard remains live while B detaches and reads it. +- **Deferred-counted objects:** on free-threaded builds, finalization cannot + remove the frame roots keeping modules, functions, and payloads alive. +- **Scoped workers:** `&S`, `&C`, and `&[u8]` may cross threads under their + Rust bounds, provided workers finish before the borrowing scope returns. + Waiting for workers that may need Python requires detachment. Contexts + and scoped handles remain `!Send + !Sync`. + +Guards neither retain individual objects nor establish attachment or +synchronization; ordinary ownership and GC rules still apply. Stored +`Py` handles never hold guards, which could make module state prevent +the shutdown needed to destroy it. + +The baseline uses the public API's allocation, shared counter, and admission +barrier. The release queue joins that barrier (see [Shutdown](#shutdown)). + +#### Shutdown progress and guard ownership + +PEP 788 already makes finalization wait for outstanding guards. Holding one +per Rust entry therefore lets a blocked daemon thread delay shutdown +indefinitely. The private entry glue prevents safe code from closing or +forgetting these guards. + + +#### Cleanup authorization + +After guard admission closes, CPython's finalization callbacks and their +synchronous descendants use **cleanup authorization**. It requires the +attached state to be the interpreter's recorded finalizing state; other +entries receive `EntryError`. All other states are marked shutting down +before the world restarts. Authorization cannot be manufactured and ends +at the [final drain point](#shutdown). + +Cleanup may detach and interoperate: the finalizer cannot advance until its +synchronous callback returns. Other threads hang in `take_gil` or +`tstate_wait_attach`; fresh threads hang in `PyGILState_Ensure`. Cleanup +must therefore never wait for a thread that needs the interpreter. +It uses the already-attached finalizing state directly; +`PyThreadState_EnsureFromView` would attempt a new guard acquisition and +fail after admission closes. + +### The thread context + +`ThreadContext<'py>` is a scoped, exclusive capability for one exact thread +state on the current OS thread. Whenever safe code can use it, that state is +attached and protected against finalization. + +```rust +pub struct ThreadContext<'py> { + ptr: NonNull, + interpreter_id: i64, + entry: NonNull, // private, stable storage in the entry guard + #[allow(clippy::type_complexity)] + _marker: PhantomData<(fn(&'py ()) -> &'py (), *mut ())>, +} +``` + +The marker makes `'py` invariant and the context `!Send + !Sync`. It is +neither `Copy` nor `Clone`, and only generated entry glue constructs it. +`'py` is a fresh lifetime per entry, and `Bound` and `Borrowed` are +invariant in it too, so separate entries cannot mix scoped values even +within one interpreter. + +| Context argument | Permitted operations | Examples | +| --- | --- | --- | +| `&ThreadContext<'py>` | Audited [leaf operations](#glossary): no attachment changes, callbacks, deallocators, or safepoints. | Checked binding, ownership-preserving casts, immutable bytes access, audited incref. | +| `&mut ThreadContext<'py>` | Operations that may reenter, allocate through reentrant paths, decref, or change attachment. | Extraction, object creation, attribute access, destruction, release draining, `detach`. | + +The audit covers failure handling, allocator hooks, and cleanup. Shared +operations return Rust errors without setting Python's error indicator; +raising requires exclusive access. New wrappers default to exclusive, and +blocking acquisition is never a leaf operation. Attachment does not prevent +concurrent mutation on free-threaded builds. + +Holding `&mut ThreadContext` does not make the context unavailable. A +private wrapper **suspends** it only around the actual reentrant C call or +closure, and never passes a suspended context to author code. + +#### Execution phases + +An **execution phase** determines which operations CPython permits when it +calls Rust. It is independent of whether a thread is attached or a context +is available or suspended. + +| Execution phase | Permitted operations | +| --- | --- | +| Ordinary execution | The entry holds a PEP 788 interpreter guard throughout and may use whatever its shared or exclusive context borrow allows, including explicit detachment. | +| Finalization cleanup | Callbacks use [cleanup authorization](#cleanup-authorization): they may detach and interoperate, but must not wait on a thread that needs the interpreter. | +| GC traversal | Callbacks use the restricted GC visitor operations, without a general-purpose `ThreadContext`, and must obey the [GC slot synchronization](#gc-slot-synchronization) rules. | + +After the [final drain point](#shutdown), no entry can obtain a context. +The module and instance lifecycle states described later (such as `Ready` +and `Cleared`) track initialization and destruction separately. + +#### Availability flag and thread-state protection + +Borrowing and leaf audits cannot constrain foreign code that detaches or +reenters while an outer context remains accessible through scoped TLS. +CPython therefore enforces two runtime invariants; interpreter guards alone establish neither: + +- An **availability flag** allows at most one available context per OS + thread, across all extensions and interpreters. Lexically scoped entry + and restoration guards need only one flag. Only the owning thread + modifies the flag, so it needs no atomics. +- A synchronized **protection count** per `PyThreadState` prevents clearing + or deletion, even from another thread, while an ordinary or cleanup entry + is active. Protection persists through suspension and interpreter + switches. `PyThreadState_Ensure` handles attachment and its matching + release; it does not replace this protection against explicit clearing + or deletion, or the availability checks. + + +#### Scope transitions + +1. **Enter:** require the flag clear and determine the + [phase](#execution-phases). An ordinary entry acquires a guard and calls + `PyThreadState_Ensure`; cleanup verifies authorization on the attached + finalizing state instead. Expose no context or payload borrow during + setup. Protect the resulting thread state before further reentrant work, + then set the flag and publish the context. On failure, follow the + [entry-glue policy](#entry-glue). +2. **Suspend:** clear the flag before a reentrant operation. Refusal by the + initial checks leaves attachment and the flag unchanged without raising. + The restoration guard keeps the context's state. Suspension does not + broaden the C operations the execution phase permits. +3. **Check:** attach, detach, swap, deletion, and generated entry paths + enforce the [transition checks](#transition-checks) in every build, + including when no state is attached. +4. **Resume:** require the flag clear and the context's state attached, + apply the [exception policy](#exceptions), then set the flag. If a + foreign call failed to restore the state, fail fatally rather than + attach over an unknown state. +5. **Exit:** clear the flag, drain or transfer releases, and retire the empty + sink. Finish the scoped cleanup, result/error transfer, and panic reporting + while still attached to the context's state. Remove its protection, then + release the ensure token, which may delete that state and restores the + caller's prior attachment (possibly none). Close the explicit interpreter + guard last. Cleanup entries have neither a token nor a guard to release. + +Private entry and restoration guards cannot be forgotten by safe code. +Forgetting a public handle may leak a reference but cannot alter runtime +protection. The entry guard keeps its release sink at a stable stack address. + +#### Transition checks + +The availability flag must be clear on: + +- the internal detach path (reached by every public release and swap), + deletion and clearing of the current thread state, `Py_MakePendingCalls`, + and the deprecated `PyEval_ReleaseLock`; +- entry into generated glue, since a nested entry under an available + context would create a second context that could detach independently of + the outer context's borrows. + +Clearing or deleting any thread state also requires a zero protection +count, regardless of the caller's availability flag. Rust wrappers check +permission before `PyEval_SaveThread`; violations by foreign code fail +fatally before changing attachment or invalidating a protected state. +Waiting instead would deadlock the current thread. + +Two paths are exempt: `_PyParkingLot_Park`, through a private thread-local +marker, and the `_PyThreadState_Suspend` stop-the-world safepoint. A path +is exempt only if no callback or Rust access can run before the exact +state is restored. The parking lot calls the public +`PyEval_ReleaseThread`/`PyEval_AcquireThread`, so the private marker is +what distinguishes it from explicit detachment. Lock paths that call +`Py_MakePendingCalls` for signals require exclusive context access and +suspension instead. Automatic release draining stays deferred during an +exempt restoration while a context is available; otherwise a decref would +introduce callbacks into a path justified by their absence. + +The check does not stop foreign code from running Python on the attached +state (`PyObject_Call` from another library, or `PyErr_CheckSignals`), +which need not violate that library's contract. Every Rust boundary such +code can reach enforces the flag in every build: a nested entry or +forbidden transition fails fatally, GC clear of a Rust object is refused +before constructing a context (leaking the cycle), and payloads are dropped +without a context as always. Foreign code can also set an exception without +a transition; wrappers handle this through the +[exception policy](#exceptions). + +#### Entry glue + +Ordinary entry uses the [PEP 788 attachment APIs](https://peps.python.org/pep-0788/#attaching-and-detaching-thread-states): + +- Generated Python callbacks already have an attached state under their C + API contract. `PyThreadState_Ensure(guard)` reuses that state for the + guarded interpreter and returns a token for the matching release. +- Native entry supplies an explicit interpreter view, acquires its guard, + and uses the same ensure/release path. It never implicitly selects the + main interpreter through `PyGILState_Ensure`. +- `PyThreadState_EnsureFromView(view)` can combine native guard acquisition + and attachment. Its successful token carries the guard's lifetime; + release it once, without separately closing the implicit guard. + +The token is opaque: it records how to undo attachment, not necessarily the +state that was attached. After success, glue reads the attached state to +construct `ThreadContext`. Each successful ensure has exactly one matching +release on the same thread, in nesting order, including on panic or later +setup failure. The ensured state must be restored before release. No scoped +handle, borrowed view, or pending Python exception may escape a native +entry; results and errors must be converted to Rust values or unbound +handles before release. + +A simplified view of the entry glue for a function call: + +```rust +// SAFETY: only generated C callbacks with an attached state and valid inputs +// may enter; the implementation enforces availability, checks the phase, +// and establishes a guarded or cleanup-authorized context, or refuses entry. +unsafe fn with_current( + f: impl for<'py> FnOnce(&mut ThreadContext<'py>) -> R, +) -> Result; +``` + +`for<'py>` makes the lifetime fresh, so `R` cannot depend on it, and macros +reject author functions that name a specific lifetime such as `'static`. +Generated wrappers convert results to raw C values before returning. +`EntryError` is a Rust error value; callback-specific glue reports it +without manufacturing a context after admission closes. + +Ensure calls return `NULL` on failure without setting a Python exception. +Glue must not release a failed token and must close any guard it acquired +separately. Native entry reports setup failure as a Rust error without +assuming attachment; it never runs the body on failure. For generated +callbacks, guard or attachment setup failure follows the callback's ABI: + +- Functions, methods, constructors, and module initialization return their + ABI's failure sentinel without calling the normal Rust body. They preserve + an acquisition exception, or report allocation failure as `MemoryError` + through private reporting authorization when no exception was set. +- Clear callbacks leave payloads and slots untouched. Module clear returns + 0 with an exception so CPython still clears the module dict; instance + clear returns failure. Refusal may leak a cycle. +- `tp_dealloc` and module free save and clear the incoming exception before + entry setup. On failure they still perform mandatory teardown + without a context: commit the dropping state, drop initialized Rust data + exactly once, and run the allocation/type cleanup guards. They skip + context-requiring clear callbacks; field drops submit releases without + attaching. Errors follow the [deallocator exception policy](#dropping). + Teardown must complete safely or abort. + +These failure paths retain any established private setup protection until +their cleanup completes, and never hand a context to a destructor. GC +traversal uses its restricted visitor capability and does not allocate an +interpreter guard. After the final drain point, failure diagnostics must +not invoke Python: use a native diagnostic or abort, while preserving +mandatory no-context teardown. + +#### Detachment and foreign wrappers + +```rust +impl<'py> ThreadContext<'py> { + pub fn detach( + &mut self, + f: impl FnOnce() -> R, + ) -> PyResult<'py, R>; + + pub fn interop( + &mut self, + f: impl FnOnce() -> R, + ) -> PyResult<'py, R>; + + pub fn drain_releases(&mut self); +} +``` + +- **`detach`** suspends, detaches, runs `f` on the same thread, and restores + the state even on panic. Nested and cleanup entries may detach. Their + [lifetime protection](#interpreter-guards-and-entry-lifetime) remains in + force, so `f` may use entry-borrowed `&S`, `&C`, and `&[u8]` under their + owner lifetimes. Waiting for another thread that needs Python requires + detachment to avoid GIL or stop-the-world deadlocks; during cleanup that + thread is hung, so waiting deadlocks regardless. +- **`interop`** suspends without detaching, so `f` can use a foreign safe + wrapper that manages its own attachment, including switching + interpreters. The interpreter guard or cleanup authorization and the + thread-state protection remain in force. The boundary verifies restoration + of the incoming state, and nested entries obey availability rules. Foreign + wrappers must uphold their own safety contracts. +- **`drain_releases`** services queued releases (see [Draining](#draining)). + +A pending exception is the only refusal: both APIs return it as `Err` +before suspending. On return, an exception left by `f` replaces its result +with `Err`, before availability is restored. See [Exceptions](#exceptions). + +The closures receive no context and need no `Send` bounds: they run on the +same thread, and the proof is exclusive access to the capability. Handles +may move in and out but remain unusable without an available context. + +This closes the holes in PyO3 +[#2141](https://github.com/PyO3/pyo3/issues/2141) and +[#3640](https://github.com/PyO3/pyo3/issues/3640), where `SendWrapper` or +scoped TLS let code reach Python without proof of attachment. Here a shared +context reference stashed in TLS blocks the exclusive borrow that detaching +needs, a `RefCell` holding the context fails a reentrant borrow, and +foreign detachment and unsuspended nested entry are rejected by runtime +enforcement in every build. Likewise a TLS hook running while a C call has +switched to interpreter B cannot use A's context, which stays exclusively +borrowed. + +### Python object handles + +| Handle | Owns a reference | Lifetime | Auto traits | Drop | +| --- | --- | --- | --- | --- | +| `Borrowed<'a, 'py, T>` | No | owner `'a` and attachment `'py` | `Copy`, `!Send`, `!Sync` | none | +| `Bound<'py, T>` | Yes | guarded entry `'py` | `!Send`, `!Sync` | enqueue in entry release sink | +| `Py` | Yes | unbound, for stored state | `Send`, `Sync` | enqueue by interpreter ID | + +`Borrowed` stores only a pointer; its owner supplies `'a` and the context +supplies `'py`. `Bound` stores the pointer and a pointer to its entry's +release sink, which records the interpreter ID. `Py` stores the pointer +and an `i64` interpreter ID. Both owning forms are two words on 64-bit +targets. The entry's `'py` lifetime proves association, not permission: +handles can exist while the context is suspended but cannot be used +without an available context. + +#### Binding + +Every safe `Py` operation that touches the object (binding, downcasting, +cloning, conversion, destruction) first compares its interpreter ID with the +context's, before dereferencing anything. Mismatch returns the Rust error +`InterpreterMismatch` without setting Python's error indicator. This also +rejects expired handles even if a later subinterpreter reuses their address. +Reuse of IDs across runtime reinitialization is covered by the +[runtime lifetime requirement](#runtime-lifetime). + +```rust +impl Py { + pub fn bind<'a, 'py>( + &'a self, + ctx: &ThreadContext<'py>, + ) -> Result, InterpreterMismatch>; +} +``` + +Scoped handles share the context's invariant `'py`, so they need no +repeated checks. Moving between forms is explicit: + +```rust +let stored: Py = exception_type.unbind(); +let current: Borrowed<'_, 'py, PyType> = stored.bind(ctx) + .map_err(|err| err.raise(ctx))?; +let retained: Bound<'py, PyType> = current.to_owned(ctx); +``` + +Owning handles do not implement `Clone`; `clone_ref(ctx)` and +`to_owned(ctx)` make increfs explicit. Raw-pointer constructors are private +and prove non-nullness, identity, liveness, and ownership. Upcasts, +unbinding, and transfers preserve the interpreter ID. + +#### References into Python memory + +Rust references into Python-owned memory (byte slices, class payloads, +module state) are bounded by both the owner's lifetime and the entry scope. +A static `Py` cannot yield a `'static` `&[u8]`, `&C`, or `&S`, and the +entry cannot return such references. A view that needs continuous +attachment also borrows the context. + +Attachment is not a lock: other threads and foreign wrappers can mutate +Python objects. Shared class and module payloads must be `Send + Sync`. + +#### Dropping + +```rust +object.drop_with(ctx)?; +``` + +`drop_with(&mut ctx)` releases directly, with exclusive context access and +the context suspended during the decref. + +Ordinary `Drop` never decrefs, attaches, or switches states, even on the +right interpreter: a finalizer could reenter Python while a shared context +is reachable through TLS. `Bound::drop` appends to its entry's sink; +`Py::drop` submits to its interpreter's queue. Neither reads the object, +and `Drop` must not unwind. Sealed markers for immortal singletons (`None`, +booleans) skip release entirely. + +**Deallocator exception policy.** Explicit release, draining, `tp_dealloc`, +and module free preserve the exact incoming exception, including none. +CPython debug builds abort if a deallocator changes the exception. After the final +drain point, diagnostics must not invoke Python. + +#### Markers and GC + +`PyAny` is the marker for any Python object; specific markers like `PyBytes` +share its transparent layout, so upcasts are free. There is no `&mut PyAny`, +since Python objects can change through aliases. + +### Deferred reference release + +#### Entry sinks + +Each entry has a release sink with a small inline buffer, owned by the entry +guard. `Bound::drop` appends to it with no allocation, lock, or Python call. +When it fills, it transfers a batch to the interpreter queue or obtains +overflow storage, without calling Python, attaching a thread state, or +decrefing. If both fail, only the incoming reference is abandoned; accepted +ones are kept. + +**Aliasing invariant:** no `&mut` to sink storage may be live across an +operation that can drop a handle into that sink. Pushes use interior +mutability, and draining extracts a batch before decrefing. + +`Py::drop` always uses the interpreter queue, even inside a same-interpreter +entry; `drop_with` avoids the queue when promptness matters. Exit drains or +transfers every buffered reference before retiring the sink, draining +locally if transfer fails. After admission closes, authorized cleanup sinks +overflow through a private cleanup route. + +#### The release queue + +CPython owns one release queue per interpreter lifetime. It accepts +references without an attached thread state through a new private API: + +```c +typedef enum { + _Py_RELEASE_QUEUED, + _Py_RELEASE_CLOSED, + _Py_RELEASE_EXPIRED, + _Py_RELEASE_NO_MEMORY, +} _PyDeferredReleaseResult; + +_PyDeferredReleaseResult _PyObject_TryDeferDecRef( + int64_t interpreter_id, + PyObject *owned_reference); +``` + +Only `_Py_RELEASE_QUEUED` consumes the reference: the queue takes it over +without an incref, and the producer must not also decref it. Failure +consumes nothing, raises nothing, and does not inspect the object. + +Submission works from any OS thread (detached, in another interpreter, +or after `Py_FinalizeEx`). It must not read the object, touch its refcount, +call Python, attach, panic, or wait for the target interpreter. Its +allocation must not call Python or attach a thread state: `PyMem_RawMalloc` +does not qualify, because under tracemalloc its hook calls +`PyGILState_Ensure`; C `malloc` and an OS mutex do. + +A process-lifetime registry maps interpreter IDs to queue endpoints and +rejects late submissions after `Py_FinalizeEx()`. Lookup, admission, and +notification synchronize with retirement, which removes the endpoint and +waits for in-flight producers before freeing the interpreter. A missing ID +means expired. Before reinitialization reuses IDs, all old endpoints must +be retired and producers quiesced under the +[runtime lifetime requirement](#runtime-lifetime); the registry cannot +distinguish stale submissions carrying a reused ID. + +#### Notification + +After submitting a reference, a producer must notify the interpreter that +its release queue needs attention. The notification cannot use a pending-call +array entry, because that fixed-size array may be full. All eight +eval-breaker bits are already assigned, so the design reuses the existing +calls-to-do bit rather than adding another. + +Each interpreter has a separate `releases_pending` flag. A producer sets +this flag when it submits work. When the flag changes from clear to set, +the producer also sets the calls-to-do eval-breaker bit, requesting that +the interpreter check its release queue. Further submissions while the +flag is set need no additional notification. This does not consume a slot +in the pending-call array. + +A producer attached to the target interpreter sets its own thread state's +eval breaker. Detached producers and those attached to another interpreter +signal the target's thread states while holding the thread-list lock. +GIL builds also check `releases_pending` when acquiring the GIL. + +Each drain triggered by eval-breaker handling or GIL acquisition processes +at most a bounded number of queue nodes, so continuous submissions cannot +keep the interpreter draining indefinitely. If work remains, or draining +is deferred because a Rust context is available, the consumer keeps the +work marked pending and requests another check. + +#### Waking the finalizer + +A finalizer waiting for guards or submissions must be woken without +attachment. Releasing the last guard may allow immediate interpreter +deletion, so the wake word and waiter count live in process-global storage. +After releasing lifetime protection, producers touch only that storage. +Wake events are setting `releases_pending`, closing the last PEP 788 guard, +and the in-flight submission count reaching zero. + +If waiters exist, a waker increments the word and calls +`_PyParkingLot_UnparkAll`: waking only one could select the wrong +interpreter's finalizer. The finalizer registers first, then repeatedly +reads the word, services releases, checks counts, and parks expecting the +word it read. Sequentially consistent atomics prevent lost wakeups. + +The loop must also run pending calls before parking, and scheduling one +must wake finalizers. Otherwise a guard holder waiting on a call scheduled +after the last `_Py_FinishPendingCalls` deadlocks (see the appendix). + +#### Draining + +Draining requires exclusive context access and suspension via +`ctx.drain_releases()`, or private release authorization between scopes. +Eval-breaker handling and GIL acquisition check availability before draining +in every build; if a context is available, they defer draining and preserve +the pending notification. This includes reattachment through the exempt +parking-lot and stop-the-world paths. + +A consumer drains while attached. An explicit drain remains covered by its +entry's interpreter guard or cleanup authorization. An automatic consumer +outside an entry does not acquire a guard merely for its batch; if a decref +callback detaches and shutdown hangs that unguarded consumer, the remaining +references leak, as references held by hung C code do today. Any Rust entry +reached by a decref acquires its own guard or cleanup authorization. The +consumer releases locks that do not require attachment before decrefing. +Each reference is released exactly once, attached to its own interpreter; +until then it counts as an external strong reference for GC, so moving an +edge out of a GC field removes the field's edge before submitting the +release. + +Deferred release promises neither destruction before `Drop` returns nor +destruction on the dropping thread. Resources needing prompt release should +expose an explicit close method. + +#### Shutdown + +1. **Open:** the queue accepts and drains normally. +2. **Pre-finalization:** the finalizer waits for threads, pending calls, + and PEP 788 guards, including those owned by Rust entries, while the + interpreter is intact. It services releases and releases its GIL and + locks while waiting. Admission remains open so guarded entries can make + nested calls; an ordinary Rust entry must finish its cleanup and close + its guard before the finalizer can close admission. +3. **Close admission:** queue admission closes with the existing + finalization-guard barrier. The final check covers queued releases and + in-flight submissions alongside threads, pending calls, and guards, with + no check-then-enqueue gap. This integrates with + `make_pre_finalization_calls` rather than running after it with the world + stopped. Afterwards, normal entries and consumers are refused. +4. **Cleanup drains:** authorized finalization callbacks drop into a cleanup + sink. External submissions get `CLOSED`, or `EXPIRED` after retirement. + Cleanup sinks drain to completion after module finalization, before + interpreter clear, and within it before the last collection, after + `_PyGC_Fini`, and at the final drain point, covering drops from + module/type cleanup, the last collection, `gc.garbage`, and teardown of + warnings state, `sysdict`, and `builtins`. +5. **Final drain point:** right after the interpreter empties `sysdict` and + `builtins`, before it releases them and clears its common constants. + After that, running any bytecode is unsafe (even `return None` loads a + common constant), so cleanup authorization ends: no entries, no + consumers, and the cleanup route closes. Later callbacks degrade safely: + deallocation drops payloads without a context, clear leaks, and functions + raise through the plain C API. Every reference accepted before this point + is released; none is accepted after. + +If the interpreter is cleared from another interpreter's thread state, no +drain can run and the queue is abandoned at retirement. + +#### Abandonment + +On `CLOSED`, `EXPIRED`, or `NO_MEMORY`, `Drop` abandons the reference and +records a diagnostic without calling Python, attaching a thread state, +allocating, or panicking. It never retries in another interpreter. This can +leak but never exposes an expired object. Since safe Rust can forget +handles, correctness never depends on destructors running. Reclaiming every +retained `Py` would need revocable ownership records on every handle +(see the appendix). + +#### Fork + +Fork support with active Rust entries is still TBD. + +### Exceptions + +```rust +#[must_use] +pub struct PyErr<'py> { + exception: Bound<'py, PyBaseException>, +} + +pub type PyResult<'py, T> = Result>; +``` + +`PyErr<'py>` owns a `Bound` exception rather than leaving it pending on the +thread state. All constructors require a context and preserve its invariant +`'py`; dropping the error queues a release without leaving a pending +exception. To retain an error beyond the entry, `unbind()` it as +`Py` and later rebind with an interpreter-ID check. +[`#[must_use]`](https://doc.rust-lang.org/std/attribute.must_use.html) warns +when an error is ignored. + +Exception types provide `raise(&mut ctx, message)`, which returns an owned +`PyErr`; the exception is set only when the generated boundary restores it: + +```rust +fn checked_size<'py>( + ctx: &mut ThreadContext<'py>, + size: usize, +) -> PyResult<'py, i32> { + i32::try_from(size) + .map_err(|_| PyOverflowError::raise(ctx, "size is too large")) +} +``` + +Before an error-sensitive C call, the wrapper captures any pending exception +and returns it without making the call. Leaf operations may preserve the +indicator; release and deallocation use their save/clear/restore policy. +On failure, the wrapper immediately takes the exception with +`_PyErr_GetRaisedException` while controlling the originating state: + +```rust +// After checking for a pending exception and making the C call, +// with the exclusive borrow held: +if ptr.is_null() { + // SAFETY: the audited C API's NULL result guarantees an exception on + // this exact, restored state. Capture transfers the raised reference. + Err(unsafe { take_raised_exception(call_state) }) +} else { + // Require a clear indicator before transferring the owned result. +} +``` + +Ambiguous sentinels also check the error indicator. A missing exception +becomes `SystemError`. Success with a pending exception discards the result, +with exclusive context access and the context suspended, and returns the +captured exception. Failures while creating or reporting an error must +leave an appropriate exception such as `MemoryError`. + +At a callback boundary, a pending exception turns `Ok` into `Err`, even if +author code made no further wrapped calls. An explicit `Err` takes priority; +secondary exceptions are unraisable. The invariant `'py` excludes errors +from other entries, so no interpreter-ID check is needed. Restoration may +decref the previous exception and requires exclusive access and suspension. + +Safe foreign code can set the error indicator without changing attachment. +Function entries therefore capture unexpected pending exceptions before +author code; deallocators [save and restore them](#dropping). + +Suspended author closures, including bytes builders, use the same capture +policy. Discarded results and panic payloads are dropped while suspended, +and the indicator is checked again afterwards. During unwinding, restoration +captures secondary exceptions for later unraisable reporting at the boundary; +it neither replaces the original panic nor unwinds itself. + +The built-in exception marker trait is sealed because its type pointer goes +straight to C. Custom exception types stored as `Py` need checked +binding and `PyExceptionClass_Check` before use as exception classes. +Converting Rust error values into a `PyErr` requires an explicit mutable +context. + +### Concrete objects and conversions + +Concrete types such as `PyBytes` and `PyModule` are transparent markers with +type-specific methods on `Bound<'py, Marker>`. The set grows with module +needs. `Bound` can add values and types. + +`PyBaseException::new_exception_type` wraps `PyErr_NewException`, returning +`PyResult<'py, Bound<'py, PyType>>`. A custom metaclass can choose the C +function's unchecked result, so the wrapper uses +[`PyExceptionClass_Check`](https://docs.python.org/3/c-api/exceptions.html#c.PyExceptionClass_Check) +to require `BaseException` or a subclass, otherwise returning `TypeError`. +A type check alone is insufficient. Constructing a typed handle from an +unchecked C result is `unsafe`. + +`PyBytes::new_with` lends a zeroed unpublished buffer; `new_with_uninit` +skips zeroing: + +```rust +let bytes = PyBytes::new_with(ctx, output_len, |output| { + encode_into(input, output); + Ok(()) +})?; + +// SAFETY: the callback writes every output element before returning `Ok(())`. +let bytes_uninit = unsafe { + PyBytes::new_with_uninit(ctx, output_len, |output| { + initialize_every_byte(output); + Ok(()) + }) +}?; +``` + +On error or panic the unpublished object is discarded unread. The closure +runs with the context suspended. + +#### Conversions + +```rust +pub trait FromPyObject<'a, 'py>: Sized { + fn extract( + ctx: &mut ThreadContext<'py>, + obj: Borrowed<'a, 'py, PyAny>, + ) -> PyResult<'py, Self>; +} + +pub trait IntoPyObject<'py> { + fn into_pyobject( + self, + ctx: &mut ThreadContext<'py>, + ) -> PyResult<'py, Bound<'py, PyAny>>; +} +``` + +Both take exclusive access and are fallible, and implementations receive an +available context. `FromPyObject` keeps the owner lifetime `'a`, so borrowed +arguments cost no refcount; a reference `'view` into Python storage requires +`'a: 'view` and `'py: 'view`. The trait cannot return views that need +continuous attachment. + +Exact, published `PyBytes` storage is immutable and safe to read while +detached, so its accessor is a leaf operation bounded by owner and entry +rather than by the context borrow: + +```rust +impl<'a, 'py> Borrowed<'a, 'py, PyBytes> { + pub fn as_bytes<'view>( + &self, + ctx: &ThreadContext<'py>, + ) -> &'view [u8] + where + 'a: 'view, + 'py: 'view; +} +``` + +It neither increfs nor copies: + +```rust +let input = data.as_bytes(ctx); +let output = ctx.detach(|| compress_without_python(input))?; +``` + +This proof does not extend to arbitrary buffer exporters (see the +appendix). + +Generic implementations compose: + +```rust +let value: Option = FromPyObject::extract(ctx, argument)?; +let object: Bound<'py, PyAny> = value.into_pyobject(ctx)?; +``` + +Callbacks may return `T` or `PyResult<'py, T>`; only `PyResult` lets the +body raise, though converting `T` can still fail. + +### Modules + +Modules use PEP 793's `PyModExport` and `PySlot`. `export_module!` builds the +method table, ABI declaration, slot array, lifecycle callbacks, and export +symbol: + +```rust +export_module! { + name: _base64, + doc: c"Base64 encoding implemented in Rust", + state: Base64State, + methods: [standard_b64encode], + exec: base64_exec, +} +``` + +It binds `name` and `doc` to `const` items in safe code before the `unsafe` +slot construction. The slot, method, and ABI table types have private fields +and `unsafe` constructors that only the macro calls. `#[pyfunction]` emits a +`ModuleMethodDef`, and `export_module!` accepts only methods with its +state `S`. + +#### Definition identity + +```rust +pub struct ModuleRef<'a, 'py, S> { + module: Borrowed<'a, 'py, PyModule>, + _state: PhantomData S>, +} +``` + +Only generated `ModuleDef` glue constructs a `ModuleRef`, ensuring a +valid state type and interpreter. Static definitions hold no +interpreter-owned objects; each module instance has its own state, types, +and exceptions. + +The module token is its `ModuleDef` static's address. A private +`UnsafeCell` field prevents const promotion and merging identical statics. +`ModuleDef`, `ModuleRef`, and `ModuleMethodDef` are invariant in `S` +via `fn(S) -> S`, per the [safety rules](#safety-rules). + +#### Module object access + +Functions and methods may request an optional +`module: ModuleRef<'a, 'py, S>` parameter immediately after `mod_state`. +The macro recognizes its type and supplies the module used to obtain +`mod_state` (the function's module or the defining class's module); it is +not a Python argument. Module initialization already receives `ModuleRef`. + +```rust +impl<'a, 'py, S> ModuleRef<'a, 'py, S> { + pub fn as_object(&self) -> Borrowed<'a, 'py, PyModule>; +} +``` + +`module.as_object()` copies the stored borrowed handle without an incref; +its lifetime remains bounded by the module owner and entry scope. This +avoids storing a reference back to the module in its state. + +#### Module state + +Every module has typed state, even if empty: + +```rust +pub trait ModuleState: Send + Sync + 'static + Sized { + type Gc: ModuleStateGc; + + fn new<'a, 'py>( + ctx: &mut ThreadContext<'py>, + module: ModuleRef<'a, 'py, Self>, + ) -> PyResult<'py, Self>; +} + +// SAFETY: implementations must validate interpreter association, report every +// strong Python edge, and synchronize traversal, mutation, and partial clear. +pub unsafe trait ModuleStateGc: 'static { + const ENABLED: bool; + + fn validate<'py>(value: &S, ctx: &mut ThreadContext<'py>) -> PyResult<'py, ()>; + fn traverse(value: &S, visit: &Visit<'_>) -> TraverseResult; + fn clear(value: &S, ctx: &mut ThreadContext<'_>); +} + +#[derive(ModuleStateGc)] +struct Base64State { + error: GcSlot, +} + +impl ModuleState for Base64State { + type Gc = Base64StateGc; + + fn new<'a, 'py>( + ctx: &mut ThreadContext<'py>, + _module: ModuleRef<'a, 'py, Self>, + ) -> PyResult<'py, Self> { + Ok(Self { error: GcSlot::empty(ctx) }) + } +} +``` + +Traced fields use `GcSlot`, which owns an optional reference with an +interpreter ID fixed at construction. GC clear permanently closes a slot, +so reentrant code cannot repopulate it: + +```rust +impl GcSlot { + pub fn empty(ctx: &ThreadContext<'_>) -> Self; + + pub fn get<'py>( + &self, + ctx: &mut ThreadContext<'py>, + ) -> Result>, InterpreterMismatch>; + + pub fn replace<'py>( + &self, + ctx: &mut ThreadContext<'py>, + value: Option>, + ) -> Result>, GcSlotReplaceError<'py, T>>; +} + +pub struct GcSlotReplaceError<'py, T> { + pub kind: GcSlotError, + pub value: Option>, +} +``` + +`get` and `replace` take exclusive access because they may park, and check +the interpreter ID before touching the pointer. `get` increfs under +protection and returns `None` if cleared. `replace` returns the old +reference so it is released outside the lock; on mismatch or a closed slot, +it returns the incoming value unchanged inside the error. Replacing with +`None` empties without closing. Nothing exposes `&mut` to the stored handle. + +#### Deriving GC policies + +`#[derive(ModuleStateGc)]` and `#[derive(PyClassGc)]` classify fields by +their actual type through a sealed trait, since an alias can hide the +spelling: + +- `GcSlot` is traced. +- `Option`, `Box`, `Vec`, arrays, and boxed slices of slots are traced slot + by slot; none can change through `&`. Spelling one directly is a macro + error asking for direct fields. +- `Arc>` and `Rc>` are compile errors, even behind an + alias, because shared ownership would double-report edges. The error + fires after monomorphization. +- `Py` is rejected by default, even inside a wrapper or behind an alias, + because the collector cannot see it and a cycle through it would leak. + Use `GcSlot` instead, or mark a field `#[gc(untraced)]` to keep a + reference that cannot be part of a cycle. +- Anything else is untraced; a slot hidden in another wrapper leaks its + cycles, which is safe. + +`ENABLED` comes from `core::mem::needs_drop` on the field types, because no +spelling proves a field holds no slot (even `u64` can be shadowed). +`GcSlot` has drop glue, so callbacks are omitted only when no field does. +Slots in `ManuallyDrop`, `MaybeUninit`, or unions escape the test but are +never released either, so the effect is a leak. Manual policies are `unsafe` +and must uphold the same rules. + +#### GC slot synchronization + +Clear uses synchronized removal through `&S`, never `&mut S` across a +decref. Traversal reads slots without locking, relying on this exclusion: + +- Slot access requires attachment. A holder never parks, runs callbacks, or + detaches while protected. +- Traversal, `gc.get_referents`, and `gc.get_referrers` run under the GIL or + stop-the-world. `tp_clear` runs after the world restarts and uses the + synchronized path. +- Debug `PyObject_GC_Track` can traverse without stopping the world, so + factories keep the object exclusive until tracking returns. + +Object traversal must not block, run Python, or hold an exclusive borrow +during a visit; lookups use the `_DuringGC` accessors. Clear releases +references only after the slot is closed and unlocked. External callers of +`tp_traverse` must provide the same exclusion. + +#### State lifecycle + +CPython creates the module and binds its functions before allocating state; +`PyModule_Exec` then allocates zeroed `StateStorage` (interpreter ID, +atomic lifecycle state, `MaybeUninit`). An unexecuted module is +reachable, from `importlib.util.module_from_spec` or `sys.modules` +mid-import, so every callback handles missing storage. The macro rejects +over-aligned types. + +| Operation | Allowed source state | Published state and access | +| --- | --- | --- | +| Any callback before exec | No storage | Functions and methods raise `SystemError("module is not initialized")`; traverse, clear, and free do nothing. | +| Claim exec | `Uninitialized` | `Initializing`; any other source raises `SystemError`. | +| Finish `S::new` | `Initializing` | Validate traced associations before publication; on failure drop the unpublished value and publish `FailedWithoutValue`; otherwise write `S` and publish `Executing`. | +| Finish exec | `Executing` | `Ready` or `FailedWithValue`; `S` remains live. | +| Read state | `Ready` | Check storage identity; borrow `&S` for the intersection of the module-owner lifetime and guarded entry scope. | +| Traverse | `Executing`, `Ready`, `FailedWithValue`, `Clearing`, `Cleared`, `ClearFailed` | Borrow `&S`; never traverse `Dropping` or `Dropped`. | +| Clear | `Ready` or `FailedWithValue` | Enter `Clearing`, clear slots through `&S`, then publish `Cleared` or `ClearFailed`; later clears are no-ops. | +| Free with value | `Executing`, `Ready`, `FailedWithValue`, `Clearing`, `Cleared`, or `ClearFailed` | Enter `Dropping`, drop once, then publish `Dropped`. | +| Free without value | `Uninitialized` or `FailedWithoutValue` | Publish `Dropped` without calling `drop_in_place`. | + +```rust +type ModuleExec = for<'a, 'py> fn( + &mut ThreadContext<'py>, + ModuleRef<'a, 'py, S>, + &S, +) -> PyResult<'py, ()>; +``` + +Publishing stores use release ordering and reads use acquire. Guards apply +failure states on unwind, and a panicking destructor is never retried. +Callbacks see state only once `Ready`, since exec may still be populating +it. + +#### Support declarations + +Generated modules declare `Py_MOD_GIL_NOT_USED`, +`Py_MOD_PER_INTERPRETER_GIL_SUPPORTED`, the internal exact-version ABI, and +the build mode. Foreign wrappers are supported through +`interop`, subject to their own safety contracts and the runtime's +availability and state-protection checks. Scoped worker threads are +permitted under the entry-lifetime rules; neither thread attachment nor +Python access may be inferred from a borrowed payload on a worker. +Process-global Rust caches may hold only Rust data or checked opaque +handles. The identity and lifetime checks apply even when import checks are +disabled. + +### Argument parsing + +```rust +#[pyfunction(signature = (data, value = 1, /))] +fn checksum<'a, 'py>( + ctx: &mut ThreadContext<'py>, + mod_state: &ChecksumState, + data: Borrowed<'a, 'py, PyBytes>, + value: u32, +) -> PyResult<'py, u32> { + // ... +} +``` + +The wrapper supplies `ctx`, `mod_state`, and any optional `module` parameter; +only the remaining parameters appear in `signature` and are parsed from +Python arguments. Methods also receive `self` and the defining class, `cls`. + +The macro checks names and emits a static `ParamSpec`. Values are extracted +with `FromPyObject`; defaults are Rust expressions. Vectorcall binding uses a +fixed stack array and allocation-free keyword matching, yielding +`Borrowed<'args, 'py, PyAny>` slots with no refcount churn. Constructor +tuple/dict parsing fills the same slots. + +The glue checks its own inputs and keeps every argument alive while it is +in use: + +- A `ParamSpec`/slot-count mismatch is a checked error. +- `kwnames` must be a tuple before its entries are read. +- Constructor keyword values are never borrowed from the `kwargs` dict: + converting one argument can run Python code, such as `__index__`, that + removes another value from the dict and frees it. Before converting any + argument, the parser takes its own references to every keyword value and + keeps them until the call and result conversion finish. It stores them in + a private tuple rather than a list, because Python code can find a list + through `gc.get_objects()` and clear it. +- Positional arguments and vectorcall arrays are borrowed without a copy: + the caller keeps them alive for the whole call, and Python code cannot + modify them. + +Argument errors raise `TypeError`. + +### Functions + +Python-callable, module-scoped functions are ordinary Rust functions whose +first two parameters are `ctx` (the thread context) and `mod_state` (module +state): + +```rust +/// Encode data using the standard Base64 alphabet. +#[pyfunction(signature = (data, /))] +fn standard_b64encode<'a, 'py>( + ctx: &mut ThreadContext<'py>, + _mod_state: &Base64State, + data: Borrowed<'a, 'py, PyBytes>, +) -> PyResult<'py, Bound<'py, PyBytes>> { + // ... +} +``` + +An optional `module` parameter provides +[module object access](#module-object-access). + +A body using only leaf operations may take `&ThreadContext`. `#[pyfunction]` +emits a `METH_FASTCALL | METH_KEYWORDS` trampoline that enters panic +containment and `with_current`, validates the module against its +`ModuleDef` and checks it has executed, binds arguments, calls the +function, and converts the result before `'py` ends. Methods use +`METH_METHOD`. + +Argument binding, defaults, and the call live in a generated safe function, +so author code never compiles under `unsafe`. The macros reject `unsafe`, +`async`, `const`, `extern`, and generic functions. + +`#[pyfunction]` and `#[pymethods]` use Rust doc comments (`///` or +`#[doc = ...]`) as the Python `__doc__`, stored in `ml_doc`. + +#### Panic containment + +The entry guard must outlive panic handling, so the catch sits inside +`with_current`: + +```rust +// SAFETY: callers must obey PyCFunctionFastWithKeywords, provide live +// callback arguments, and invoke this function with an attached thread state. +unsafe extern "C" fn trampoline( + /* existing PyCFunctionFastWithKeywords arguments */ +) -> *mut ffi::PyObject { + let boundary = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + // Author code runs in this safe closure, outside any unsafe block. + let callback = |ctx: &mut ThreadContext<'_>| { + let body = std::panic::catch_unwind( + std::panic::AssertUnwindSafe(|| { + let value = call_rust_function(ctx); + convert_callback_result(ctx, value) + }), + ); + let result = match body { + Ok(result) => result, + Err(payload) => Err(panic_to_owned_error(ctx, payload)), + }; + // The context and entry guard still protect cleanup and the + // final transfer of an owned result/error to the C caller. + finish_callback(ctx, result) + }; + // SAFETY: callback inputs satisfy entry's attached-state contract. + let entered = unsafe { with_current(callback) }; + match entered { + Ok(ptr) => ptr, + // SAFETY: this helper establishes private reporting authorization + // or fails fatally; it never uses a rejected or stale context. + Err(status) => unsafe { translate_entry_error_to_raw(status) }, + } + })); + match boundary { + Ok(ptr) => ptr, + // An entry/restoration/reporting invariant failed. Do not attempt to + // report through a context whose guard may already have been dropped. + Err(_) => std::process::abort(), + } +} +``` + +`panic_to_owned_error` drops the payload with the context suspended, then builds a fixed-message +`SystemError`. Entry, restoration, and lifecycle guards never unwind; the +catch is a last-resort abort. + +| Callback kind | Panic policy | +| --- | --- | +| Function, method, getter, setter, and constructor | Set a fixed `SystemError` and return the ABI's `NULL` or `-1` error sentinel. | +| Module create | Set a fixed `SystemError` and return `NULL`. | +| Module exec | Commit the appropriate failure state, set a fixed `SystemError`, and return `-1`. | +| Module or instance clear | Finish the lifecycle transition and set `SystemError`, which CPython reports as unraisable. Instance clear returns failure. Module clear returns 0, because CPython skips clearing the module dict when `m_clear` fails. | +| Traverse | Preserve incoming exception state and propagate the visitor's exact nonzero return and any error it sets. A Rust panic is fatal because continuing with an incomplete reachability report can violate GC invariants. | +| `tp_dealloc` and module free | Run mandatory non-panicking cleanup guards, then report the panic as unraisable under the [deallocator exception policy](#dropping). Abort if cleanup itself cannot be completed safely. | + +Before dropping a payload or module state, deallocation commits the +destroyed state and installs cleanup guards, so a panic cannot cause a +double drop or skip cleanup. Destructors run with the context suspended and +receive no context. During finalization, suspension does not broaden what +the execution phase permits. + +### Classes + +A payload type `C` identifies a class's layout, static definition, module +state, and GC policy. Each module instance creates its own runtime type: + +```rust +pub trait PyClass: Send + Sync + 'static + Sized { + type State: ModuleState; + type Gc: PyClassGc; + + fn definition() -> &'static ClassDef; +} + +// SAFETY: implementations must validate interpreter association, report every +// strong Python edge, and synchronize traversal, mutation, and partial clear. +pub unsafe trait PyClassGc: 'static { + const ENABLED: bool; + + fn validate<'py>(value: &C, ctx: &mut ThreadContext<'py>) -> PyResult<'py, ()>; + fn traverse(value: &C, visit: &Visit<'_>) -> TraverseResult; + fn clear(value: &C, ctx: &mut ThreadContext<'_>); +} + +#[derive(PyClassGc)] +struct Encoder { + level: u32, +} + +impl PyClass for Encoder { + type State = ModuleStateType; + type Gc = EncoderGc; + + fn definition() -> &'static ClassDef { + &ENCODER_CLASS + } +} + +static ENCODER_CLASS: ClassDef = + ClassDef::new(c"example.Encoder") + .methods(Encoder::PY_METHODS) + .getsets(Encoder::PY_GETSETS); +``` + +`#[derive(PyClassGc)]` uses the [module-state field rules](#deriving-gc-policies). +Every instance also owns an edge to its heap type, which generated traversal +reports even when `ENABLED` is false. + +#### Class identity + +Downcasts trust the address returned by `definition()` as the class token. +`ClassDef` uses the same private `UnsafeCell` field as `ModuleDef`. The +class marker trait is sealed. Generating the static through an `unsafe` or +sealed trait, as `export_module!` does, would also remove reliance on a safe +`definition()` method. + +`ClassDef`, `PyType`, `PyInstance`, generated method tables, and +`GcSlot` are invariant. This prevents coercing a `Payload` +type object to `Payload`, constructing an instance, and +downcasting it back. Handles may be covariant in their marker because each +parameterized marker is invariant. + +`PyType` and `PyInstance` are transparent markers. +Upcasts to `PyType` or `PyAny` are free; downcasts check the context, then +the class token. + +#### Types and instances + +`ClassDef::create(ctx, ModuleRef)` returns +`Bound>`. Each `C` has exactly one definition; a second needs a +Rust newtype. The definition address is also the `Py_tp_token`, which +proves layout, not interpreter identity: several module instances can have +types with the same token. There is no global type cache. + +`type_object.new_instance(ctx, payload)` validates the token and every +payload slot's interpreter ID before publishing. It allocates with +`_PyType_AllocNoTrack` (`PyType_GenericAlloc` tracks too early), writes the +payload and lifecycle metadata, then calls `PyObject_GC_Track`. On failure or +unwind, the unpublished payload is dropped with the context suspended. + +Methods recover the defining module through `METH_METHOD`, validate its +state and interpreter ID, and bound borrows by owner and entry (see +[Methods](#methods)). `&self` borrows the instance's Rust payload; +concurrent calls on free-threaded builds require interior mutability rather +than `&mut self`. + +```rust +#[derive(PyClassGc)] +struct Node { + parent: GcSlot>, + value: u64, +} +``` + +All generated instances set `Py_TPFLAGS_HAVE_GC` and traverse their type +edge; `Node` also traverses and clears its payload edge: + +| Operation | Invariant | +| --- | --- | +| Allocate | Validate payload interpreter IDs against the type/context ID; write `C` into an untracked object, publish ready, then call `PyObject_GC_Track`. | +| Traverse | Visit the actual heap type and payload edges through shared access; only call `visitproc`, propagate its exact return, and never invoke Python or block independently of the visitor. | +| Clear | Remove payload edges through synchronized `GcSlot` access using `&C`; release removed references after leaving slot protection; later clears are no-ops. The instance retains its type until deallocation. | +| Deallocate | Untrack first. Contain any required clear independently; then commit dropping, drop `C` once, run allocation/type guards, and report the first panic. | + +Types are immutable, final (preserving the `METH_METHOD` defining-class +link), and non-instantiable unless they define `tp_new`. + +#### Methods + +Instance methods take `&self`, `cls`, `ctx`, and `mod_state`, followed by +their Python arguments: + +```rust +#[pymethods] +impl Encoder { + #[pyfunction(signature = (data, /))] + fn encode<'a, 'py>( + &self, + cls: Borrowed<'a, 'py, PyType>, + ctx: &mut ThreadContext<'py>, + mod_state: &ModuleStateType, + data: Borrowed<'a, 'py, PyBytes>, + ) -> PyResult<'py, Bound<'py, PyBytes>> { + // ... + } +} +``` + +`cls` is the defining class supplied by `METH_METHOD`; `mod_state` is a +shared borrow of that class's module state (`&C::State`). The wrapper +supplies these parameters before parsing Python arguments. + +`#[pymethods]` accepts: + +- `#[pyfunction]`: an instance method with a `PyCMethod` trampoline; +- `#[getter]` and `#[setter]`: halves of a `ClassGetSetDef`, using the + same `&self`, `cls`, `ctx`, and `mod_state` prefix. Their glue obtains + `cls` from the instance's type, since generated types are final; +- `#[new]`: a `tp_new` trampoline taking `cls`, `ctx`, and `mod_state` + before Python arguments. It constructs `C` and moves it into a new + allocation. + +Setters reject `NULL` (attribute deletion) with `AttributeError` and `-1` +before conversion or author code; Python `None` is an ordinary value. A +future `#[deleter]` could share the C `PyGetSetDef.set` slot. + +`#[pymethods]` emits `C::PY_METHODS`, `C::PY_GETSETS`, and `C::PY_NEW`, +installed with `methods`, `getsets`, and `constructor`. All function-glue +rules apply. + +## Testing + +CI runs `cargo fmt` and [Clippy](https://github.com/rust-lang/rust-clippy), +checks context or sealed capabilities on Python-touching safe wrappers, and +audits shared wrappers as leaf operations. Both release and debug builds +exercise the [transition checks](#transition-checks). + +A new `_testrustapi` module in the regular test suite covers the API's +correctness. Entry tests cover already-attached and native callers, +restoration of the caller's prior attachment, cleanup authorization, and +balanced token/guard release on success, setup failure, and panic. + +## Appendix: Open decisions and future work + +- **Upstream CPython prerequisites.** These bugs reproduce without Rust: + 1. Module-state allocation publishes `md_state` before zeroing it. Concurrent + callbacks need initialized storage and synchronized pointer publication + (see [State lifecycle](#state-lifecycle)). + 2. A PEP 788 guard held by a thread that does not survive `fork()` stays + counted, hanging the child's finalization. Public guards can be handed + between threads, so a guard from a vanished thread may belong to the + survivor. Guards need an ownership or fork-invalidation contract. + 3. A daemon thread blocked in a subinterpreter ended by main-interpreter + finalization can keep a dangling interpreter pointer and read freed + memory on reattach. This design inherits the bug and does not fix it. + 4. A guard holder waiting on a pending call scheduled after the finalizer's + last `_Py_FinishPendingCalls` deadlocks. Scheduling must wake + finalizers, and the finalizer must run pending calls before parking. +- **Static builds.** Static builds register modules through `PyInit_*` + inittab entries, so `PyModExport_` needs inittab support. +- **Reclamation after shutdown.** Revocable ownership records and reserved + storage could release references that are currently abandoned, at a cost + to every handle. +- **Buffer protocol.** A safe `PyBuffer` must distinguish read-only and + writable exporters, specify contiguity, format, alignment, ownership, and + release, and prevent slices from racing Python mutation or escaping across + detach. +- **Locking.** A lock that cooperates with attachment, the GIL, and + stop-the-world is needed, and could be modeled after [PyO3's `PyOnceLock`](https://github.com/PyO3/pyo3/blob/4045f8d92045fbf55438eb6cae175b853c5835ce/src/sync/once_lock.rs#L42). +- **Guard performance.** If measurements justify avoiding the public + guards' allocation or shared-counter contention, a private optimization + must preserve whole-entry lifetime and admission guarantees. + Per-thread-state counters require atomic acquisition while attached, no + safepoint between the admission check and increment, and a synchronized + final recheck of every counter under stop-the-world or the GIL. Fork interaction and + finalizer wakeups must be proved again; an optional guard around explicit + detaches is not an equivalent optimization. +- **Shutdown progress.** Whole-entry guards allow blocked daemon threads to + delay finalization indefinitely. A future nonblocking design would need + a complete ownership proof for every allocation backing an exposed Rust + reference, including ancestor entries and deferred-counted instances. + Pinning only the current entry's module does not establish that proof. +- **Mutable GC containers.** Caches, callback registries, and `_asyncio` + task sets need mutable collections of traced references. Current payloads + expose only `&self`, and containers of `GcSlot` cannot change through + `&`. An untraced `Mutex` permits mutation but leaks cycles. Future + `GcVec` or `GcMap` containers must preserve these invariants: + - every reference is checked against the container's interpreter ID + when it is inserted, not at GC time; + - traversal reports each strong reference exactly once; + - traversal reads without locking, so no thread may be paused + mid-mutation while holding the container's lock; + - clear closes the container, so finalizers cannot re-add edges, and + releases references only after the lock is dropped. + + `Mutex>>` fails the third requirement: traversal can neither + lock a mutex held by a paused thread nor read a buffer that a concurrent + `push` may reallocate. +- **Borrowed `GcSlot` reads.** `get` increfs because another thread may + replace and decref the value. A `Borrowed` result would need a guard that + blocks replacement and permits only leaf operations, perhaps by borrowing + `&ThreadContext` throughout. Whether a reader may pause on a contended + lock while holding this guard remains open. +- **Explicit thread state C APIs** and inlining the entry counters could + speed hot paths without changing the Rust API. +- **Panic strategy.** The API requires `std` and `panic=unwind`. A + `panic=abort` build is ABI-safe but cannot recover; a `no_std` build would + need panic-abort or another no-unwind boundary. diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index f8828a56b4c7da..fb091fd713fb2e 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -3,6 +3,7 @@ from __future__ import annotations import _imp +import os import os.path import sys import sysconfig @@ -14,12 +15,17 @@ SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) STDLIB_PATH = os.path.join(SRC_DIR, 'Lib') +MODULES_PATH = os.path.join(SRC_DIR, 'Modules') IGNORE = { '__init__', '__pycache__', 'site-packages', + # Rust build-system support modules + 'cpython-build-helper', + 'cpython-rust-staticlib', + # Test modules and packages '__hello__', '__phello__', @@ -85,6 +91,19 @@ def list_modules_setup_extensions(names: set[str]) -> None: names.update(checker.list_module_names(all=True)) +def list_rust_modules(names: set[str]) -> None: + if not os.path.isdir(MODULES_PATH): + return + for entry in os.scandir(MODULES_PATH): + if not entry.is_dir(): + continue + if entry.name == "cpython-sys": + continue + cargo_toml = os.path.join(entry.path, "Cargo.toml") + if os.path.isfile(cargo_toml): + names.add(entry.name) + + # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c). # Use the "./Programs/_testembed list_frozen" command. def list_frozen(names: set[str]) -> None: @@ -110,6 +129,7 @@ def list_modules() -> set[str]: list_builtin_modules(names) list_modules_setup_extensions(names) + list_rust_modules(names) list_packages(names) list_python_modules(names) list_frozen(names) diff --git a/configure b/configure index 65d58fd1e810e2..8fb876eefa3115 100755 --- a/configure +++ b/configure @@ -691,6 +691,8 @@ MODULE_BINASCII_FALSE MODULE_BINASCII_TRUE MODULE_ZLIB_FALSE MODULE_ZLIB_TRUE +MODULE__BASE64_FALSE +MODULE__BASE64_TRUE MODULE__UUID_FALSE MODULE__UUID_TRUE MODULE__TKINTER_FALSE @@ -826,6 +828,7 @@ MODULE_TIME_FALSE MODULE_TIME_TRUE MODULE__IO_FALSE MODULE__IO_TRUE +RUST_STATICLIB_DEP MODULE_BUILDTYPE _PYTHREAD_NAME_MAXLEN BUILD_DETAILS @@ -895,6 +898,13 @@ TCLTK_LIBS TCLTK_CFLAGS LIBSQLITE3_LIBS LIBSQLITE3_CFLAGS +LLVM_TARGET +CARGO_DYLIB_SUFFIX +CARGO_TARGET_LINKER_ENV +CARGO_TARGET +CARGO_PROFILE +CARGO_TARGET_DIR +CARGO LIBMPDEC_LIBS LIBMPDEC_CFLAGS MODULE__CTYPES_MALLOC_CLOSURE @@ -910,7 +920,8 @@ SHLIBS CFLAGSFORSHARED LINKFORSHARED CCSHARED -BLDSHARED +BLDSHARED_ARGS +BLDSHARED_EXE LDCXXSHARED LDSHARED SHLIB_SUFFIX @@ -7555,6 +7566,13 @@ printf %s "checking for the platform triplet based on compiler characteristics.. if $CPP $CPPFLAGS $srcdir/Misc/platform_triplet.c >conftest.out 2>/dev/null; then PLATFORM_TRIPLET=`grep '^PLATFORM_TRIPLET=' conftest.out | tr -d ' '` PLATFORM_TRIPLET="${PLATFORM_TRIPLET#PLATFORM_TRIPLET=}" + RUST_TARGET=`grep '^RUST_TARGET=' conftest.out | tr -d ' '` + RUST_TARGET="${RUST_TARGET#RUST_TARGET=}" + LLVM_TARGET=`grep '^LLVM_TARGET=' conftest.out | tr -d ' '` + LLVM_TARGET="${LLVM_TARGET#LLVM_TARGET=}" + if test -z "$LLVM_TARGET"; then + LLVM_TARGET="$RUST_TARGET" + fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PLATFORM_TRIPLET" >&5 printf "%s\n" "$PLATFORM_TRIPLET" >&6; } else @@ -10203,6 +10221,8 @@ fi as_fn_append LINKFORSHARED " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520" + as_fn_append LDFLAGS_NODIST " -fwasm-exceptions" + as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES" as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,_PyGILState_GetThisThreadState,NODEFS" @@ -14605,6 +14625,7 @@ fi + # SHLIB_SUFFIX is the extension of shared libraries `(including the dot!) # -- usually .so, .sl on HP-UX, .dll on Cygwin { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking the extension of shared libraries" >&5 @@ -14634,7 +14655,8 @@ if test -z "$LDSHARED" then case $ac_sys_system/$ac_sys_release in AIX*) - BLDSHARED="Modules/ld_so_aix \$(CC) -bI:Modules/python.exp" + BLDSHARED_EXE="Modules/ld_so_aix" + BLDSHARED_ARGS="\$(CC) -bI:Modules/python.exp" LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" ;; SunOS/5*) @@ -14658,7 +14680,8 @@ then LDCXXSHARED='$(CXX) -bundle' if test "$enable_framework" ; then # Link against the framework. All externals should be defined. - BLDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDCXXSHARED="$LDCXXSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' else @@ -14669,14 +14692,15 @@ then Darwin/1.4*|Darwin/5.*|Darwin/6.*) LDSHARED='$(CC) -bundle' LDCXXSHARED='$(CXX) -bundle' + BLDSHARED_EXE="$LDSHARED" if test "$enable_framework" ; then # Link against the framework. All externals should be defined. - BLDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + BLDSHARED_ARGS='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDCXXSHARED="$LDCXXSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' else # No framework, use the Python app as bundle-loader - BLDSHARED="$LDSHARED "'-bundle_loader $(BUILDPYTHON)' + BLDSHARED_ARGS='-bundle_loader $(BUILDPYTHON)' LDSHARED="$LDSHARED "'-bundle_loader $(BINDIR)/python$(VERSION)$(EXE)' LDCXXSHARED="$LDCXXSHARED "'-bundle_loader $(BINDIR)/python$(VERSION)$(EXE)' fi ;; @@ -14697,13 +14721,15 @@ then # building for OS X 10.3 and later LDSHARED='$(CC) -bundle -undefined dynamic_lookup' LDCXXSHARED='$(CXX) -bundle -undefined dynamic_lookup' - BLDSHARED="$LDSHARED" + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS="" fi ;; iOS/*|tvOS/*|visionOS/*|watchOS/*) LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' - BLDSHARED="$LDSHARED" + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS="" ;; Emscripten*|WASI*) LDSHARED='$(CC) -shared' @@ -14752,18 +14778,25 @@ fi;; fi if test "$enable_wasm_dynamic_linking" = "yes" -a "$ac_sys_system" = "Emscripten"; then - BLDSHARED='$(CC) -shared -sSIDE_MODULE=1' + BLDSHARED_EXE='$(CC) -shared' + BLDSHARED_ARGS='-sSIDE_MODULE=1' fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDSHARED" >&5 printf "%s\n" "$LDSHARED" >&6; } LDCXXSHARED=${LDCXXSHARED-$LDSHARED} -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BLDSHARED flags" >&5 -printf %s "checking BLDSHARED flags... " >&6; } -BLDSHARED=${BLDSHARED-$LDSHARED} -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BLDSHARED" >&5 -printf "%s\n" "$BLDSHARED" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BLDSHARED_EXE executable" >&5 +printf %s "checking BLDSHARED_EXE executable... " >&6; } +BLDSHARED_EXE=${BLDSHARED_EXE-$LDSHARED} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BLDSHARED_EXE" >&5 +printf "%s\n" "$BLDSHARED_EXE" >&6; } + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking BLDSHARED_ARGS flags" >&5 +printf %s "checking BLDSHARED_ARGS flags... " >&6; } +BLDSHARED_ARGS=${BLDSHARED_ARGS-} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BLDSHARED_ARGS" >&5 +printf "%s\n" "$BLDSHARED_ARGS" >&6; } # CCSHARED are the C *flags* used to create objects to go into a shared # library (module) -- this is only needed for a few systems @@ -17406,6 +17439,101 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_decimal_contextvar" >&5 printf "%s\n" "$with_decimal_contextvar" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for Rust" >&5 +printf %s "checking for Rust... " >&6; } +CARGO_TARGET_DIR= +CARGO_PROFILE= +if test "x$CARGO_HOME" = "x"; then + CARGO_HOME="$HOME/.cargo" +fi +# Extract the first word of "cargo", so it can be a program name with args. +set dummy cargo; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_path_CARGO+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) case $CARGO in + [\\/]* | ?:[\\/]*) + ac_cv_path_CARGO="$CARGO" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR$CARGO_HOME/bin +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_path_CARGO="$as_dir$ac_word$ac_exec_ext" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_path_CARGO" && ac_cv_path_CARGO="no" + ;; +esac ;; +esac +fi +CARGO=$ac_cv_path_CARGO +if test -n "$CARGO"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CARGO" >&5 +printf "%s\n" "$CARGO" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + +if test "$CARGO" = "no"; then + HAVE_CARGO=no + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: n/a" >&5 +printf "%s\n" "n/a" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: Could not find the cargo executable. It can be installed via rustup" >&5 +printf "%s\n" "$as_me: WARNING: Could not find the cargo executable. It can be installed via rustup" >&2;} +else + HAVE_CARGO=yes + if test "$Py_OPT" = 'true'; then + CARGO_TARGET_DIR='release' + CARGO_PROFILE='release' + else + CARGO_TARGET_DIR='debug' + CARGO_PROFILE='dev' + fi + # RUST_TARGET is detected from Misc/platform_triplet.c using the + # C preprocessor, giving us the correct Rust target triple for the + # platform being compiled for. + CARGO_TARGET="$RUST_TARGET" + if test -z "$CARGO_TARGET" || test "$CARGO_TARGET" = "unknown"; then + as_fn_error $? "Could not determine a Rust target triple for this platform. + Rust may not support this target. Set CARGO_TARGET manually or + disable Rust support." "$LINENO" 5 + fi +fi +if test -n "$CARGO_TARGET"; then + CARGO_TARGET_LINKER_ENV="CARGO_TARGET_$(echo "$CARGO_TARGET" | tr 'a-z-' 'A-Z_')_LINKER" +else + CARGO_TARGET_LINKER_ENV="" +fi +case "$host" in + *-apple-*) CARGO_DYLIB_SUFFIX=".dylib";; + *) CARGO_DYLIB_SUFFIX="$SHLIB_SUFFIX";; +esac + + + + + + + @@ -35827,6 +35955,17 @@ then : fi fi +if test "$HAVE_CARGO" != "no" -a "$MODULE_BUILDTYPE" = "static"; then + RUST_STATICLIB_DEP=cpython-rust-staticlib + if test "$CARGO_TARGET" = "wasm32-wasip3" +then : + as_fn_append LDFLAGS_NODIST " -Wl,--export=cabi_realloc" +fi +else + RUST_STATICLIB_DEP= +fi + + MODULE_BLOCK= @@ -38037,6 +38176,47 @@ printf "%s\n" "$py_cv_module__uuid" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _base64" >&5 +printf %s "checking for stdlib extension module _base64... " >&6; } + if test "$py_cv_module__base64" != "n/a" +then : + + if test "$HAVE_CARGO" = yes +then : + if test "$HAVE_CARGO" = yes +then : + py_cv_module__base64=yes +else case e in #( + e) py_cv_module__base64=missing ;; +esac +fi +else case e in #( + e) py_cv_module__base64=disabled ;; +esac +fi + +fi + as_fn_append MODULE_BLOCK "MODULE__BASE64_STATE=$py_cv_module__base64$as_nl" + if test "x$py_cv_module__base64" = xyes +then : + + + + +fi + if test "$py_cv_module__base64" = yes; then + MODULE__BASE64_TRUE= + MODULE__BASE64_FALSE='#' +else + MODULE__BASE64_TRUE='#' + MODULE__BASE64_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__base64" >&5 +printf "%s\n" "$py_cv_module__base64" >&6; } + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module zlib" >&5 printf %s "checking for stdlib extension module zlib... " >&6; } if test "$py_cv_module_zlib" != "n/a" @@ -39340,6 +39520,10 @@ if test -z "${MODULE__UUID_TRUE}" && test -z "${MODULE__UUID_FALSE}"; then as_fn_error $? "conditional \"MODULE__UUID\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__BASE64_TRUE}" && test -z "${MODULE__BASE64_FALSE}"; then + as_fn_error $? "conditional \"MODULE__BASE64\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE_ZLIB_TRUE}" && test -z "${MODULE_ZLIB_FALSE}"; then as_fn_error $? "conditional \"MODULE_ZLIB\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index a041f0115bd7ab..4c587212826c39 100644 --- a/configure.ac +++ b/configure.ac @@ -1347,6 +1347,14 @@ AC_MSG_CHECKING([for the platform triplet based on compiler characteristics]) if $CPP $CPPFLAGS $srcdir/Misc/platform_triplet.c >conftest.out 2>/dev/null; then PLATFORM_TRIPLET=`grep '^PLATFORM_TRIPLET=' conftest.out | tr -d ' '` PLATFORM_TRIPLET="${PLATFORM_TRIPLET@%:@PLATFORM_TRIPLET=}" + RUST_TARGET=`grep '^RUST_TARGET=' conftest.out | tr -d ' '` + RUST_TARGET="${RUST_TARGET@%:@RUST_TARGET=}" + dnl LLVM_TARGET is only emitted when it differs from RUST_TARGET + LLVM_TARGET=`grep '^LLVM_TARGET=' conftest.out | tr -d ' '` + LLVM_TARGET="${LLVM_TARGET@%:@LLVM_TARGET=}" + if test -z "$LLVM_TARGET"; then + LLVM_TARGET="$RUST_TARGET" + fi AC_MSG_RESULT([$PLATFORM_TRIPLET]) else AC_MSG_RESULT([none]) @@ -2630,6 +2638,8 @@ AS_CASE([$ac_sys_system], dnl Start with 20 MB and allow to grow AS_VAR_APPEND([LINKFORSHARED], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -fwasm-exceptions"]) + dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES"]) @@ -3798,7 +3808,8 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], AC_SUBST([SHLIB_SUFFIX]) AC_SUBST([LDSHARED]) AC_SUBST([LDCXXSHARED]) -AC_SUBST([BLDSHARED]) +AC_SUBST([BLDSHARED_EXE]) +AC_SUBST([BLDSHARED_ARGS]) AC_SUBST([CCSHARED]) AC_SUBST([LINKFORSHARED]) @@ -3828,7 +3839,8 @@ if test -z "$LDSHARED" then case $ac_sys_system/$ac_sys_release in AIX*) - BLDSHARED="Modules/ld_so_aix \$(CC) -bI:Modules/python.exp" + BLDSHARED_EXE="Modules/ld_so_aix" + BLDSHARED_ARGS="\$(CC) -bI:Modules/python.exp" LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" ;; SunOS/5*) @@ -3852,7 +3864,8 @@ then LDCXXSHARED='$(CXX) -bundle' if test "$enable_framework" ; then # Link against the framework. All externals should be defined. - BLDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDCXXSHARED="$LDCXXSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' else @@ -3863,14 +3876,15 @@ then Darwin/1.4*|Darwin/5.*|Darwin/6.*) LDSHARED='$(CC) -bundle' LDCXXSHARED='$(CXX) -bundle' + BLDSHARED_EXE="$LDSHARED" if test "$enable_framework" ; then # Link against the framework. All externals should be defined. - BLDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' + BLDSHARED_ARGS='$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDSHARED="$LDSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' LDCXXSHARED="$LDCXXSHARED "'$(PYTHONFRAMEWORKPREFIX)/$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' else # No framework, use the Python app as bundle-loader - BLDSHARED="$LDSHARED "'-bundle_loader $(BUILDPYTHON)' + BLDSHARED_ARGS='-bundle_loader $(BUILDPYTHON)' LDSHARED="$LDSHARED "'-bundle_loader $(BINDIR)/python$(VERSION)$(EXE)' LDCXXSHARED="$LDCXXSHARED "'-bundle_loader $(BINDIR)/python$(VERSION)$(EXE)' fi ;; @@ -3891,13 +3905,15 @@ then # building for OS X 10.3 and later LDSHARED='$(CC) -bundle -undefined dynamic_lookup' LDCXXSHARED='$(CXX) -bundle -undefined dynamic_lookup' - BLDSHARED="$LDSHARED" + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS="" fi ;; iOS/*|tvOS/*|visionOS/*|watchOS/*) LDSHARED='$(CC) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' LDCXXSHARED='$(CXX) -dynamiclib -F . -framework $(PYTHONFRAMEWORK)' - BLDSHARED="$LDSHARED" + BLDSHARED_EXE="$LDSHARED" + BLDSHARED_ARGS="" ;; Emscripten*|WASI*) LDSHARED='$(CC) -shared' @@ -3943,18 +3959,24 @@ then esac fi -dnl Emscripten's emconfigure sets LDSHARED. Set BLDSHARED outside the -dnl test -z $LDSHARED block to configure BLDSHARED for side module support. +dnl Emscripten's emconfigure sets LDSHARED. Set BLDSHARED_EXE and BLDSHARED_ARGS +dnl outside the test -z $LDSHARED block to configure BLDSHARED for side module +dnl support. if test "$enable_wasm_dynamic_linking" = "yes" -a "$ac_sys_system" = "Emscripten"; then - BLDSHARED='$(CC) -shared -sSIDE_MODULE=1' + BLDSHARED_EXE='$(CC) -shared' + BLDSHARED_ARGS='-sSIDE_MODULE=1' fi AC_MSG_RESULT([$LDSHARED]) LDCXXSHARED=${LDCXXSHARED-$LDSHARED} -AC_MSG_CHECKING([BLDSHARED flags]) -BLDSHARED=${BLDSHARED-$LDSHARED} -AC_MSG_RESULT([$BLDSHARED]) +AC_MSG_CHECKING([BLDSHARED_EXE executable]) +BLDSHARED_EXE=${BLDSHARED_EXE-$LDSHARED} +AC_MSG_RESULT([$BLDSHARED_EXE]) + +AC_MSG_CHECKING([BLDSHARED_ARGS flags]) +BLDSHARED_ARGS=${BLDSHARED_ARGS-} +AC_MSG_RESULT([$BLDSHARED_ARGS]) # CCSHARED are the C *flags* used to create objects to go into a shared # library (module) -- this is only needed for a few systems @@ -4727,6 +4749,60 @@ fi AC_MSG_RESULT([$with_decimal_contextvar]) +dnl Try to detect cargo in the environment. Cargo and rustup +dnl install into CARGO_HOME and RUSTUP_HOME, so check for those initially +AC_MSG_CHECKING([for Rust]) +CARGO_TARGET_DIR= +CARGO_PROFILE= +if test "x$CARGO_HOME" = "x"; then + dnl try to guess the default UNIX value of ~/.cargo + CARGO_HOME="$HOME/.cargo" +fi +AC_PATH_PROG([CARGO], [cargo], [no], [$PATH$PATH_SEPARATOR$CARGO_HOME/bin]) +if test "$CARGO" = "no"; then + HAVE_CARGO=no + AC_MSG_RESULT([n/a]) + AC_MSG_WARN([Could not find the cargo executable. It can be installed via rustup]) +else + HAVE_CARGO=yes + if test "$Py_OPT" = 'true'; then + CARGO_TARGET_DIR='release' + CARGO_PROFILE='release' + else + CARGO_TARGET_DIR='debug' + CARGO_PROFILE='dev' + fi + # RUST_TARGET is detected from Misc/platform_triplet.c using the + # C preprocessor, giving us the correct Rust target triple for the + # platform being compiled for. + CARGO_TARGET="$RUST_TARGET" + if test -z "$CARGO_TARGET" || test "$CARGO_TARGET" = "unknown"; then + AC_MSG_ERROR([Could not determine a Rust target triple for this platform. + Rust may not support this target. Set CARGO_TARGET manually or + disable Rust support.]) + fi +fi +dnl Compute the CARGO_TARGET__LINKER env var name so we can +dnl tell Cargo which linker to use for the target platform. +if test -n "$CARGO_TARGET"; then + CARGO_TARGET_LINKER_ENV="CARGO_TARGET_$(echo "$CARGO_TARGET" | tr 'a-z-' 'A-Z_')_LINKER" +else + CARGO_TARGET_LINKER_ENV="" +fi +dnl Cargo's cdylib crate type uses the platform's native shared library +dnl extension, which differs from CPython's SHLIB_SUFFIX on macOS/iOS +dnl (CPython uses .so, Cargo uses .dylib). +case "$host" in + *-apple-*) CARGO_DYLIB_SUFFIX=".dylib";; + *) CARGO_DYLIB_SUFFIX="$SHLIB_SUFFIX";; +esac +AC_SUBST([CARGO_TARGET_DIR]) +AC_SUBST([CARGO_PROFILE]) +AC_SUBST([CARGO_TARGET]) +AC_SUBST([CARGO_TARGET_LINKER_ENV]) +AC_SUBST([CARGO_DYLIB_SUFFIX]) +AC_SUBST([LLVM_TARGET]) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -8557,6 +8633,20 @@ AS_IF([test "$ac_sys_system" = "iOS" && test -z "$PYTHONFRAMEWORK"], [AS_IF([test "$MODULE_BUILDTYPE" != "static"], [AC_MSG_ERROR([iOS non-framework builds must use MODULE_BUILDTYPE=static])])]) +dnl Set RUST_STATICLIB_DEP to cpython-rust-staticlib when building Rust +dnl modules statically so that the python binary depends on it. +if test "$HAVE_CARGO" != "no" -a "$MODULE_BUILDTYPE" = "static"; then + RUST_STATICLIB_DEP=cpython-rust-staticlib + dnl Rust's std for wasm32-wasip3 defines its own weak cabi_realloc, which + dnl may take precedence over wasi-libc's exported one. The component + dnl encoder requires it to be exported, as rustc does when it links. + AS_IF([test "$CARGO_TARGET" = "wasm32-wasip3"], + [AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--export=cabi_realloc"])]) +else + RUST_STATICLIB_DEP= +fi +AC_SUBST([RUST_STATICLIB_DEP]) + dnl _MODULE_BLOCK_ADD([VAR], [VALUE]) dnl internal: adds $1=quote($2) to MODULE_BLOCK AC_DEFUN([_MODULE_BLOCK_ADD], [AS_VAR_APPEND([MODULE_BLOCK], ["$1=_AS_QUOTE([$2])$as_nl"])]) @@ -8889,6 +8979,10 @@ PY_STDLIB_MOD([_uuid], [], [test "$have_uuid" = "yes"], [$LIBUUID_CFLAGS], [$LIBUUID_LIBS]) +PY_STDLIB_MOD([_base64], + [test "$HAVE_CARGO" = yes], [test "$HAVE_CARGO" = yes], + [], []) + dnl compression libs PY_STDLIB_MOD([zlib], [], [test "$have_zlib" = yes], [$ZLIB_CFLAGS], [$ZLIB_LIBS])