diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4d31afe411d..7919c91f0c6 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,7 +5,7 @@ contact_links: about: Ask and answer questions about PyO3 on Discussions - name: 🔧 Troubleshooting url: https://github.com/PyO3/pyo3/discussions - about: For troubleshooting help, see the Discussions + about: For troubleshooting help, see the Discussions - name: 👋 Chat - url: https://gitter.im/PyO3/Lobby - about: Engage with PyO3's users and developers on Gitter \ No newline at end of file + url: https://discord.gg/33kcChzH7f + about: Engage with PyO3's users and developers on Discord diff --git a/.github/actions/fetch-merge-base/action.yml b/.github/actions/fetch-merge-base/action.yml new file mode 100644 index 00000000000..7ddbce5b65e --- /dev/null +++ b/.github/actions/fetch-merge-base/action.yml @@ -0,0 +1,31 @@ +name: 'Fetch Merge Base' +description: 'Fetches the merge base between two branches, deepening the git checkout until complete' +inputs: + base_ref: + description: 'The base branch reference' + required: true + head_ref: + description: 'The head branch reference' + required: true +outputs: + merge_base: + description: 'The merge base commit SHA' + value: ${{ steps.fetch_merge_base.outputs.merge_base }} +runs: + using: "composite" + steps: + - name: Fetch Merge Base + id: fetch_merge_base + shell: bash + run: | + # fetch the merge commit between the PR base and head + git fetch -u --progress --depth=1 origin "+$BASE_REF:$BASE_REF" "+$HEAD_REF:$HEAD_REF" + while [ -z "$(git merge-base "$BASE_REF" "$HEAD_REF")" ]; do + git fetch -u -q --deepen="10" origin "$BASE_REF" "$HEAD_REF"; + done + + MERGE_BASE=$(git merge-base "$BASE_REF" "$HEAD_REF") + echo "merge_base=$MERGE_BASE" >> $GITHUB_OUTPUT + env: + BASE_REF: "${{ inputs.base_ref }}" + HEAD_REF: "${{ inputs.head_ref }}" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 33f29d79418..f8a257e28f3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,8 +5,18 @@ version: 2 updates: - - package-ecosystem: "cargo" # See documentation for possible values - directory: "/" # Location of package manifests + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "cargo" + directory: "/pyo3-benches/" + schedule: + interval: "weekly" + + - package-ecosystem: "cargo" + directory: "/pyo3-ffi-check/" schedule: interval: "weekly" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b344525cabe..2156c1d7322 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,9 +5,8 @@ By submitting these contributions you agree for them to be dual-licensed under P Please consider adding the following to your pull request: - an entry for this PR in newsfragments - see [https://pyo3.rs/main/contributing.html#documenting-changes] - or start the PR title with `docs:` if this is a docs-only change to skip the check + - or start the PR title with `ci:` if this is a ci-only change to skip the check - docs to all new functions and / or detail in the guide - tests for all new or changed functions -PyO3's CI pipeline will check your pull request. To run its tests -locally, you can run ```nox```. See ```nox --list-sessions``` -for a list of supported actions. +PyO3's CI pipeline will check your pull request, thus make sure you have checked the `Contributing.md` guidelines. To run most of its tests locally, you can run `nox`. See `nox --list-sessions` for a list of supported actions. diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 01ab5f802dd..19dd9d145c4 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -9,34 +9,41 @@ on: # performance analysis in order to generate initial data. workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}-benches + cancel-in-progress: true + jobs: benchmarks: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" - uses: dtolnay/rust-toolchain@stable with: components: rust-src - - uses: actions/cache@v4 + - uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - pyo3-benches/target - target - key: cargo-${{ runner.os }}-bench-${{ hashFiles('**/Cargo.toml') }} - continue-on-error: true + workspaces: | + . + pyo3-benches + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} + cache-all-crates: "true" + cache-workspace-crates: "true" - - name: Install cargo-codspeed - run: cargo install cargo-codspeed + - uses: taiki-e/install-action@v2 + with: + tool: cargo-codspeed - name: Install nox - run: pip install nox + run: pip install nox[uv] - name: Run the benchmarks - uses: CodSpeedHQ/action@v2 + uses: CodSpeedHQ/action@v4 with: run: nox -s codspeed token: ${{ secrets.CODSPEED_TOKEN }} + mode: instrumentation diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af05eb20376..f8e6229cc00 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,116 +16,104 @@ on: rust-target: required: true type: string - extra-features: + MSRV: required: true type: string + verbose: + type: boolean + default: false + +env: + NOX_DEFAULT_VENV_BACKEND: uv jobs: build: - continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7"]'), inputs.python-version) || inputs.rust == 'beta' || inputs.rust == 'nightly' }} + continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "3.8"]'), inputs.python-version) || contains(fromJSON('["beta", "nightly"]'), inputs.rust) }} runs-on: ${{ inputs.os }} + if: ${{ !(startsWith(inputs.python-version, 'graalpy') && startsWith(inputs.os, 'windows')) }} steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + with: + # For PRs, we need to run on the real PR head, not the resultant merge of the PR into the target branch. + # + # This is necessary for coverage reporting to make sense; we then get exactly the coverage change + # between the base branch and the real PR head. + # + # If it were run on the merge commit the problem is that the coverage potentially does not align + # with the commit diff, because the merge may affect line numbers. + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + + # installs using setup-python do not work for arm macOS 3.9 and below + - if: ${{ !(inputs.os == 'macos-latest' && contains(fromJSON('["3.7", "3.8", "3.9"]'), inputs.python-version) && inputs.python-architecture == 'x64') }} + name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v6 with: python-version: ${{ inputs.python-version }} architecture: ${{ inputs.python-architecture }} - check-latest: ${{ startsWith(inputs.python-version, 'pypy') }} # PyPy can have FFI changes within Python versions, which creates pain in CI + # PyPy can have FFI changes within Python versions, which creates pain in CI + check-latest: ${{ startsWith(inputs.python-version, 'pypy') }} + + # workaround for the above, only available for 3.9 + - if: ${{ inputs.os == 'macos-latest' && contains(fromJSON('["3.9"]'), inputs.python-version) && inputs.python-architecture == 'x64' }} + name: Set up Python ${{ inputs.python-version }} + uses: astral-sh/setup-uv@v7 + with: + python-version: cpython-${{ inputs.python-version }}-macos-x86-64 - name: Install nox - run: python -m pip install --upgrade pip && pip install nox + run: python -m pip install --upgrade pip && pip install nox[uv] - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master with: toolchain: ${{ inputs.rust }} targets: ${{ inputs.rust-target }} - # needed to correctly format errors, see #1865 - components: rust-src + # rust-src needed to correctly format errors, see #1865 + components: rust-src,llvm-tools-preview + + # On windows 32 bit, we are running on an x64 host, so we need to specifically set the target + # NB we don't do this for *all* jobs because it breaks coverage of proc macros to have an + # explicit target set. + - name: Set Rust target for Windows 32-bit + if: inputs.os == 'windows-latest' && inputs.python-architecture == 'x86' + shell: bash + run: | + echo "CARGO_BUILD_TARGET=i686-pc-windows-msvc" >> $GITHUB_ENV + + # windows on arm image contains x86-64 libclang + - name: Install LLVM and Clang + if: inputs.os == 'windows-11-arm' + uses: KyleMayes/install-llvm-action@v2 + with: + # to match windows-2022 images + version: "18" + + - name: Install zoneinfo backport for Python 3.7 / 3.8 + if: contains(fromJSON('["3.7", "3.8"]'), inputs.python-version) + run: python -m pip install backports.zoneinfo - uses: Swatinem/rust-cache@v2 with: - key: cargo-${{ inputs.python-architecture }} - continue-on-error: true + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - if: inputs.os == 'ubuntu-latest' name: Prepare LD_LIBRARY_PATH (Ubuntu only) run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV - - if: inputs.rust == '1.56.0' - name: Prepare minimal package versions (MSRV only) - run: nox -s set-minimal-package-versions + - if: inputs.rust == inputs.MSRV + name: Prepare MSRV package versions + run: nox -s set-msrv-package-versions - if: inputs.rust != 'stable' name: Ignore changed error messages when using trybuild run: echo "TRYBUILD=overwrite" >> "$GITHUB_ENV" - - name: Build docs - run: nox -s docs - - - name: Build (no features) - run: cargo build --lib --tests --no-default-features - - # --no-default-features when used with `cargo build/test -p` doesn't seem to work! - - name: Build pyo3-build-config (no features) - run: | - cd pyo3-build-config - cargo build --no-default-features - - # Run tests (except on PyPy, because no embedding API). - - if: ${{ !startsWith(inputs.python-version, 'pypy') }} - name: Test (no features) - run: cargo test --no-default-features --lib --tests - - # --no-default-features when used with `cargo build/test -p` doesn't seem to work! - - name: Test pyo3-build-config (no features) - run: | - cd pyo3-build-config - cargo test --no-default-features - - - name: Build (all additive features) - run: cargo build --lib --tests --no-default-features --features "full ${{ inputs.extra-features }}" - - - if: ${{ startsWith(inputs.python-version, 'pypy') }} - name: Build PyPy (abi3-py37) - run: cargo build --lib --tests --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}" - - # Run tests (except on PyPy, because no embedding API). - - if: ${{ !startsWith(inputs.python-version, 'pypy') }} - name: Test - run: cargo test --no-default-features --features "full ${{ inputs.extra-features }}" - - # Run tests again, but in abi3 mode - - if: ${{ !startsWith(inputs.python-version, 'pypy') }} - name: Test (abi3) - run: cargo test --no-default-features --features "abi3 full ${{ inputs.extra-features }}" - - # Run tests again, for abi3-py37 (the minimal Python version) - - if: ${{ (!startsWith(inputs.python-version, 'pypy')) && (inputs.python-version != '3.7') }} - name: Test (abi3-py37) - run: cargo test --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}" - - - name: Test proc-macro code - run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml - - - name: Test build config - run: cargo test --manifest-path=pyo3-build-config/Cargo.toml - - - name: Test python examples and tests - shell: bash - run: nox -s test-py - env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target - - uses: dorny/paths-filter@v3 - # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here - if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' }} + if: ${{ inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy') }} id: ffi-changes with: - base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} - ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }} + base: ${{ github.event.merge_group.base_ref }} + ref: ${{ github.event.merge_group.head_ref }} filters: | changed: - 'pyo3-ffi/**' @@ -134,68 +122,52 @@ jobs: - '.github/workflows/build.yml' - name: Run pyo3-ffi-check - # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor - # is pypy 3.9 on windows - if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} + # TODO: investigate graalpy failures + if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy')) }} run: nox -s ffi-check + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov - - name: Test cross compilation - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.9' }} - uses: PyO3/maturin-action@v1 - env: - PYO3_CROSS_LIB_DIR: /opt/python/cp39-cp39/lib - with: - target: aarch64-unknown-linux-gnu - manylinux: auto - args: --release -i python3.9 -m examples/maturin-starter/Cargo.toml - - - run: sudo rm -rf examples/maturin-starter/target - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.9' }} - - name: Test cross compile to same architecture - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.9' }} - uses: PyO3/maturin-action@v1 - env: - PYO3_CROSS_LIB_DIR: /opt/python/cp39-cp39/lib - with: - target: x86_64-unknown-linux-gnu - manylinux: auto - args: --release -i python3.9 -m examples/maturin-starter/Cargo.toml + - name: Prepare coverage environment + run: | + cargo llvm-cov clean --workspace --profraw-only + nox -s set-coverage-env - - name: Test cross compilation - if: ${{ inputs.os == 'macos-latest' && inputs.python-version == '3.9' }} - uses: PyO3/maturin-action@v1 - with: - target: aarch64-apple-darwin - args: --release -i python3.9 -m examples/maturin-starter/Cargo.toml + - name: Build docs + run: nox -s docs + + - name: Run Rust tests + run: nox -s test-rust - - name: Test cross compile to Windows - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.8' }} + - name: Test python examples and tests + shell: bash + run: nox -s test-py env: - XWIN_ARCH: x86_64 - run: | - set -ex - sudo apt-get install -y mingw-w64 llvm - rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc - pip install cargo-xwin - # abi3 - cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu - cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc - # non-abi3 - export PYO3_CROSS_PYTHON_VERSION=3.9 - cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu - cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc - - - name: Test cross compile to Windows with maturin - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.8' }} - uses: PyO3/maturin-action@v1 + CARGO_TARGET_DIR: ${{ github.workspace }}/target + + - name: Generate coverage report + # needs investigation why llvm-cov fails on windows-11-arm + continue-on-error: ${{ inputs.os == 'windows-11-arm' }} + run: cargo llvm-cov + --package=pyo3 + --package=pyo3-build-config + --package=pyo3-macros-backend + --package=pyo3-macros + --package=pyo3-ffi + report --codecov --output-path coverage.json + + - name: Upload coverage report + uses: codecov/codecov-action@v5 + # needs investigation why llvm-cov fails on windows-11-arm + continue-on-error: ${{ inputs.os == 'windows-11-arm' }} with: - target: x86_64-pc-windows-gnu - args: -i python3.8 -m examples/maturin-starter/Cargo.toml --features abi3 + files: coverage.json + name: ${{ inputs.os }}/${{ inputs.python-version }}/${{ inputs.rust }} + token: ${{ secrets.CODECOV_TOKEN }} env: - CARGO_TERM_VERBOSE: true - CARGO_BUILD_TARGET: ${{ inputs.rust-target }} + CARGO_TERM_VERBOSE: ${{ inputs.verbose }} RUST_BACKTRACE: 1 RUSTFLAGS: "-D warnings" RUSTDOCFLAGS: "-D warnings" diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml new file mode 100644 index 00000000000..02e8a6ab3cb --- /dev/null +++ b/.github/workflows/cache-cleanup.yml @@ -0,0 +1,31 @@ +name: CI Cache Cleanup +on: + pull_request_target: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Cleanup + run: | + gh extension install actions/gh-actions-cache + + echo "Fetching list of cache key" + cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete -R $REPO -B $BRANCH --confirm -- $cacheKey + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + BRANCH: refs/pull/${{ github.event.pull_request.number }}/merge diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 0a782b4010f..760918ce40a 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -9,7 +9,9 @@ jobs: name: Check changelog entry runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - run: python -m pip install --upgrade pip && pip install nox + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - run: python -m pip install --upgrade pip && pip install nox[uv] - run: nox -s check-changelog diff --git a/.github/workflows/ci-cache-warmup.yml b/.github/workflows/ci-cache-warmup.yml new file mode 100644 index 00000000000..2b05a1e98c0 --- /dev/null +++ b/.github/workflows/ci-cache-warmup.yml @@ -0,0 +1,34 @@ +name: CI Cache Warmup + +on: + push: + branches: + - main + +jobs: + cross-compilation-windows: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-pc-windows-gnu,x86_64-pc-windows-msvc + components: rust-src + - uses: actions/cache/restore@v4 + with: + # https://github.com/PyO3/maturin/discussions/1953 + path: ~/.cache/cargo-xwin + key: cargo-xwin-cache + - name: Test cross compile to Windows + run: | + set -ex + sudo apt-get install -y mingw-w64 llvm + pip install nox + nox -s test-cross-compilation-windows + - uses: actions/cache/save@v4 + with: + path: ~/.cache/cargo-xwin + key: cargo-xwin-cache diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22281185caa..2270eb1432f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,6 @@ name: CI on: - push: - branches: - - main pull_request: merge_group: types: [checks_requested] @@ -18,12 +15,13 @@ env: jobs: fmt: - if: github.ref != 'refs/heads/main' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - run: python -m pip install --upgrade pip && pip install nox + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - run: python -m pip install --upgrade pip && pip install nox[uv] - uses: dtolnay/rust-toolchain@stable with: components: rustfmt @@ -31,122 +29,109 @@ jobs: run: nox -s ruff - name: Check rust formatting (rustfmt) run: nox -s rustfmt + - name: Check markdown formatting (rumdl) + run: nox -s rumdl + + resolve: + runs-on: ubuntu-latest + outputs: + MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} + verbose: ${{ runner.debug == '1' }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: resolve MSRV + id: resolve-msrv + run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["workspace"]["package"]["rust-version"])'` >> $GITHUB_OUTPUT semver-checks: - if: github.ref != 'refs/heads/main' + if: github.event_name == 'pull_request' needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Fetch merge base + id: fetch_merge_base + uses: ./.github/actions/fetch-merge-base + with: + base_ref: "refs/heads/${{ github.event.pull_request.base.ref }}" + head_ref: "refs/pull/${{ github.event.pull_request.number }}/merge" - uses: obi1kenobi/cargo-semver-checks-action@v2 + with: + baseline-rev: ${{ steps.fetch_merge_base.outputs.merge_base }} check-msrv: - needs: [fmt] + needs: [fmt, resolve] runs-on: ubuntu-latest - if: github.ref != 'refs/heads/main' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@master with: - toolchain: 1.56.0 - targets: x86_64-unknown-linux-gnu + toolchain: ${{ needs.resolve.outputs.MSRV }} components: rust-src - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: - architecture: "x64" + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - key: check-msrv-1.56.0 - continue-on-error: true - - run: python -m pip install --upgrade pip && pip install nox - - name: Prepare minimal package versions - run: nox -s set-minimal-package-versions - - run: nox -s check-all + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} + - run: python -m pip install --upgrade pip && pip install nox[uv] + # This is a smoke test to confirm that CI will run on MSRV (including dev dependencies) + - name: Check with MSRV package versions + run: | + nox -s set-msrv-package-versions + nox -s check-all env: CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu clippy: needs: [fmt] - runs-on: ${{ matrix.platform.os }} - if: github.ref != 'refs/heads/main' + runs-on: ubuntu-24.04-arm strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} matrix: rust: [stable] - platform: [ - { - os: "macos-latest", - python-architecture: "x64", - rust-target: "x86_64-apple-darwin", - }, - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "x86_64-unknown-linux-gnu", - }, - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "powerpc64le-unknown-linux-gnu", - }, - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "s390x-unknown-linux-gnu", - }, - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "wasm32-wasi", - }, - { - os: "windows-latest", - python-architecture: "x64", - rust-target: "x86_64-pc-windows-msvc", - }, - { - os: "windows-latest", - python-architecture: "x86", - rust-target: "i686-pc-windows-msvc", - }, - ] + target: + [ + "aarch64-apple-darwin", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "wasm32-wasip1", + "x86_64-pc-windows-msvc", + "i686-pc-windows-msvc", + "aarch64-pc-windows-msvc", + ] include: # Run beta clippy as a way to detect any incoming lints which may affect downstream users - rust: beta - platform: - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "x86_64-unknown-linux-gnu", - } - name: clippy/${{ matrix.platform.rust-target }}/${{ matrix.rust }} - continue-on-error: ${{ matrix.platform.rust != 'stable' }} + target: "x86_64-unknown-linux-gnu" + name: clippy/${{ matrix.target }}/${{ matrix.rust }} + continue-on-error: ${{ matrix.rust != 'stable' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.rust }} - targets: ${{ matrix.platform.rust-target }} + targets: ${{ matrix.target }} components: clippy,rust-src - - uses: actions/setup-python@v5 - with: - architecture: ${{ matrix.platform.python-architecture }} - - uses: Swatinem/rust-cache@v2 - with: - key: clippy-${{ matrix.platform.rust-target }}-${{ matrix.platform.os }}-${{ matrix.rust }} - continue-on-error: true - - run: python -m pip install --upgrade pip && pip install nox - - run: nox -s clippy-all + - uses: astral-sh/setup-uv@v7 + - run: uvx nox -s clippy-all env: - CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }} + CARGO_BUILD_TARGET: ${{ matrix.target }} build-pr: if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} - needs: [fmt] + needs: [fmt, resolve] uses: ./.github/workflows/build.yml with: os: ${{ matrix.platform.os }} @@ -154,27 +139,32 @@ jobs: python-architecture: ${{ matrix.platform.python-architecture }} rust: ${{ matrix.rust }} rust-target: ${{ matrix.platform.rust-target }} - extra-features: ${{ matrix.platform.extra-features }} + MSRV: ${{ needs.resolve.outputs.MSRV }} + verbose: ${{ needs.resolve.outputs.verbose == 'true' }} secrets: inherit strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} matrix: - extra-features: ["multiple-pymethods"] rust: [stable] - python-version: ["3.12"] + python-version: ["3.14"] platform: [ { os: "macos-latest", - python-architecture: "x64", - rust-target: "x86_64-apple-darwin", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", }, { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", }, + { + os: "ubuntu-24.04-arm", + python-architecture: "arm64", + rust-target: "aarch64-unknown-linux-gnu", + }, { os: "windows-latest", python-architecture: "x64", @@ -185,23 +175,38 @@ jobs: python-architecture: "x86", rust-target: "i686-pc-windows-msvc", }, + { + os: "windows-11-arm", + python-architecture: "arm64", + rust-target: "aarch64-pc-windows-msvc", + }, ] include: # Test nightly Rust on PRs so that PR authors have a chance to fix nightly # failures, as nightly does not block merge. - rust: nightly - python-version: "3.12" + python-version: "3.14" platform: { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } - extra-features: "nightly multiple-pymethods" + # Also test free-threaded Python just for latest Python version, on ubuntu + # (run for all OSes on build-full) + - rust: stable + python-version: "3.14t" + platform: + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + } + build-full: - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} - needs: [fmt] + needs: [fmt, resolve] uses: ./.github/workflows/build.yml with: os: ${{ matrix.platform.os }} @@ -209,32 +214,35 @@ jobs: python-architecture: ${{ matrix.platform.python-architecture }} rust: ${{ matrix.rust }} rust-target: ${{ matrix.platform.rust-target }} - extra-features: ${{ matrix.platform.extra-features }} + MSRV: ${{ needs.resolve.outputs.MSRV }} + verbose: ${{ needs.resolve.outputs.verbose == 'true' }} secrets: inherit strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} matrix: - extra-features: ["multiple-pymethods"] # Because MSRV doesn't support this rust: [stable] - python-version: [ - "3.7", - "3.8", - "3.9", - "3.10", - "3.11", - "3.12", - "pypy3.7", - "pypy3.8", - "pypy3.9", - "pypy3.10", - ] + python-version: + [ + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "3.12", + "3.13", + "3.13t", + "3.14", + "3.14t", + "pypy3.11", + "graalpy25.0", + ] platform: [ { os: "macos-latest", - python-architecture: "x64", - rust-target: "x86_64-apple-darwin", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", }, { os: "ubuntu-latest", @@ -249,63 +257,149 @@ jobs: ] include: # Test minimal supported Rust version - - rust: 1.56.0 - python-version: "3.12" + - rust: ${{ needs.resolve.outputs.MSRV }} + python-version: "3.14" platform: { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } - extra-features: "" # Test the `nightly` feature - rust: nightly - python-version: "3.12" + python-version: "3.14" platform: { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } - extra-features: "nightly multiple-pymethods" # Run rust beta to help catch toolchain regressions - rust: beta - python-version: "3.12" + python-version: "3.14" platform: { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } - extra-features: "multiple-pymethods" - # Test 32-bit Windows only with the latest Python version + # Test 32-bit Windows and x64 macOS only with the latest Python version - rust: stable - python-version: "3.12" + python-version: "3.14" platform: { os: "windows-latest", python-architecture: "x86", rust-target: "i686-pc-windows-msvc", } - extra-features: "multiple-pymethods" + - rust: stable + python-version: "3.14" + platform: + { + os: "macos-latest", + python-architecture: "x64", + rust-target: "x86_64-apple-darwin", + } + # ubuntu-latest (24.04) no longer supports 3.7, so run on 22.04 + - rust: stable + python-version: "3.7" + platform: + { + os: "ubuntu-22.04", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + } + + # arm64 macOS Python not available on GitHub Actions until 3.10, test older versions on x64 + - rust: stable + python-version: "3.7" + platform: + { + os: "macos-15-intel", + python-architecture: "x64", + rust-target: "x86_64-apple-darwin", + } + - rust: stable + python-version: "3.8" + platform: + { + os: "macos-15-intel", + python-architecture: "x64", + rust-target: "x86_64-apple-darwin", + } + - rust: stable + python-version: "3.9" + platform: + { + os: "macos-15-intel", + python-architecture: "x64", + rust-target: "x86_64-apple-darwin", + } + # test latest Python on arm64 linux & windows runners + - rust: stable + python-version: "3.14" + platform: + { + os: "ubuntu-24.04-arm", + python-architecture: "arm64", + rust-target: "aarch64-unknown-linux-gnu", + } + - rust: stable + python-version: "3.14" + platform: + { + os: "windows-11-arm", + python-architecture: "arm64", + rust-target: "aarch64-pc-windows-msvc", + } + exclude: + # ubuntu-latest (24.04) no longer supports 3.7 + - python-version: "3.7" + platform: { os: "ubuntu-latest" } + # arm64 macOS Python not available on GitHub Actions until 3.10 + - rust: stable + python-version: "3.7" + platform: + { + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + } + - rust: stable + python-version: "3.8" + platform: + { + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + } + - rust: stable + python-version: "3.9" + platform: + { + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + } valgrind: - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - key: cargo-valgrind - continue-on-error: true + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@valgrind - - run: python -m pip install --upgrade pip && pip install nox + - run: python -m pip install --upgrade pip && pip install nox[uv] - run: nox -s test-rust -- release skip-full env: CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: valgrind --leak-check=no --error-exitcode=1 @@ -313,90 +407,54 @@ jobs: TRYBUILD: overwrite careful: - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - key: cargo-careful - continue-on-error: true + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@nightly with: components: rust-src - uses: taiki-e/install-action@cargo-careful - - run: python -m pip install --upgrade pip && pip install nox + - run: python -m pip install --upgrade pip && pip install nox[uv] - run: nox -s test-rust -- careful skip-full env: RUST_BACKTRACE: 1 TRYBUILD: overwrite docsrs: - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - key: cargo-careful - continue-on-error: true + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@nightly with: components: rust-src - - run: cargo rustdoc --lib --no-default-features --features full -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]" - - coverage: - needs: [fmt] - name: coverage-${{ matrix.os }} - strategy: - matrix: - os: ["windows", "macos", "ubuntu"] - runs-on: ${{ matrix.os }}-latest - steps: - - if: ${{ github.event_name == 'pull_request' && matrix.os != 'ubuntu' }} - id: should-skip - shell: bash - run: echo 'skip=true' >> $GITHUB_OUTPUT - - uses: actions/checkout@v4 - if: steps.should-skip.outputs.skip != 'true' - - uses: actions/setup-python@v5 - if: steps.should-skip.outputs.skip != 'true' - - uses: Swatinem/rust-cache@v2 - if: steps.should-skip.outputs.skip != 'true' - with: - key: coverage-cargo-${{ matrix.os }} - continue-on-error: true - - uses: dtolnay/rust-toolchain@stable - if: steps.should-skip.outputs.skip != 'true' - with: - components: llvm-tools-preview,rust-src - - name: Install cargo-llvm-cov - if: steps.should-skip.outputs.skip != 'true' - uses: taiki-e/install-action@cargo-llvm-cov - - run: python -m pip install --upgrade pip && pip install nox - if: steps.should-skip.outputs.skip != 'true' - - run: nox -s coverage - if: steps.should-skip.outputs.skip != 'true' - - uses: codecov/codecov-action@v4 - if: steps.should-skip.outputs.skip != 'true' - with: - file: coverage.json - name: ${{ matrix.os }} - token: ${{ secrets.CODECOV_TOKEN }} + - run: cargo rustdoc --lib --no-default-features --features full,jiff-02 -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]" emscripten: name: emscripten - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} + needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: - # TODO bump emscripten builds to test on 3.12 + # TODO bump emscripten builds to test on 3.13 python-version: 3.11 id: setup-python - name: Install Rust toolchain @@ -404,42 +462,47 @@ jobs: with: targets: wasm32-unknown-emscripten components: rust-src - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: - node-version: 14 - - run: python -m pip install --upgrade pip && pip install nox - - uses: actions/cache@v4 + node-version: 18 + - run: python -m pip install --upgrade pip && pip install nox[uv] + - uses: actions/cache/restore@v4 id: cache with: path: | .nox/emscripten - key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }} + key: emscripten-${{ hashFiles('emscripten/*') }}-${{ hashFiles('noxfile.py') }}-${{ steps.setup-python.outputs.python-path }} - uses: Swatinem/rust-cache@v2 with: - key: cargo-emscripten-wasm32 + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - name: Build if: steps.cache.outputs.cache-hit != 'true' run: nox -s build-emscripten - name: Test run: nox -s test-emscripten + - uses: actions/cache/save@v4 + if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} + with: + path: | + .nox/emscripten + key: emscripten-${{ hashFiles('emscripten/*') }}-${{ hashFiles('noxfile.py') }}-${{ steps.setup-python.outputs.python-path }} test-debug: + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] - if: github.ref != 'refs/heads/main' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: Swatinem/rust-cache@v2 with: - key: cargo-test-debug - continue-on-error: true + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@stable with: components: rust-src - name: Install python3 standalone debug build with nox run: | - PBS_RELEASE="20231002" - PBS_PYTHON_VERSION="3.12.0" + PBS_RELEASE="20241219" + PBS_PYTHON_VERSION="3.13.1" PBS_ARCHIVE="cpython-${PBS_PYTHON_VERSION}+${PBS_RELEASE}-x86_64-unknown-linux-gnu-debug-full.tar.zst" wget "https://github.com/indygreg/python-build-standalone/releases/download/$%7BPBS_RELEASE%7D/$%7BPBS_ARCHIVE%7D" tar -I zstd -xf "${PBS_ARCHIVE}" @@ -450,15 +513,15 @@ jobs: echo PYTHONHOME=$(pwd)/python/install >> $GITHUB_ENV echo PYO3_PYTHON=$(pwd)/python/install/bin/python3 >> $GITHUB_ENV - run: python3 -m sysconfig - - run: python3 -m pip install --upgrade pip && pip install nox + - run: python3 -m pip install --upgrade pip && pip install nox[uv] - run: | PYO3_CONFIG_FILE=$(mktemp) cat > $PYO3_CONFIG_FILE << EOF implementation=CPython - version=3.12 + version=3.13 shared=true abi3=false - lib_name=python3.12d + lib_name=python3.13d lib_dir=${{ github.workspace }}/python/install/lib executable=${{ github.workspace }}/python/install/bin/python3 pointer_width=64 @@ -470,16 +533,190 @@ jobs: test-version-limits: needs: [fmt] - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" - uses: Swatinem/rust-cache@v2 - continue-on-error: true + with: + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@stable - - run: python3 -m pip install --upgrade pip && pip install nox + - run: python3 -m pip install --upgrade pip && pip install nox[uv] - run: python3 -m nox -s test-version-limits + check-feature-powerset: + needs: [fmt, resolve] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} + runs-on: ubuntu-latest + name: check-feature-powerset ${{ matrix.rust }} + strategy: + # run on stable and MSRV to check that all combinations of features are expected to build fine on our supported + # range of compilers + matrix: + rust: ["stable"] + include: + - rust: ${{ needs.resolve.outputs.MSRV }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: taiki-e/install-action@v2 + with: + tool: cargo-hack,cargo-minimal-versions + - run: python3 -m pip install --upgrade pip && pip install nox[uv] + - run: python3 -m nox -s check-feature-powerset -- ${{ matrix.rust != 'stable' && 'minimal-versions' || '' }} + + test-cross-compilation: + needs: [fmt] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} + runs-on: ${{ matrix.os }} + name: test-cross-compilation ${{ matrix.os }} -> ${{ matrix.target }} + strategy: + # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present + fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} + matrix: + include: + # ubuntu "cross compile" to itself + - os: "ubuntu-latest" + target: "x86_64-unknown-linux-gnu" + flags: "-i python3.13" + manylinux: auto + # ubuntu x86_64 -> aarch64 + - os: "ubuntu-latest" + target: "aarch64-unknown-linux-gnu" + flags: "-i python3.13" + manylinux: auto + # ubuntu x86_64 -> windows x86_64 + - os: "ubuntu-latest" + target: "x86_64-pc-windows-gnu" + flags: "-i python3.13 --features generate-import-lib" + # windows x86_64 -> aarch64 + - os: "windows-latest" + target: "aarch64-pc-windows-msvc" + flags: "-i python3.13 --features generate-import-lib" + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - uses: Swatinem/rust-cache@v2 + with: + workspaces: examples/maturin-starter + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} + key: ${{ matrix.target }} + - name: Setup cross-compiler + if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }} + run: sudo apt-get install -y mingw-w64 llvm + - name: Compile version-specific library + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux }} + args: --release -m examples/maturin-starter/Cargo.toml ${{ matrix.flags }} + - name: Compile abi3 library + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux }} + args: --release -m examples/maturin-starter/Cargo.toml --features abi3 ${{ matrix.flags }} + + test-cross-compilation-windows: + needs: [fmt] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-pc-windows-gnu,x86_64-pc-windows-msvc + components: rust-src + # load cache (prepared in ci-cache-warmup.yml) + - uses: actions/cache/restore@v4 + with: + path: ~/.cache/cargo-xwin + key: cargo-xwin-cache + - name: Test cross compile to Windows + run: | + set -ex + sudo apt-get install -y mingw-w64 llvm + pip install nox + nox -s test-cross-compilation-windows + + test-introspection: + needs: [fmt] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-test-introspection') || contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} + strategy: + matrix: + platform: + [ + { + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + }, + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + }, + { + os: "windows-latest", + python-architecture: "x64", + rust-target: "x86_64-pc-windows-msvc", + }, + { + os: "windows-latest", + python-architecture: "x86", + rust-target: "i686-pc-windows-msvc", + }, + ] + runs-on: ${{ matrix.platform.os }} + steps: + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.platform.rust-target }} + components: rust-src + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + architecture: ${{ matrix.platform.python-architecture }} + - uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} + - run: python -m pip install --upgrade pip && pip install nox[uv] + - run: nox -s test-introspection + env: + CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }} + + test-introspection-pr: + needs: [fmt] + if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@stable + with: + components: rust-src + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - run: python -m pip install --upgrade pip && pip install nox[uv] + - run: nox -s test-introspection + conclusion: needs: - fmt @@ -490,10 +727,13 @@ jobs: - valgrind - careful - docsrs - - coverage - emscripten - test-debug - test-version-limits + - check-feature-powerset + - test-cross-compilation + - test-cross-compilation-windows + - test-introspection if: always() runs-on: ubuntu-latest steps: diff --git a/.github/workflows/coverage-pr-base.yml b/.github/workflows/coverage-pr-base.yml new file mode 100644 index 00000000000..7b21b399266 --- /dev/null +++ b/.github/workflows/coverage-pr-base.yml @@ -0,0 +1,33 @@ +# This runs as a separate job because it needs to run on the `pull_request_target` event +# in order to access the CODECOV_TOKEN secret. +# +# This is safe because this doesn't run arbitrary code from PRs. + +name: Set Codecov PR base +on: + # See safety note / doc at the top of this file. + pull_request_target: + +jobs: + coverage-pr-base: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - name: Fetch merge base + id: fetch_merge_base + uses: ./.github/actions/fetch-merge-base + with: + base_ref: "refs/heads/${{ github.event.pull_request.base.ref }}" + head_ref: "refs/pull/${{ github.event.pull_request.number }}/head" + - name: Set PR base on codecov + run: | + pip install codecov-cli + codecovcli pr-base-picking \ + --base-sha ${{ steps.fetch_merge_base.outputs.merge_base }} \ + --pr ${{ github.event.number }} \ + --slug PyO3/pyo3 \ + --token ${{ secrets.CODECOV_TOKEN }} \ + --service github diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml deleted file mode 100644 index ec8874d5841..00000000000 --- a/.github/workflows/gh-pages.yml +++ /dev/null @@ -1,139 +0,0 @@ -name: gh-pages - -on: - push: - branches: - - main - pull_request: - release: - types: [published] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -env: - CARGO_TERM_COLOR: always - -jobs: - guide-build: - runs-on: ubuntu-latest - outputs: - tag_name: ${{ steps.prepare_tag.outputs.tag_name }} - steps: - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@nightly - - - name: Setup mdBook - uses: peaceiris/actions-mdbook@v1 - with: - mdbook-version: "0.4.19" - - - name: Prepare tag - id: prepare_tag - run: | - TAG_NAME="${GITHUB_REF##*/}" - echo "::set-output name=tag_name::${TAG_NAME}" - - # This builds the book in target/guide. - - name: Build the guide - run: | - python -m pip install --upgrade pip && pip install nox - nox -s build-guide - env: - PYO3_VERSION_TAG: ${{ steps.prepare_tag.outputs.tag_name }} - - - name: Deploy docs and the guide - if: ${{ github.event_name == 'release' }} - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./target/guide/ - destination_dir: ${{ steps.prepare_tag.outputs.tag_name }} - full_commit_message: "Upload documentation for ${{ steps.prepare_tag.outputs.tag_name }}" - - cargo-benchmark: - if: ${{ github.ref_name == 'main' }} - name: Cargo benchmark - needs: guide-build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - uses: dtolnay/rust-toolchain@stable - - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: cargo-${{ runner.os }}-bench-${{ hashFiles('**/Cargo.toml') }} - continue-on-error: true - - - name: Run benchmarks - run: | - python -m pip install --upgrade pip && pip install nox - for bench in pyo3-benches/benches/*.rs; do - bench_name=$(basename "$bench" .rs) - nox -s bench -- --bench "$bench_name" -- --output-format bencher | tee -a output.txt - done - - # Download previous benchmark result from cache (if exists) - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - - # Run `github-action-benchmark` action - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: pyo3-bench - # What benchmark tool the output.txt came from - tool: "cargo" - # Where the output from the benchmark tool is stored - output-file-path: output.txt - # GitHub API token to make a commit comment - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: ${{ github.event_name != 'pull_request' }} - - pytest-benchmark: - if: ${{ github.ref_name == 'main' }} - name: Pytest benchmark - needs: cargo-benchmark - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - uses: dtolnay/rust-toolchain@stable - - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: cargo-${{ runner.os }}-pytest-bench-${{ hashFiles('**/Cargo.toml') }} - continue-on-error: true - - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-pytest-benchmark - - - name: Run benchmarks - run: | - python -m pip install --upgrade pip && pip install nox - nox -f pytests/noxfile.py -s bench -- --benchmark-json $(pwd)/output.json - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: pytest-bench - tool: "pytest" - output-file-path: output.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/netlify-build.yml b/.github/workflows/netlify-build.yml new file mode 100644 index 00000000000..6150af3db24 --- /dev/null +++ b/.github/workflows/netlify-build.yml @@ -0,0 +1,78 @@ +name: netlify-build + +on: + push: + branches: + - main + pull_request: + release: + types: [published] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + guide-build: + runs-on: ubuntu-latest + outputs: + tag_name: ${{ steps.prepare_tag.outputs.tag_name }} + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - uses: dtolnay/rust-toolchain@nightly + + - name: Setup mdBook + uses: taiki-e/install-action@v2 + with: + tool: mdbook, mdbook-linkcheck, lychee + + - name: Prepare tag + id: prepare_tag + run: | + TAG_NAME="${GITHUB_REF##*/}" + echo "tag_name=${TAG_NAME}" >> $GITHUB_OUTPUT + + # This builds the book in target/guide/. + - name: Build the guide + run: | + python -m pip install --upgrade pip && pip install nox[uv] + nox -s ${{ github.event_name == 'release' && 'build-guide' || 'check-guide' }} + env: + PYO3_VERSION_TAG: ${{ steps.prepare_tag.outputs.tag_name }} + # allows lychee to get better rate limits from github + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # We store the versioned guides on GitHub's gh-pages branch for convenience + # (the full gh-pages branch is pulled in the build-netlify-site step) + - name: Deploy the guide + if: ${{ github.event_name == 'release' }} + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./target/guide/ + destination_dir: ${{ steps.prepare_tag.outputs.tag_name }} + full_commit_message: "Upload documentation for ${{ steps.prepare_tag.outputs.tag_name }}" + + - name: Get current PyO3 version + run: | + PYO3_VERSION=$(cargo search pyo3 --limit 1 | head -1 | tr -s ' ' | cut -d ' ' -f 3 | tr -d '"') + echo "PYO3_VERSION=${PYO3_VERSION}" >> $GITHUB_ENV + + - name: Build the site + run: | + python -m pip install --upgrade pip && pip install nox[uv] towncrier requests + nox -s build-netlify-site -- ${{ (github.ref != 'refs/heads/main' && '--preview') || '' }} + + # Upload the built site as an artifact for deploy workflow to consume + - name: Upload Build Artifact + uses: actions/upload-artifact@v4 + with: + name: site + path: ./netlify_build diff --git a/.github/workflows/netlify-deploy.yml b/.github/workflows/netlify-deploy.yml new file mode 100644 index 00000000000..0985e3a6584 --- /dev/null +++ b/.github/workflows/netlify-deploy.yml @@ -0,0 +1,69 @@ +# This runs as a separate job because it needs to run on the `workflow_run` event +# in order to access the netlify secrets. +# +# This is safe because this doesn't run arbitrary code from PRs. + +name: netlify-deploy + +on: + workflow_run: + workflows: ["netlify-build"] + types: + - completed + +env: + CARGO_TERM_COLOR: always + +jobs: + deploy: + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion == 'success' + environment: netlify + + steps: + - name: Download Build Artifact + uses: actions/download-artifact@v5 + with: + name: site + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Install netlify-cli + run: | + npm install -g netlify-cli + + - name: Deploy to Netlify + run: | + ls -la + DEBUG=* netlify deploy \ + --site ${{ secrets.NETLIFY_SITE_ID }} \ + --auth ${{ secrets.NETLIFY_TOKEN }} \ + ${{ ((github.event.workflow_run.head_repository.full_name == 'PyO3/pyo3') && (github.event.workflow_run.head_branch == 'main') && '--prod') || '' }} \ + --json | tee deploy_output.json + + # credit: https://www.raulmelo.me/en/blog/deploying-netlify-github-actions-guide + - name: Generate URL Preview + id: url_preview + if: ${{ github.event.workflow_run.event == 'pull_request' }} + run: | + NETLIFY_PREVIEW_URL=$(jq -r '.deploy_url' deploy_output.json) + echo "NETLIFY_PREVIEW_URL=$NETLIFY_PREVIEW_URL" >> "$GITHUB_OUTPUT" + + - name: Post Netlify Preview Status to PR + if: ${{ github.event.workflow_run.event == 'pull_request' }} + uses: actions/github-script@v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const previewUrl = '${{ steps.url_preview.outputs.NETLIFY_PREVIEW_URL }}'; + const commitSha = '${{ github.event.workflow_run.head_sha }}'; + + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: commitSha, + state: 'success', + target_url: previewUrl, + description: 'click to view Netlify preview deploy', + context: 'netlify-deploy / preview' + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..d7642ab10b5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,34 @@ +name: Release Rust Crate + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + version: + description: The version to build + +jobs: + release: + permissions: + id-token: write + + runs-on: ubuntu-latest + environment: release + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + # The tag to build or the tag received by the tag event + ref: ${{ github.event.inputs.version || github.ref }} + persist-credentials: false + + - uses: astral-sh/setup-uv@v7 + + - uses: rust-lang/crates-io-auth-action@v1 + id: auth + + - name: Publish to crates.io + run: uvx nox -s publish + env: + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} diff --git a/.gitignore b/.gitignore index 4240d326f71..da64603db5f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ dist/ .eggs/ venv* guide/book/ +guide/src/LICENSE-APACHE +guide/src/LICENSE-MIT *.so *.out *.egg-info @@ -22,5 +24,7 @@ pip-wheel-metadata valgrind-python.supp *.pyd lcov.info +coverage.json netlify_build/ .nox/ +.vscode/ diff --git a/.netlify/build.sh b/.netlify/build.sh index 10ec241c1d7..dedacb28fb3 100755 --- a/.netlify/build.sh +++ b/.netlify/build.sh @@ -2,6 +2,7 @@ set -uex +rustup update nightly rustup default nightly PYO3_VERSION=$(cargo search pyo3 --limit 1 | head -1 | tr -s ' ' | cut -d ' ' -f 3 | tr -d '"') @@ -48,6 +49,15 @@ done # Add latest redirect echo "/latest/* /v${PYO3_VERSION}/:splat 302" >> netlify_build/_redirects +# some backwards compatbiility redirects +echo "/latest/building_and_distribution/* /latest/building-and-distribution/:splat 302" >> netlify_build/_redirects +echo "/latest/building-and-distribution/multiple_python_versions/* /latest/building-and-distribution/multiple-python-versions:splat 302" >> netlify_build/_redirects +echo "/latest/function/error_handling/* /latest/function/error-handling/:splat 302" >> netlify_build/_redirects +echo "/latest/getting_started/* /latest/getting-started/:splat 302" >> netlify_build/_redirects +echo "/latest/python_from_rust/* /latest/python-from-rust/:splat 302" >> netlify_build/_redirects +echo "/latest/python_typing_hints/* /latest/python-typing-hints/:splat 302" >> netlify_build/_redirects +echo "/latest/trait_bounds/* /latest/trait-bounds/:splat 302" >> netlify_build/_redirects + ## Add landing page redirect if [ "${CONTEXT}" == "deploy-preview" ]; then echo "/ /main/" >> netlify_build/_redirects @@ -71,9 +81,25 @@ if [ "${INSTALLED_MDBOOK_VERSION}" != "mdbook v${MDBOOK_VERSION}" ]; then cargo install mdbook@${MDBOOK_VERSION} --force fi -pip install nox +# Install latest mdbook-linkcheck. Netlify will cache the cargo bin dir, so this will +# only build mdbook-linkcheck if needed. +MDBOOK_LINKCHECK_VERSION=$(cargo search mdbook-linkcheck --limit 1 | head -1 | tr -s ' ' | cut -d ' ' -f 3 | tr -d '"') +INSTALLED_MDBOOK_LINKCHECK_VERSION=$(mdbook-linkcheck --version || echo "none") +if [ "${INSTALLED_MDBOOK_LINKCHECK_VERSION}" != "mdbook v${MDBOOK_LINKCHECK_VERSION}" ]; then + cargo install mdbook-linkcheck@${MDBOOK_LINKCHECK_VERSION} --force +fi + +# Install latest mdbook-tabs. Netlify will cache the cargo bin dir, so this will +# only build mdbook-tabs if needed. +MDBOOK_TABS_VERSION=$(cargo search mdbook-tabs --limit 1 | head -1 | tr -s ' ' | cut -d ' ' -f 3 | tr -d '"') +INSTALLED_MDBOOK_TABS_VERSION=$(mdbook-tabs --version || echo "none") +if [ "${INSTALLED_MDBOOK_TABS_VERSION}" != "mdbook-tabs v${MDBOOK_TABS_VERSION}" ]; then + cargo install mdbook-tabs@${MDBOOK_TABS_VERSION} --force +fi + +pip install nox[uv] nox -s build-guide -mv target/guide netlify_build/main/ +mv target/guide/ netlify_build/main/ ## Build public docs diff --git a/.netlify/redirect.sh b/.netlify/redirect.sh new file mode 100644 index 00000000000..46d2dbeedef --- /dev/null +++ b/.netlify/redirect.sh @@ -0,0 +1,49 @@ +# Add redirect for each documented version +set +x # these loops get very spammy and fill the deploy log + +for d in netlify_build/v*; do + version="${d/netlify_build\/v/}" + echo "/v$version/doc/* https://docs.rs/pyo3/$version/:splat" >> netlify_build/_redirects + if [ $version != $PYO3_VERSION ]; then + # for old versions, mark the files in the latest version as the canonical URL + for file in $(find $d -type f); do + file_path="${file/$d\//}" + # remove index.html and/or .html suffix to match the page URL on the + # final netlfiy site + url_path="$file_path" + if [[ $file_path == index.html ]]; then + url_path="" + elif [[ $file_path == *.html ]]; then + url_path="${file_path%.html}" + fi + echo "/v$version/$url_path" >> netlify_build/_headers + if test -f "netlify_build/v$PYO3_VERSION/$file_path"; then + echo " Link: ; rel=\"canonical\"" >> netlify_build/_headers + else + # this file doesn't exist in the latest guide, don't index it + echo " X-Robots-Tag: noindex" >> netlify_build/_headers + fi + done + fi +done + +# Add latest redirect +echo "/latest/* /v${PYO3_VERSION}/:splat 302" >> netlify_build/_redirects + +# some backwards compatbiility redirects +echo "/latest/building_and_distribution/* /latest/building-and-distribution/:splat 302" >> netlify_build/_redirects +echo "/latest/building-and-distribution/multiple_python_versions/* /latest/building-and-distribution/multiple-python-versions:splat 302" >> netlify_build/_redirects +echo "/latest/function/error_handling/* /latest/function/error-handling/:splat 302" >> netlify_build/_redirects +echo "/latest/getting_started/* /latest/getting-started/:splat 302" >> netlify_build/_redirects +echo "/latest/python_from_rust/* /latest/python-from-rust/:splat 302" >> netlify_build/_redirects +echo "/latest/python_typing_hints/* /latest/python-typing-hints/:splat 302" >> netlify_build/_redirects +echo "/latest/trait_bounds/* /latest/trait-bounds/:splat 302" >> netlify_build/_redirects + +## Add landing page redirect +if [ "${CONTEXT}" == "deploy-preview" ]; then + echo "/ /main/" >> netlify_build/_redirects +else + echo "/ /v${PYO3_VERSION}/ 302" >> netlify_build/_redirects +fi + +set -x \ No newline at end of file diff --git a/.python-version b/.python-version deleted file mode 100644 index e4fba218358..00000000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.12 diff --git a/Architecture.md b/Architecture.md index 9ec20931c10..2b081a487e4 100644 --- a/Architecture.md +++ b/Architecture.md @@ -37,18 +37,15 @@ automated tooling because: - it gives us best control about how to adapt C conventions to Rust, and - there are many Python interpreter versions we support in a single set of files. -We aim to provide straight-forward Rust wrappers resembling the file structure of -[`cpython/Include`](https://github.com/python/cpython/tree/v3.9.2/Include). +We aim to provide straight-forward Rust wrappers resembling the file structure of [`cpython/Include`](https://github.com/python/cpython/tree/main/Include). -However, we still lack some APIs and are continuously updating the module to match -the file contents upstream in CPython. -The tracking issue is [#1289](https://github.com/PyO3/pyo3/issues/1289), and contribution is welcome. +We are continuously updating the module to match the latest CPython version which PyO3 supports (i.e. as of time of writing Python 3.13). The tracking issue is [#1289](https://github.com/PyO3/pyo3/issues/1289), and contribution is welcome. In the [`pyo3-ffi`] crate, there is lots of conditional compilation such as `#[cfg(Py_LIMITED_API)]`, `#[cfg(Py_3_7)]`, and `#[cfg(PyPy)]`. `Py_LIMITED_API` corresponds to `#define Py_LIMITED_API` macro in Python/C API. With `Py_LIMITED_API`, we can build a Python-version-agnostic binary called an -[abi3 wheel](https://pyo3.rs/latest/building_and_distribution.html#py_limited_apiabi3). +[abi3 wheel](https://pyo3.rs/latest/building-and-distribution.html#py_limited_apiabi3). `Py_3_7` means that the API is available from Python >= 3.7. There are also `Py_3_8`, `Py_3_9`, and so on. `PyPy` means that the API definition is for PyPy. @@ -59,6 +56,7 @@ Those flags are set in [`build.rs`](#6-buildrs-and-pyo3-build-config). [`src/types`] contains bindings to [built-in types](https://docs.python.org/3/library/stdtypes.html) of Python, such as `dict` and `list`. For historical reasons, Python's `object` is called `PyAny` in PyO3 and located in [`src/types/any.rs`]. + Currently, `PyAny` is a straightforward wrapper of `ffi::PyObject`, defined as: ```rust @@ -66,38 +64,16 @@ Currently, `PyAny` is a straightforward wrapper of `ffi::PyObject`, defined as: pub struct PyAny(UnsafeCell); ``` -All built-in types are defined as a C struct. -For example, `dict` is defined as: - -```c -typedef struct { - /* Base object */ - PyObject ob_base; - /* Number of items in the dictionary */ - Py_ssize_t ma_used; - /* Dictionary version */ - uint64_t ma_version_tag; - PyDictKeysObject *ma_keys; - PyObject **ma_values; -} PyDictObject; -``` - -However, we cannot access such a specific data structure with `#[cfg(Py_LIMITED_API)]` set. -Thus, all builtin objects are implemented as opaque types by wrapping `PyAny`, e.g.,: +Concrete Python objects are implemented by wrapping `PyAny`, e.g.,: ```rust #[repr(transparent)] pub struct PyDict(PyAny); ``` -Note that `PyAny` is not a pointer, and it is usually used as a pointer to the object in the -Python heap, as `&PyAny`. -This design choice can be changed -(see the discussion in [#1056](https://github.com/PyO3/pyo3/issues/1056)). +These types are not intended to be accessed directly, and instead are used through the `Py` and `Bound` smart pointers. -Since we need lots of boilerplate for implementing common traits for these types -(e.g., `AsPyPointer`, `AsRef`, and `Debug`), we have some macros in -[`src/types/mod.rs`]. +We have some macros in [`src/types/mod.rs`] which make it easier to implement APIs for concrete Python types. ## 3. `PyClass` and related functionalities @@ -109,23 +85,23 @@ To realize object-oriented programming in C, all Python objects have `ob_base: P first field in their structure definition. Thanks to this guarantee, casting `*mut A` to `*mut PyObject` is valid if `A` is a Python object. -To ensure this guarantee, we have a wrapper struct `PyCell` in [`src/pycell.rs`] which is roughly: +To ensure this guarantee, we have a wrapper struct `PyClassObject` in [`src/pycell/impl_.rs`] which is roughly: ```rust #[repr(C)] -pub struct PyCell { +pub struct PyClassObject { ob_base: crate::ffi::PyObject, inner: T, } ``` -Thus, when copying a Rust struct to a Python object, we first allocate `PyCell` on the Python heap and then +Thus, when copying a Rust struct to a Python object, we first allocate `PyClassObject` on the Python heap and then move `T` into it. -Also, `PyCell` provides [RefCell](https://doc.rust-lang.org/std/cell/struct.RefCell.html)-like methods -to ensure Rust's borrow rules. -See [the documentation](https://docs.rs/pyo3/latest/pyo3/pycell/struct.PyCell.html) for more. -`PyCell` requires that `T` implements `PyClass`. +The primary way to interact with Python objects implemented in Rust is through the `Bound<'py, T>` smart pointer. +By having the `'py` lifetime of the `Python<'py>` token, this ties the lifetime of the `Bound<'py, T>` smart pointer to the lifetime for which the thread is attached to the Python interpreter and allows PyO3 to call Python APIs at maximum efficiency. + +`Bound<'py, T>` requires that `T` implements `PyClass`. This trait is somewhat complex and derives many traits, but the most important one is `PyTypeInfo` in [`src/type_object.rs`]. `PyTypeInfo` is also implemented for built-in types. diff --git a/CHANGELOG.md b/CHANGELOG.md index 9104db2921f..2872fac91a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,747 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.27.0] - 2025-10-19 + +### Packaging + +- Extend range of supported versions of `hashbrown` optional dependency to include version 0.16. [#5428](https://github.com/PyO3/pyo3/pull/5428) +- Bump optional `num-bigint` dependency minimum version to 0.4.4. [#5471](https://github.com/PyO3/pyo3/pull/5471) +- Test against Python 3.14 final release. [#5499](https://github.com/PyO3/pyo3/pull/5499) +- Drop support for PyPy 3.9 and 3.10. [#5516](https://github.com/PyO3/pyo3/pull/5516) +- Provide a better error message when building an outdated PyO3 for a too-new Python version. [#5519](https://github.com/PyO3/pyo3/pull/5519) + +### Added + +- Add `FromPyObjectOwned` as convenient trait bound for `FromPyObject` when the data is not borrowed from Python. [#4390](https://github.com/PyO3/pyo3/pull/4390) +- Add `Borrowed::extract`, same as `PyAnyMethods::extract`, but does not restrict the lifetime by deref. [#4390](https://github.com/PyO3/pyo3/pull/4390) +- `experimental-inspect`: basic support for `#[derive(IntoPyObject)]` (no struct fields support yet). [#5365](https://github.com/PyO3/pyo3/pull/5365) +- `experimental-inspect`: support `#[pyo3(get, set)]` and `#[pyclass(get_all, set_all)]`. [#5370](https://github.com/PyO3/pyo3/pull/5370) +- Add `PyTypeCheck::classinfo_object` that returns an object that can be used as parameter in `isinstance` or `issubclass`. [#5387](https://github.com/PyO3/pyo3/pull/5387) +- Implement `PyTypeInfo` on `datetime.*` types even when the limited API is enabled. [#5388](https://github.com/PyO3/pyo3/pull/5388) +- Implement `PyTypeInfo` on `PyIterator`, `PyMapping` and `PySequence`. [#5402](https://github.com/PyO3/pyo3/pull/5402) +- Implement `PyTypeInfo` on `PyCode` when using the stable ABI. [#5403](https://github.com/PyO3/pyo3/pull/5403) +- Implement `PyTypeInfo` on `PyWeakrefReference` when using the stable ABI. [#5404](https://github.com/PyO3/pyo3/pull/5404) +- Add `pyo3::sync::RwLockExt` trait, analogous to `pyo3::sync::MutexExt` for readwrite locks. [#5435](https://github.com/PyO3/pyo3/pull/5435) +- Add `PyString::from_bytes`. [#5437](https://github.com/PyO3/pyo3/pull/5437) +- Implement `AsRef<[u8]>` for `PyBytes`. [#5445](https://github.com/PyO3/pyo3/pull/5445) +- Add `CastError` and `CastIntoError`. [#5468](https://github.com/PyO3/pyo3/pull/5468) +- Add `PyCapsuleMethods::pointer_checked` and `PyCapsuleMethods::is_valid_checked`. [#5474](https://github.com/PyO3/pyo3/pull/5474) +- Add `Borrowed::cast`, `Borrowed::cast_exact` and `Borrowed::cast_unchecked`. [#5475](https://github.com/PyO3/pyo3/pull/5475) +- Add conversions for `jiff::civil::ISOWeekDate`. [#5478](https://github.com/PyO3/pyo3/pull/5478) +- Add conversions for `&Cstr`, `Cstring` and `Cow`. [#5482](https://github.com/PyO3/pyo3/pull/5482) +- add `#[pyclass(skip_from_py_object)]` option, to opt-out of the `FromPyObject: PyClass + Clone` blanket impl. [#5488](https://github.com/PyO3/pyo3/pull/5488) +- Add `PyErr::add_note`. [#5489](https://github.com/PyO3/pyo3/pull/5489) +- Add `FromPyObject` impl for `Cow` & `Cow`. [#5497](https://github.com/PyO3/pyo3/pull/5497) +- Add `#[pyclass(from_py_object)]` pyclass option, to opt-in to the extraction of pyclasses by value (requires `Clone`). [#5506](https://github.com/PyO3/pyo3/pull/5506) + +### Changed + +- Rework `FromPyObject` trait for flexibility and performance: [#4390](https://github.com/PyO3/pyo3/pull/4390) + - Add a second lifetime to `FromPyObject`, to allow borrowing data from Python objects (e.g. `&str` from Python `str`). + - Replace `extract_bound` with `extract`, which takes `Borrowed<'a, 'py, PyAny>`. +- Optimize `FromPyObject` implementations for `Vec` and `[u8; N]` from `bytes` and `bytearray`. [#5244](https://github.com/PyO3/pyo3/pull/5244) +- Deprecate `#[pyfn]` attribute. [#5384](https://github.com/PyO3/pyo3/pull/5384) +- Fetch type name dynamically on cast errors instead of using `PyTypeCheck::NAME`. [#5387](https://github.com/PyO3/pyo3/pull/5387) +- Deprecate `PyTypeCheck::NAME` in favour of `PyTypeCheck::classinfo_object` which provides the type information at runtime. [#5387](https://github.com/PyO3/pyo3/pull/5387) +- `PyClassGuard(Mut)` and `PyRef(Mut)` extraction now returns an opaque Rust error [#5413](https://github.com/PyO3/pyo3/pull/5413) +- Fetch type name dynamically when exporting types implementing `PyTypeInfo` with `#[pymodule_use]`. [#5414](https://github.com/PyO3/pyo3/pull/5414) +- Improve `Debug` representation of `PyBuffer`. [#5442](https://github.com/PyO3/pyo3/pull/5442) +- `experimental-inspect`: change the way introspection data is emitted in the binaries to avoid a pointer indirection and simplify parsing. [#5450](https://github.com/PyO3/pyo3/pull/5450) +- Optimize `Py::drop` for the case when attached to the Python interpreter. [#5454](https://github.com/PyO3/pyo3/pull/5454) +- Replace `DowncastError` and `DowncastIntoError` with `CastError` and `CastIntoError`. [#5468](https://github.com/PyO3/pyo3/pull/5468) +- Enable fast-path for 128-bit integer conversions on `GraalPy`. [#5471](https://github.com/PyO3/pyo3/pull/5471) +- Deprecate `PyAnyMethods::downcast` functions in favour of `Bound::cast` functions. [#5472](https://github.com/PyO3/pyo3/pull/5472) +- Make `PyTypeCheck` an `unsafe trait`. [#5473](https://github.com/PyO3/pyo3/pull/5473) +- Deprecate unchecked `PyCapsuleMethods`: `pointer()`, `reference()`, and `is_valid()`. [#5474](https://github.com/PyO3/pyo3/pull/5474) +- Reduce lifetime of return value in `PyCapsuleMethods::reference`. [#5474](https://github.com/PyO3/pyo3/pull/5474) +- `PyCapsuleMethods::name` now returns `CapsuleName` wrapper instead of `&CStr`. [#5474](https://github.com/PyO3/pyo3/pull/5474) +- Deprecate `import_exception_bound` in favour of `import_exception`. [#5480](https://github.com/PyO3/pyo3/pull/5480) +- `PyList::get_item_unchecked`, `PyTuple::get_item_unchecked`, and `PyTuple::get_borrowed_item_unchecked` no longer check for null values at the provided index. [#5494](https://github.com/PyO3/pyo3/pull/5494) +- Allow converting naive datetime into chrono `DateTime`. [#5507](https://github.com/PyO3/pyo3/pull/5507) + +### Removed + +- Removed `FromPyObjectBound` trait. [#4390](https://github.com/PyO3/pyo3/pull/4390) + +### Fixed + +- Fix compilation failure on `wasm32-wasip2`. [#5368](https://github.com/PyO3/pyo3/pull/5368) +- Fix `OsStr` conversion for non-utf8 strings on Windows. [#5444](https://github.com/PyO3/pyo3/pull/5444) +- Fix issue with `cargo vendor` caused by gitignored build artifact `emscripten/pybuilddir.txt`. [#5456](https://github.com/PyO3/pyo3/pull/5456) +- Stop leaking `PyMethodDef` instances inside `#[pyfunction]` macro generated code. [#5459](https://github.com/PyO3/pyo3/pull/5459) +- Don't export definition of FFI struct `PyObjectObFlagsAndRefcnt` on 32-bit Python 3.14 (doesn't exist). [#5499](https://github.com/PyO3/pyo3/pull/5499) +- Fix failure to build for `abi3` interpreters on Windows using maturin's built-in sysconfig in combination with the `generate-import-lib` feature. [#5503](https://github.com/PyO3/pyo3/pull/5503) +- Fix FFI definitions `PyModule_ExecDef` and `PyModule_FromDefAndSpec2` on PyPy. [#5529](https://github.com/PyO3/pyo3/pull/5529) + +## [0.26.0] - 2025-08-29 + +### Packaging + +- Bump hashbrown dependency to 0.15. [#5152](https://github.com/PyO3/pyo3/pull/5152) +- Update MSRV to 1.74. [#5171](https://github.com/PyO3/pyo3/pull/5171) +- Set the same maximum supported version for alternative interpreters as for CPython. [#5192](https://github.com/PyO3/pyo3/pull/5192) +- Add optional `bytes` dependency to add conversions for `bytes::Bytes`. [#5252](https://github.com/PyO3/pyo3/pull/5252) +- Publish new crate `pyo3-introspection` to pair with the `experimental-inspect` feature. [#5300](https://github.com/PyO3/pyo3/pull/5300) +- The `PYO3_BUILD_EXTENSION_MODULE` now causes the same effect as the `extension-module` feature. Eventually we expect maturin and setuptools-rust to set this environment variable automatically. Users with their own build systems will need to do the same. [#5343](https://github.com/PyO3/pyo3/pull/5343) + +### Added + +- Add `#[pyo3(warn(message = "...", category = ...))]` attribute for automatic warnings generation for `#[pyfunction]` and `#[pymethods]`. [#4364](https://github.com/PyO3/pyo3/pull/4364) +- Add `PyMutex`, available on Python 3.13 and newer. [#4523](https://github.com/PyO3/pyo3/pull/4523) +- Add FFI definition `PyMutex_IsLocked`, available on Python 3.14 and newer. [#4523](https://github.com/PyO3/pyo3/pull/4523) +- Add `PyString::from_encoded_object`. [#5017](https://github.com/PyO3/pyo3/pull/5017) +- `experimental-inspect`: add basic input type annotations. [#5089](https://github.com/PyO3/pyo3/pull/5089) +- Add FFI function definitions for `PyFrameObject` from CPython 3.13. [#5154](https://github.com/PyO3/pyo3/pull/5154) +- `experimental-inspect`: tag modules created using `#[pymodule]` or `#[pymodule_init]` functions as incomplete. [#5207](https://github.com/PyO3/pyo3/pull/5207) +- `experimental-inspect`: add basic return type support. [#5208](https://github.com/PyO3/pyo3/pull/5208) +- Add `PyCode::compile` and `PyCodeMethods::run` to create and execute code objects. [#5217](https://github.com/PyO3/pyo3/pull/5217) +- Add `PyOnceLock` type for thread-safe single-initialization. [#5223](https://github.com/PyO3/pyo3/pull/5223) +- Add `PyClassGuard(Mut)` pyclass holders. In the future they will replace `PyRef(Mut)`. [#5233](https://github.com/PyO3/pyo3/pull/5233) +- `experimental-inspect`: allow annotations in `#[pyo3(signature)]` signature attribute. [#5241](https://github.com/PyO3/pyo3/pull/5241) +- Implement `MutexExt` for parking_lot's/lock_api `ReentrantMutex`. [#5258](https://github.com/PyO3/pyo3/pull/5258) +- `experimental-inspect`: support class associated constants. [#5272](https://github.com/PyO3/pyo3/pull/5272) +- Add `Bound::cast` family of functions superseding the `PyAnyMethods::downcast` family. [#5289](https://github.com/PyO3/pyo3/pull/5289) +- Add FFI definitions `Py_Version` and `Py_IsFinalizing`. [#5317](https://github.com/PyO3/pyo3/pull/5317) +- `experimental-inspect`: add output type annotation for `#[pyclass]`. [#5320](https://github.com/PyO3/pyo3/pull/5320) +- `experimental-inspect`: support `#[pyclass(eq, eq_int, ord, hash, str)]`. [#5338](https://github.com/PyO3/pyo3/pull/5338) +- `experimental-inspect`: add basic support for `#[derive(FromPyObject)]` (no struct fields support yet). [#5339](https://github.com/PyO3/pyo3/pull/5339) +- Add `Python::try_attach`. [#5342](https://github.com/PyO3/pyo3/pull/5342) + +### Changed + +- Use `Py_TPFLAGS_DISALLOW_INSTANTIATION` instead of a `__new__` which always fails for a `#[pyclass]` without a `#[new]` on Python 3.10 and up. [#4568](https://github.com/PyO3/pyo3/pull/4568) +- `PyModule::from_code` now defaults `file_name` to `` if empty. [#4777](https://github.com/PyO3/pyo3/pull/4777) +- Deprecate `PyString::from_object` in favour of `PyString::from_encoded_object`. [#5017](https://github.com/PyO3/pyo3/pull/5017) +- When building with `abi3` for a Python version newer than pyo3 supports, automatically fall back to an abi3 build for the latest supported version. [#5144](https://github.com/PyO3/pyo3/pull/5144) +- Change `is_instance_of` trait bound from `PyTypeInfo` to `PyTypeCheck`. [#5146](https://github.com/PyO3/pyo3/pull/5146) +- Many PyO3 proc macros now report multiple errors instead of only the first one. [#5159](https://github.com/PyO3/pyo3/pull/5159) +- Change `MutexExt` return type to be an associated type. [#5201](https://github.com/PyO3/pyo3/pull/5201) +- Use `PyCallArgs` for `Py::call` and friends so they're equivalent to their `Bound` counterpart. [#5206](https://github.com/PyO3/pyo3/pull/5206) +- Rename `Python::with_gil` to `Python::attach`. [#5209](https://github.com/PyO3/pyo3/pull/5209) +- Rename `Python::allow_threads` to `Python::detach` [#5221](https://github.com/PyO3/pyo3/pull/5221) +- Deprecate `GILOnceCell` type in favour of `PyOnceLock`. [#5223](https://github.com/PyO3/pyo3/pull/5223) +- Rename `pyo3::prepare_freethreaded_python` to `Python::initialize`. [#5247](https://github.com/PyO3/pyo3/pull/5247) +- Convert `PyMemoryError` into/from `io::ErrorKind::OutOfMemory`. [#5256](https://github.com/PyO3/pyo3/pull/5256) +- Deprecate `GILProtected`. [#5285](https://github.com/PyO3/pyo3/pull/5285) +- Move `#[pyclass]` docstring formatting from import time to compile time. [#5286](https://github.com/PyO3/pyo3/pull/5286) +- `Python::attach` will now panic if the Python interpreter is in the process of shutting down. [#5317](https://github.com/PyO3/pyo3/pull/5317) +- Add fast-path to `PyTypeInfo::type_object` for `#[pyclass]` types. [#5324](https://github.com/PyO3/pyo3/pull/5324) +- Deprecate `PyObject` type alias for `Py`. [#5325](https://github.com/PyO3/pyo3/pull/5325) +- Rename `Python::with_gil_unchecked` to `Python::attach_unchecked`. [#5340](https://github.com/PyO3/pyo3/pull/5340) +- Rename `Python::assume_gil_acquired` to `Python::assume_attached`. [#5354](https://github.com/PyO3/pyo3/pull/5354) + +### Removed + +- Remove FFI definition of internals of `PyFrameObject`. [#5154](https://github.com/PyO3/pyo3/pull/5154) +- Remove `Eq` and `PartialEq` implementations on `PyGetSetDef` FFI definition. [#5196](https://github.com/PyO3/pyo3/pull/5196) +- Remove private FFI definitions `_Py_IsCoreInitialized` and `_Py_InitializeMain`. [#5317](https://github.com/PyO3/pyo3/pull/5317) + +### Fixed + +- Use critical section in `PyByteArray::to_vec` on freethreaded build to replicate GIL-enabled "soundness". [#4742](https://github.com/PyO3/pyo3/pull/4742) +- Fix precision loss when converting `bigdecimal` into Python. [#5198](https://github.com/PyO3/pyo3/pull/5198) +- Don't treat win7 target as a cross-compilation. [#5210](https://github.com/PyO3/pyo3/pull/5210) +- WASM targets no longer require exception handling support for Python < 3.14. [#5239](https://github.com/PyO3/pyo3/pull/5239) +- Fix segfault when dropping `PyBuffer` after the Python interpreter has been finalized. [#5242](https://github.com/PyO3/pyo3/pull/5242) +- `experimental-inspect`: better automated imports generation. [#5251](https://github.com/PyO3/pyo3/pull/5251) +- `experimental-inspect`: fix introspection of `__richcmp__`, `__concat__`, `__repeat__`, `__inplace_concat__` and `__inplace_repeat__`. [#5273](https://github.com/PyO3/pyo3/pull/5273) +- fixed a leaked borrow, when converting a mutable sub class into a frozen base class using `PyRef::into_super` [#5281](https://github.com/PyO3/pyo3/pull/5281) +- Fix FFI definition `Py_Exit` (never returns, was `()` return value, now `!`). [#5317](https://github.com/PyO3/pyo3/pull/5317) +- `experimental-inspect`: fix handling of module members gated behind `#[cfg(...)]` attributes. [#5318](https://github.com/PyO3/pyo3/pull/5318) + +## [0.25.1] - 2025-06-12 +### Packaging + +- Add support for Windows on ARM64. [#5145](https://github.com/PyO3/pyo3/pull/5145) +- Add `chrono-local` feature for optional conversions for chrono's `Local` timezone & `DateTime` instances. [#5174](https://github.com/PyO3/pyo3/pull/5174) + +### Added + +- Add FFI definition `PyBytes_AS_STRING`. [#5121](https://github.com/PyO3/pyo3/pull/5121) +- Add support for module associated consts introspection. [#5150](https://github.com/PyO3/pyo3/pull/5150) + +### Changed + +- Enable "vectorcall" FFI definitions on GraalPy. [#5121](https://github.com/PyO3/pyo3/pull/5121) +- Use `Py_Is` function on GraalPy [#5121](https://github.com/PyO3/pyo3/pull/5121) + +### Fixed + +- Report a better compile error for `async` declarations when not using `experimental-async` feature. [#5156](https://github.com/PyO3/pyo3/pull/5156) +- Fix implementation of `FromPyObject` for `uuid::Uuid` on big-endian architectures. [#5161](https://github.com/PyO3/pyo3/pull/5161) +- Fix segmentation faults on 32-bit x86 with Python 3.14. [#5180](https://github.com/PyO3/pyo3/pull/5180) + +## [0.25.0] - 2025-05-14 + +### Packaging + +- Support Python 3.14.0b1. [#4811](https://github.com/PyO3/pyo3/pull/4811) +- Bump supported GraalPy version to 24.2. [#5116](https://github.com/PyO3/pyo3/pull/5116) +- Add optional `bigdecimal` dependency to add conversions for `bigdecimal::BigDecimal`. [#5011](https://github.com/PyO3/pyo3/pull/5011) +- Add optional `time` dependency to add conversions for `time` types. [#5057](https://github.com/PyO3/pyo3/pull/5057) +- Remove `cfg-if` dependency. [#5110](https://github.com/PyO3/pyo3/pull/5110) +- Add optional `ordered_float` dependency to add conversions for `ordered_float::NotNan` and `ordered_float::OrderedFloat`. [#5114](https://github.com/PyO3/pyo3/pull/5114) + +### Added + +- Add initial type stub generation to the `experimental-inspect` feature. [#3977](https://github.com/PyO3/pyo3/pull/3977) +- Add `#[pyclass(generic)]` option to support runtime generic typing. [#4926](https://github.com/PyO3/pyo3/pull/4926) +- Implement `OnceExt` & `MutexExt` for `parking_lot` & `lock_api`. Use the new extension traits by enabling the `arc_lock`, `lock_api`, or `parking_lot` cargo features. [#5044](https://github.com/PyO3/pyo3/pull/5044) +- Implement `From`/`Into` for `Borrowed` -> `Py`. [#5054](https://github.com/PyO3/pyo3/pull/5054) +- Add `PyTzInfo` constructors. [#5055](https://github.com/PyO3/pyo3/pull/5055) +- Add FFI definition `PY_INVALID_STACK_EFFECT`. [#5064](https://github.com/PyO3/pyo3/pull/5064) +- Implement `AsRef>` for `Py`, `Bound` and `Borrowed`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Add FFI definition `PyModule_Add` and `compat::PyModule_Add`. [#5085](https://github.com/PyO3/pyo3/pull/5085) +- Add FFI definitions `Py_HashBuffer`, `Py_HashPointer`, and `PyObject_GenericHash`. [#5086](https://github.com/PyO3/pyo3/pull/5086) +- Support `#[pymodule_export]` on `const` items in declarative modules. [#5096](https://github.com/PyO3/pyo3/pull/5096) +- Add `#[pyclass(immutable_type)]` option (on Python 3.14+ with `abi3`, or 3.10+ otherwise) for immutable type objects. [#5101](https://github.com/PyO3/pyo3/pull/5101) +- Support `#[pyo3(rename_all)]` support on `#[derive(IntoPyObject)]`. [#5112](https://github.com/PyO3/pyo3/pull/5112) +- Add `PyRange` wrapper. [#5117](https://github.com/PyO3/pyo3/pull/5117) + +### Changed + +- Enable use of `datetime` types with `abi3` feature enabled. [#4970](https://github.com/PyO3/pyo3/pull/4970) +- Deprecate `timezone_utc` in favor of `PyTzInfo::utc`. [#5055](https://github.com/PyO3/pyo3/pull/5055) +- Reduce visibility of some CPython implementation details: [#5064](https://github.com/PyO3/pyo3/pull/5064) + - The FFI definition `PyCodeObject` is now an opaque struct on all Python versions. + - The FFI definition `PyFutureFeatures` is now only defined up until Python 3.10 (it was present in CPython headers but unused in 3.11 and 3.12). +- Change `PyAnyMethods::is` to take `other: &Bound`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Change `Py::is` to take `other: &Py`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Change `PyVisit::call` to take `T: Into>>`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Expose `PyDateTime_DATE_GET_TZINFO` and `PyDateTime_TIME_GET_TZINFO` on PyPy 3.10 and later. [#5079](https://github.com/PyO3/pyo3/pull/5079) +- Add `#[track_caller]` to `with_gil` and `with_gil_unchecked`. [#5109](https://github.com/PyO3/pyo3/pull/5109) +- Use `std::thread::park()` instead of `libc::pause()` or `sleep(9999999)`. [#5115](https://github.com/PyO3/pyo3/pull/5115) + +### Removed + +- Remove all functionality deprecated in PyO3 0.23. [#4982](https://github.com/PyO3/pyo3/pull/4982) +- Remove deprecated `IntoPy` and `ToPyObject` traits. [#5010](https://github.com/PyO3/pyo3/pull/5010) +- Remove private types from `pyo3-ffi` (i.e. starting with `_Py`) which are not referenced by public APIs: `_PyLocalMonitors`, `_Py_GlobalMonitors`, `_PyCoCached`, `_PyCoLineInstrumentationData`, `_PyCoMonitoringData`, `_PyCompilerSrcLocation`, `_PyErr_StackItem`. [#5064](https://github.com/PyO3/pyo3/pull/5064) +- Remove FFI definition `PyCode_GetNumFree` (PyO3 cannot support it due to knowledge of the code object). [#5064](https://github.com/PyO3/pyo3/pull/5064) +- Remove `AsPyPointer` trait. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Remove support for the deprecated string form of `from_py_with`. [#5097](https://github.com/PyO3/pyo3/pull/5097) +- Remove FFI definitions of private static variables: `_PyMethodWrapper_Type`, `_PyCoroWrapper_Type`, `_PyImport_FrozenBootstrap`, `_PyImport_FrozenStdlib`, `_PyImport_FrozenTest`, `_PyManagedBuffer_Type`, `_PySet_Dummy`, `_PyWeakref_ProxyType`, and `_PyWeakref_CallableProxyType`. [#5105](https://github.com/PyO3/pyo3/pull/5105) +- Remove FFI definitions `PyASCIIObjectState`, `PyUnicode_IS_ASCII`, `PyUnicode_IS_COMPACT`, and `PyUnicode_IS_COMPACT_ASCII` on Python 3.14 and newer. [#5133](https://github.com/PyO3/pyo3/pull/5133) + +### Fixed + +- Correctly pick up the shared state for conda-based Python installation when reading information from sysconfigdata. [#5037](https://github.com/PyO3/pyo3/pull/5037) +- Fix compile failure with `#[derive(IntoPyObject, FromPyObject)]` when using `#[pyo3()]` options recognised by only one of the two derives. [#5070](https://github.com/PyO3/pyo3/pull/5070) +- Fix various compile errors from missing FFI definitions using certain feature combinations on PyPy and GraalPy. [#5091](https://github.com/PyO3/pyo3/pull/5091) +- Fallback on `backports.zoneinfo` for python <3.9 when converting timezones into python. [#5120](https://github.com/PyO3/pyo3/pull/5120) + +## [0.24.2] - 2025-04-21 + +### Fixed + +- Fix `unused_imports` lint of `#[pyfunction]` and `#[pymethods]` expanded in `macro_rules` context. [#5030](https://github.com/PyO3/pyo3/pull/5030) +- Fix size of `PyCodeObject::_co_instrumentation_version` ffi struct member on Python 3.13 for systems where `uintptr_t` is not 64 bits. [#5048](https://github.com/PyO3/pyo3/pull/5048) +- Fix struct-type complex enum variant fields incorrectly exposing raw identifiers as `r#ident` in Python bindings. [#5050](https://github.com/PyO3/pyo3/pull/5050) + +## [0.24.1] - 2025-03-31 + +### Added + +- Add `abi3-py313` feature. [#4969](https://github.com/PyO3/pyo3/pull/4969) +- Add `PyAnyMethods::getattr_opt`. [#4978](https://github.com/PyO3/pyo3/pull/4978) +- Add `PyInt::new` constructor for all supported number types (i32, u32, i64, u64, isize, usize). [#4984](https://github.com/PyO3/pyo3/pull/4984) +- Add `pyo3::sync::with_critical_section2`. [#4992](https://github.com/PyO3/pyo3/pull/4992) +- Implement `PyCallArgs` for `Borrowed<'_, 'py, PyTuple>`, `&Bound<'py, PyTuple>`, and `&Py`. [#5013](https://github.com/PyO3/pyo3/pull/5013) + +### Fixed + +- Fix `is_type_of` for native types not using same specialized check as `is_type_of_bound`. [#4981](https://github.com/PyO3/pyo3/pull/4981) +- Fix `Probe` class naming issue with `#[pymethods]`. [#4988](https://github.com/PyO3/pyo3/pull/4988) +- Fix compile failure with required `#[pyfunction]` arguments taking `Option<&str>` and `Option<&T>` (for `#[pyclass]` types). [#5002](https://github.com/PyO3/pyo3/pull/5002) +- Fix `PyString::from_object` causing of bounds reads with `encoding` and `errors` parameters which are not nul-terminated. [#5008](https://github.com/PyO3/pyo3/pull/5008) +- Fix compile error when additional options follow after `crate` for `#[pyfunction]`. [#5015](https://github.com/PyO3/pyo3/pull/5015) + +## [0.24.0] - 2025-03-09 + +### Packaging + +- Add supported CPython/PyPy versions to cargo package metadata. [#4756](https://github.com/PyO3/pyo3/pull/4756) +- Bump `target-lexicon` dependency to 0.13. [#4822](https://github.com/PyO3/pyo3/pull/4822) +- Add optional `jiff` dependency to add conversions for `jiff` datetime types. [#4823](https://github.com/PyO3/pyo3/pull/4823) +- Add optional `uuid` dependency to add conversions for `uuid::Uuid`. [#4864](https://github.com/PyO3/pyo3/pull/4864) +- Bump minimum supported `inventory` version to 0.3.5. [#4954](https://github.com/PyO3/pyo3/pull/4954) + +### Added + +- Add `PyIterator::send` method to allow sending values into a python generator. [#4746](https://github.com/PyO3/pyo3/pull/4746) +- Add `PyCallArgs` trait for passing arguments into the Python calling protocol. This enabled using a faster calling convention for certain types, improving performance. [#4768](https://github.com/PyO3/pyo3/pull/4768) +- Add `#[pyo3(default = ...']` option for `#[derive(FromPyObject)]` to set a default value for extracted fields of named structs. [#4829](https://github.com/PyO3/pyo3/pull/4829) +- Add `#[pyo3(into_py_with = ...)]` option for `#[derive(IntoPyObject, IntoPyObjectRef)]`. [#4850](https://github.com/PyO3/pyo3/pull/4850) +- Add FFI definitions `PyThreadState_GetFrame` and `PyFrame_GetBack`. [#4866](https://github.com/PyO3/pyo3/pull/4866) +- Optimize `last` for `BoundListIterator`, `BoundTupleIterator` and `BorrowedTupleIterator`. [#4878](https://github.com/PyO3/pyo3/pull/4878) +- Optimize `Iterator::count()` for `PyDict`, `PyList`, `PyTuple` & `PySet`. [#4878](https://github.com/PyO3/pyo3/pull/4878) +- Optimize `nth`, `nth_back`, `advance_by` and `advance_back_by` for `BoundTupleIterator` [#4897](https://github.com/PyO3/pyo3/pull/4897) +- Add support for `types.GenericAlias` as `pyo3::types::PyGenericAlias`. [#4917](https://github.com/PyO3/pyo3/pull/4917) +- Add `MutextExt` trait to help avoid deadlocks with the GIL while locking a `std::sync::Mutex`. [#4934](https://github.com/PyO3/pyo3/pull/4934) +- Add `#[pyo3(rename_all = "...")]` option for `#[derive(FromPyObject)]`. [#4941](https://github.com/PyO3/pyo3/pull/4941) + +### Changed + +- Optimize `nth`, `nth_back`, `advance_by` and `advance_back_by` for `BoundListIterator`. [#4810](https://github.com/PyO3/pyo3/pull/4810) +- Use `DerefToPyAny` in blanket implementations of `From>` and `From>` for `PyObject`. [#4593](https://github.com/PyO3/pyo3/pull/4593) +- Map `io::ErrorKind::IsADirectory`/`NotADirectory` to the corresponding Python exception on Rust 1.83+. [#4747](https://github.com/PyO3/pyo3/pull/4747) +- `PyAnyMethods::call` and friends now require `PyCallArgs` for their positional arguments. [#4768](https://github.com/PyO3/pyo3/pull/4768) +- Expose FFI definitions for `PyObject_Vectorcall(Method)` on the stable abi on 3.12+. [#4853](https://github.com/PyO3/pyo3/pull/4853) +- `#[pyo3(from_py_with = ...)]` now take a path rather than a string literal [#4860](https://github.com/PyO3/pyo3/pull/4860) +- Format Python traceback in impl Debug for PyErr. [#4900](https://github.com/PyO3/pyo3/pull/4900) +- Convert `PathBuf` & `Path` into Python `pathlib.Path` instead of `PyString`. [#4925](https://github.com/PyO3/pyo3/pull/4925) +- Relax parsing of exotic Python versions. [#4949](https://github.com/PyO3/pyo3/pull/4949) +- PyO3 threads now hang instead of `pthread_exit` trying to acquire the GIL when the interpreter is shutting down. This mimics the [Python 3.14](https://github.com/python/cpython/issues/87135) behavior and avoids undefined behavior and crashes. [#4874](https://github.com/PyO3/pyo3/pull/4874) + +### Removed + +- Remove implementations of `Deref` for `PyAny` and other "native" types. [#4593](https://github.com/PyO3/pyo3/pull/4593) +- Remove implicit default of trailing optional arguments (see #2935) [#4729](https://github.com/PyO3/pyo3/pull/4729) +- Remove the deprecated implicit eq fallback for simple enums. [#4730](https://github.com/PyO3/pyo3/pull/4730) + +### Fixed + +- Correct FFI definition of `PyIter_Send` to return a `PySendResult`. [#4746](https://github.com/PyO3/pyo3/pull/4746) +- Fix a thread safety issue in the runtime borrow checker used by mutable pyclass instances on the free-threaded build. [#4948](https://github.com/PyO3/pyo3/pull/4948) + + +## [0.23.5] - 2025-02-22 + +### Packaging + +- Add support for PyPy3.11 [#4760](https://github.com/PyO3/pyo3/pull/4760) + +### Fixed + +- Fix thread-unsafe implementation of freelist pyclasses on the free-threaded build. [#4902](https://github.com/PyO3/pyo3/pull/4902) +- Re-enable a workaround for situations where CPython incorrectly does not add `__builtins__` to `__globals__` in code executed by `Python::py_run` (was removed in PyO3 0.23.0). [#4921](https://github.com/PyO3/pyo3/pull/4921) + +## [0.23.4] - 2025-01-10 + +### Added + +- Add `PyList::locked_for_each`, which uses a critical section to lock the list on the free-threaded build. [#4789](https://github.com/PyO3/pyo3/pull/4789) +- Add `pyo3_build_config::add_python_framework_link_args` build script API to set rpath when using macOS system Python. [#4833](https://github.com/PyO3/pyo3/pull/4833) + +### Changed + +- Use `datetime.fold` to distinguish ambiguous datetimes when converting to and from `chrono::DateTime` (rather than erroring). [#4791](https://github.com/PyO3/pyo3/pull/4791) +- Optimize PyList iteration on the free-threaded build. [#4789](https://github.com/PyO3/pyo3/pull/4789) + +### Fixed + +- Fix unnecessary internal `py.allow_threads` GIL-switch when attempting to access contents of a `PyErr` which originated from Python (could lead to unintended deadlocks). [#4766](https://github.com/PyO3/pyo3/pull/4766) +- Fix thread-unsafe access of dict internals in `BoundDictIterator` on the free-threaded build. [#4788](https://github.com/PyO3/pyo3/pull/4788) +* Fix unnecessary critical sections in `BoundDictIterator` on the free-threaded build. [#4788](https://github.com/PyO3/pyo3/pull/4788) +- Fix time-of-check to time-of-use issues with list iteration on the free-threaded build. [#4789](https://github.com/PyO3/pyo3/pull/4789) +- Fix `chrono::DateTime` to-Python conversion when `Tz` is `chrono_tz::Tz`. [#4790](https://github.com/PyO3/pyo3/pull/4790) +- Fix `#[pyclass]` not being able to be named `Probe`. [#4794](https://github.com/PyO3/pyo3/pull/4794) +- Fix not treating cross-compilation from x64 to aarch64 on Windows as a cross-compile. [#4800](https://github.com/PyO3/pyo3/pull/4800) +- Fix missing struct fields on GraalPy when subclassing builtin classes. [#4802](https://github.com/PyO3/pyo3/pull/4802) +- Fix generating import lib for PyPy when `abi3` feature is enabled. [#4806](https://github.com/PyO3/pyo3/pull/4806) +- Fix generating import lib for python3.13t when `abi3` feature is enabled. [#4808](https://github.com/PyO3/pyo3/pull/4808) +- Fix compile failure for raw identifiers like `r#box` in `derive(FromPyObject)`. [#4814](https://github.com/PyO3/pyo3/pull/4814) +- Fix compile failure for `#[pyclass]` enum variants with more than 12 fields. [#4832](https://github.com/PyO3/pyo3/pull/4832) + + +## [0.23.3] - 2024-12-03 + +### Packaging + +- Bump optional `python3-dll-a` dependency to 0.2.11. [#4749](https://github.com/PyO3/pyo3/pull/4749) + +### Fixed + +- Fix unresolved symbol link failures on Windows when compiling for Python 3.13t with `abi3` features enabled. [#4733](https://github.com/PyO3/pyo3/pull/4733) +- Fix unresolved symbol link failures on Windows when compiling for Python 3.13t using the `generate-import-lib` feature. [#4749](https://github.com/PyO3/pyo3/pull/4749) +- Fix compile-time regression in PyO3 0.23.0 where changing `PYO3_CONFIG_FILE` would not reconfigure PyO3 for the new interpreter. [#4758](https://github.com/PyO3/pyo3/pull/4758) + +## [0.23.2] - 2024-11-25 + +### Added + +- Add `IntoPyObjectExt` trait. [#4708](https://github.com/PyO3/pyo3/pull/4708) + +### Fixed + +- Fix compile failures when building for free-threaded Python when the `abi3` or `abi3-pyxx` features are enabled. [#4719](https://github.com/PyO3/pyo3/pull/4719) +- Fix `ambiguous_associated_items` lint error in `#[pyclass]` and `#[derive(IntoPyObject)]` macros. [#4725](https://github.com/PyO3/pyo3/pull/4725) + + +## [0.23.1] - 2024-11-16 + +Re-release of 0.23.0 with fixes to docs.rs build. + +## [0.23.0] - 2024-11-15 + +### Packaging + +- Drop support for PyPy 3.7 and 3.8. [#4582](https://github.com/PyO3/pyo3/pull/4582) +- Extend range of supported versions of `hashbrown` optional dependency to include version 0.15. [#4604](https://github.com/PyO3/pyo3/pull/4604) +- Bump minimum version of `eyre` optional dependency to 0.6.8. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Bump minimum version of `hashbrown` optional dependency to 0.14.5. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Bump minimum version of `indexmap` optional dependency to 2.5.0. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Bump minimum version of `num-complex` optional dependency to 0.4.6. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Bump minimum version of `chrono-tz` optional dependency to 0.10. [#4617](https://github.com/PyO3/pyo3/pull/4617) +- Support free-threaded Python 3.13t. [#4588](https://github.com/PyO3/pyo3/pull/4588) + +### Added + +- Add `IntoPyObject` (fallible) conversion trait to convert from Rust to Python values. [#4060](https://github.com/PyO3/pyo3/pull/4060) +- Add `#[pyclass(str="")]` option to generate `__str__` based on a `Display` implementation or format string. [#4233](https://github.com/PyO3/pyo3/pull/4233) +- Implement `PartialEq` for `Bound<'py, PyInt>` with `u8`, `u16`, `u32`, `u64`, `u128`, `usize`, `i8`, `i16`, `i32`, `i64`, `i128` and `isize`. [#4317](https://github.com/PyO3/pyo3/pull/4317) +- Implement `PartialEq` and `PartialEq` for `Bound<'py, PyFloat>`. [#4348](https://github.com/PyO3/pyo3/pull/4348) +- Add `as_super` and `into_super` methods for `Bound`. [#4351](https://github.com/PyO3/pyo3/pull/4351) +- Add FFI definitions `PyCFunctionFast` and `PyCFunctionFastWithKeywords` [#4415](https://github.com/PyO3/pyo3/pull/4415) +- Add FFI definitions for `PyMutex` on Python 3.13 and newer. [#4421](https://github.com/PyO3/pyo3/pull/4421) +- Add `PyDict::locked_for_each` to iterate efficiently on freethreaded Python. [#4439](https://github.com/PyO3/pyo3/pull/4439) +- Add FFI definitions `PyObject_GetOptionalAttr`, `PyObject_GetOptionalAttrString`, `PyObject_HasAttrWithError`, `PyObject_HasAttrStringWithError`, `Py_CONSTANT_*` constants, `Py_GetConstant`, `Py_GetConstantBorrowed`, and `PyType_GetModuleByDef` on Python 3.13 and newer. [#4447](https://github.com/PyO3/pyo3/pull/4447) +- Add FFI definitions for the Python critical section API available on Python 3.13 and newer. [#4477](https://github.com/PyO3/pyo3/pull/4477) +- Add derive macro for `IntoPyObject`. [#4495](https://github.com/PyO3/pyo3/pull/4495) +- Add `Borrowed::as_ptr`. [#4520](https://github.com/PyO3/pyo3/pull/4520) +- Add FFI definition for `PyImport_AddModuleRef`. [#4529](https://github.com/PyO3/pyo3/pull/4529) +- Add `PyAnyMethods::try_iter`. [#4553](https://github.com/PyO3/pyo3/pull/4553) +- Add `pyo3::sync::with_critical_section`, a wrapper around the Python Critical Section API added in Python 3.13. [#4587](https://github.com/PyO3/pyo3/pull/4587) +- Add `#[pymodule(gil_used = false)]` option to declare that a module supports the free-threaded build. [#4588](https://github.com/PyO3/pyo3/pull/4588) +- Add `PyModule::gil_used` method to declare that a module supports the free-threaded build. [#4588](https://github.com/PyO3/pyo3/pull/4588) +- Add FFI definition `PyDateTime_CAPSULE_NAME`. [#4634](https://github.com/PyO3/pyo3/pull/4634) +- Add `PyMappingProxy` type to represent the `mappingproxy` Python class. [#4644](https://github.com/PyO3/pyo3/pull/4644) +- Add FFI definitions `PyList_Extend` and `PyList_Clear`. [#4667](https://github.com/PyO3/pyo3/pull/4667) +- Add derive macro for `IntoPyObjectRef`. [#4674](https://github.com/PyO3/pyo3/pull/4674) +- Add `pyo3::sync::OnceExt` and `pyo3::sync::OnceLockExt` traits. [#4676](https://github.com/PyO3/pyo3/pull/4676) + +### Changed + +- Prefer `IntoPyObject` over `IntoPy>>` for `#[pyfunction]` and `#[pymethods]` return types. [#4060](https://github.com/PyO3/pyo3/pull/4060) +- Report multiple errors from `#[pyclass]` and `#[pyo3(..)]` attributes. [#4243](https://github.com/PyO3/pyo3/pull/4243) +- Nested declarative `#[pymodule]` are automatically treated as submodules (no `PyInit_` entrypoint is created). [#4308](https://github.com/PyO3/pyo3/pull/4308) +- Deprecate `PyAnyMethods::is_ellipsis` (`Py::is_ellipsis` was deprecated in PyO3 0.20). [#4322](https://github.com/PyO3/pyo3/pull/4322) +- Deprecate `PyLong` in favor of `PyInt`. [#4347](https://github.com/PyO3/pyo3/pull/4347) +- Rename `IntoPyDict::into_py_dict_bound` to `IntoPyDict::into_py_dict`. [#4388](https://github.com/PyO3/pyo3/pull/4388) +- `PyModule::from_code` now expects `&CStr` as arguments instead of `&str`. [#4404](https://github.com/PyO3/pyo3/pull/4404) +- Use "fastcall" Python calling convention for `#[pyfunction]`s when compiling on abi3 for Python 3.10 and up. [#4415](https://github.com/PyO3/pyo3/pull/4415) +- Remove `Copy` and `Clone` from `PyObject` struct FFI definition. [#4434](https://github.com/PyO3/pyo3/pull/4434) +- `Python::eval` and `Python::run` now take a `&CStr` instead of `&str`. [#4435](https://github.com/PyO3/pyo3/pull/4435) +- Deprecate `IPowModulo`, `PyClassAttributeDef`, `PyGetterDef`, `PyMethodDef`, `PyMethodDefType`, and `PySetterDef` from PyO3's public API. [#4441](https://github.com/PyO3/pyo3/pull/4441) +- `IntoPyObject` impls for `Vec`, `&[u8]`, `[u8; N]`, `Cow<[u8]>` and `SmallVec<[u8; N]>` now convert into Python `bytes` rather than a `list` of integers. [#4442](https://github.com/PyO3/pyo3/pull/4442) +- Emit a compile-time error when attempting to subclass a class that doesn't allow subclassing. [#4453](https://github.com/PyO3/pyo3/pull/4453) +- `IntoPyDict::into_py_dict` is now fallible due to `IntoPyObject` migration. [#4493](https://github.com/PyO3/pyo3/pull/4493) +- The `abi3` feature will now override config files provided via `PYO3_BUILD_CONFIG`. [#4497](https://github.com/PyO3/pyo3/pull/4497) +- Disable the `GILProtected` struct on free-threaded Python. [#4504](https://github.com/PyO3/pyo3/pull/4504) +- Updated FFI definitions for functions and struct fields that have been deprecated or removed from CPython. [#4534](https://github.com/PyO3/pyo3/pull/4534) +- Disable `PyListMethods::get_item_unchecked` on free-threaded Python. [#4539](https://github.com/PyO3/pyo3/pull/4539) +- Add `GILOnceCell::import`. [#4542](https://github.com/PyO3/pyo3/pull/4542) +- Deprecate `PyAnyMethods::iter` in favour of `PyAnyMethods::try_iter`. [#4553](https://github.com/PyO3/pyo3/pull/4553) +- The `#[pyclass]` macro now requires a types to be `Sync`. (Except for `#[pyclass(unsendable)]` types). [#4566](https://github.com/PyO3/pyo3/pull/4566) +- `PyList::new` and `PyTuple::new` are now fallible due to `IntoPyObject` migration. [#4580](https://github.com/PyO3/pyo3/pull/4580) +- `PyErr::matches` is now fallible due to `IntoPyObject` migration. [#4595](https://github.com/PyO3/pyo3/pull/4595) +- Deprecate `ToPyObject` in favour of `IntoPyObject` [#4595](https://github.com/PyO3/pyo3/pull/4595) +- Deprecate `PyWeakrefMethods::get_option`. [#4597](https://github.com/PyO3/pyo3/pull/4597) +- Seal `PyWeakrefMethods` trait. [#4598](https://github.com/PyO3/pyo3/pull/4598) +- Remove `PyNativeTypeInitializer` and `PyObjectInit` from the PyO3 public API. [#4611](https://github.com/PyO3/pyo3/pull/4611) +- Deprecate `IntoPy` in favor of `IntoPyObject` [#4618](https://github.com/PyO3/pyo3/pull/4618) +- Eagerly normalize exceptions in `PyErr::take()` and `PyErr::fetch()` on Python 3.11 and older. [#4655](https://github.com/PyO3/pyo3/pull/4655) +- Move `IntoPy::type_output` to `IntoPyObject::type_output`. [#4657](https://github.com/PyO3/pyo3/pull/4657) +- Change return type of `PyMapping::keys`, `PyMapping::values` and `PyMapping::items` to `Bound<'py, PyList>` instead of `Bound<'py, PySequence>`. [#4661](https://github.com/PyO3/pyo3/pull/4661) +- Complex enums now allow field types that either implement `IntoPyObject` by reference or by value together with `Clone`. This makes `Py` available as field type. [#4694](https://github.com/PyO3/pyo3/pull/4694) + + +### Removed + +- Remove all functionality deprecated in PyO3 0.20. [#4322](https://github.com/PyO3/pyo3/pull/4322) +- Remove all functionality deprecated in PyO3 0.21. [#4323](https://github.com/PyO3/pyo3/pull/4323) +- Deprecate `PyUnicode` in favour of `PyString`. [#4370](https://github.com/PyO3/pyo3/pull/4370) +- Remove deprecated `gil-refs` feature. [#4378](https://github.com/PyO3/pyo3/pull/4378) +- Remove private FFI definitions `_Py_IMMORTAL_REFCNT`, `_Py_IsImmortal`, `_Py_TPFLAGS_STATIC_BUILTIN`, `_Py_Dealloc`, `_Py_IncRef`, `_Py_DecRef`. [#4447](https://github.com/PyO3/pyo3/pull/4447) +- Remove private FFI definitions `_Py_c_sum`, `_Py_c_diff`, `_Py_c_neg`, `_Py_c_prod`, `_Py_c_quot`, `_Py_c_pow`, `_Py_c_abs`. [#4521](https://github.com/PyO3/pyo3/pull/4521) +- Remove `_borrowed` methods of `PyWeakRef` and `PyWeakRefProxy`. [#4528](https://github.com/PyO3/pyo3/pull/4528) +- Removed private FFI definition `_PyErr_ChainExceptions`. [#4534](https://github.com/PyO3/pyo3/pull/4534) + +### Fixed + +- Fix invalid library search path `lib_dir` when cross-compiling. [#4389](https://github.com/PyO3/pyo3/pull/4389) +- Fix FFI definition `Py_Is` for PyPy on 3.10 to call the function defined by PyPy. [#4447](https://github.com/PyO3/pyo3/pull/4447) +- Fix compile failure when using `#[cfg]` attributes for simple enum variants. [#4509](https://github.com/PyO3/pyo3/pull/4509) +- Fix compiler warning for `non_snake_case` method names inside `#[pymethods]` generated code. [#4567](https://github.com/PyO3/pyo3/pull/4567) +- Fix compile error with `#[derive(FromPyObject)]` generic struct with trait bounds. [#4645](https://github.com/PyO3/pyo3/pull/4645) +- Fix compile error for `#[classmethod]` and `#[staticmethod]` on magic methods. [#4654](https://github.com/PyO3/pyo3/pull/4654) +- Fix compile warning for `unsafe_op_in_unsafe_fn` in generated macro code. [#4674](https://github.com/PyO3/pyo3/pull/4674) +- Fix incorrect deprecation warning for `#[pyclass] enum`s with custom `__eq__` implementation. [#4692](https://github.com/PyO3/pyo3/pull/4692) +- Fix `non_upper_case_globals` lint firing for generated `__match_args__` on complex enums. [#4705](https://github.com/PyO3/pyo3/pull/4705) + +## [0.22.5] - 2024-10-15 + +### Fixed + +- Fix regression in 0.22.4 of naming collision in `__clear__` slot and `clear` method generated code. [#4619](https://github.com/PyO3/pyo3/pull/4619) + + +## [0.22.4] - 2024-10-12 + +### Added + +- Add FFI definition `PyWeakref_GetRef` and `compat::PyWeakref_GetRef`. [#4528](https://github.com/PyO3/pyo3/pull/4528) + +### Changed + +- Deprecate `_borrowed` methods on `PyWeakRef` and `PyWeakrefProxy` (just use the owning forms). [#4590](https://github.com/PyO3/pyo3/pull/4590) + +### Fixed + +- Revert removal of private FFI function `_PyLong_NumBits` on Python 3.13 and later. [#4450](https://github.com/PyO3/pyo3/pull/4450) +- Fix `__traverse__` functions for base classes not being called by subclasses created with `#[pyclass(extends = ...)]`. [#4563](https://github.com/PyO3/pyo3/pull/4563) +- Fix regression in 0.22.3 failing compiles under `#![forbid(unsafe_code)]`. [#4574](https://github.com/PyO3/pyo3/pull/4574) +- Fix `create_exception` macro triggering lint and compile errors due to interaction with `gil-refs` feature. [#4589](https://github.com/PyO3/pyo3/pull/4589) +- Workaround possible use-after-free in `_borrowed` methods on `PyWeakRef` and `PyWeakrefProxy` by leaking their contents. [#4590](https://github.com/PyO3/pyo3/pull/4590) +- Fix crash calling `PyType_GetSlot` on static types before Python 3.10. [#4599](https://github.com/PyO3/pyo3/pull/4599) + + +## [0.22.3] - 2024-09-15 + +### Added + +- Add `pyo3::ffi::compat` namespace with compatibility shims for C API functions added in recent versions of Python. +- Add FFI definition `PyDict_GetItemRef` on Python 3.13 and newer, and `compat::PyDict_GetItemRef` for all versions. [#4355](https://github.com/PyO3/pyo3/pull/4355) +- Add FFI definition `PyList_GetItemRef` on Python 3.13 and newer, and `pyo3_ffi::compat::PyList_GetItemRef` for all versions. [#4410](https://github.com/PyO3/pyo3/pull/4410) +- Add FFI definitions `compat::Py_NewRef` and `compat::Py_XNewRef`. [#4445](https://github.com/PyO3/pyo3/pull/4445) +- Add FFI definitions `compat::PyObject_CallNoArgs` and `compat::PyObject_CallMethodNoArgs`. [#4461](https://github.com/PyO3/pyo3/pull/4461) +- Add `GilOnceCell>::clone_ref`. [#4511](https://github.com/PyO3/pyo3/pull/4511) + +### Changed + +- Improve error messages for `#[pyfunction]` defined inside `#[pymethods]`. [#4349](https://github.com/PyO3/pyo3/pull/4349) +- Improve performance of calls to Python by using the vectorcall calling convention where possible. [#4456](https://github.com/PyO3/pyo3/pull/4456) +- Mention the type name in the exception message when trying to instantiate a class with no constructor defined. [#4481](https://github.com/PyO3/pyo3/pull/4481) + +### Removed + +- Remove private FFI definition `_Py_PackageContext`. [#4420](https://github.com/PyO3/pyo3/pull/4420) + +### Fixed + +- Fix compile failure in declarative `#[pymodule]` under presence of `#![no_implicit_prelude]`. [#4328](https://github.com/PyO3/pyo3/pull/4328) +- Fix use of borrowed reference in `PyDict::get_item` (unsafe in free-threaded Python). [#4355](https://github.com/PyO3/pyo3/pull/4355) +- Fix `#[pyclass(eq)]` macro hygiene issues for structs and enums. [#4359](https://github.com/PyO3/pyo3/pull/4359) +- Fix hygiene/span issues of `#[pyfunction]` and `#[pymethods]` generated code which affected expansion in `macro_rules` context. [#4382](https://github.com/PyO3/pyo3/pull/4382) +- Fix `unsafe_code` lint error in `#[pyclass]` generated code. [#4396](https://github.com/PyO3/pyo3/pull/4396) +- Fix async functions returning a tuple only returning the first element to Python. [#4407](https://github.com/PyO3/pyo3/pull/4407) +- Fix use of borrowed reference in `PyList::get_item` (unsafe in free-threaded Python). [#4410](https://github.com/PyO3/pyo3/pull/4410) +- Correct FFI definition `PyArg_ParseTupleAndKeywords` to take `*const *const c_char` instead of `*mut *mut c_char` on Python 3.13 and up. [#4420](https://github.com/PyO3/pyo3/pull/4420) +- Fix a soundness bug with `PyClassInitializer`: panic if adding subclass to existing instance via `PyClassInitializer::from(Py).add_subclass(SubClass)`. [#4454](https://github.com/PyO3/pyo3/pull/4454) +- Fix illegal reference counting op inside implementation of `__traverse__` handlers. [#4479](https://github.com/PyO3/pyo3/pull/4479) + +## [0.22.2] - 2024-07-17 + +### Packaging + +- Require opt-in to freethreaded Python using the `UNSAFE_PYO3_BUILD_FREE_THREADED=1` environment variable (it is not yet supported by PyO3). [#4327](https://github.com/PyO3/pyo3/pull/4327) + +### Changed + +- Use FFI function calls for reference counting on all abi3 versions. [#4324](https://github.com/PyO3/pyo3/pull/4324) +- `#[pymodule(...)]` now directly accepts all relevant `#[pyo3(...)]` options. [#4330](https://github.com/PyO3/pyo3/pull/4330) + +### Fixed + +- Fix compile failure in declarative `#[pymodule]` under presence of `#![no_implicit_prelude]`. [#4328](https://github.com/PyO3/pyo3/pull/4328) +- Fix compile failure due to c-string literals on Rust < 1.79. [#4353](https://github.com/PyO3/pyo3/pull/4353) + +## [0.22.1] - 2024-07-06 + +### Added + +- Add `#[pyo3(submodule)]` option for declarative `#[pymodule]`s. [#4301](https://github.com/PyO3/pyo3/pull/4301) +- Implement `PartialEq` for `Bound<'py, PyBool>`. [#4305](https://github.com/PyO3/pyo3/pull/4305) + +### Fixed + +- Return `NotImplemented` instead of raising `TypeError` from generated equality method when comparing different types. [#4287](https://github.com/PyO3/pyo3/pull/4287) +- Handle full-path `#[pyo3::prelude::pymodule]` and similar for `#[pyclass]` and `#[pyfunction]` in declarative modules. [#4288](https://github.com/PyO3/pyo3/pull/4288) +- Fix 128-bit int regression on big-endian platforms with Python <3.13. [#4291](https://github.com/PyO3/pyo3/pull/4291) +- Stop generating code that will never be covered with declarative modules. [#4297](https://github.com/PyO3/pyo3/pull/4297) +- Fix invalid deprecation warning for trailing optional on `#[setter]` function. [#4304](https://github.com/PyO3/pyo3/pull/4304) + +## [0.22.0] - 2024-06-24 + +### Packaging + +- Update `heck` dependency to 0.5. [#3966](https://github.com/PyO3/pyo3/pull/3966) +- Extend range of supported versions of `chrono-tz` optional dependency to include version 0.10. [#4061](https://github.com/PyO3/pyo3/pull/4061) +- Update MSRV to 1.63. [#4129](https://github.com/PyO3/pyo3/pull/4129) +- Add optional `num-rational` feature to add conversions with Python's `fractions.Fraction`. [#4148](https://github.com/PyO3/pyo3/pull/4148) +- Support Python 3.13. [#4184](https://github.com/PyO3/pyo3/pull/4184) + +### Added + +- Add `PyWeakref`, `PyWeakrefReference` and `PyWeakrefProxy`. [#3835](https://github.com/PyO3/pyo3/pull/3835) +- Support `#[pyclass]` on enums that have tuple variants. [#4072](https://github.com/PyO3/pyo3/pull/4072) +- Add support for scientific notation in `Decimal` conversion. [#4079](https://github.com/PyO3/pyo3/pull/4079) +- Add `pyo3_disable_reference_pool` conditional compilation flag to avoid the overhead of the global reference pool at the cost of known limitations as explained in the performance section of the guide. [#4095](https://github.com/PyO3/pyo3/pull/4095) +- Add `#[pyo3(constructor = (...))]` to customize the generated constructors for complex enum variants. [#4158](https://github.com/PyO3/pyo3/pull/4158) +- Add `PyType::module`, which always matches Python `__module__`. [#4196](https://github.com/PyO3/pyo3/pull/4196) +- Add `PyType::fully_qualified_name` which matches the "fully qualified name" defined in [PEP 737](https://peps.python.org/pep-0737). [#4196](https://github.com/PyO3/pyo3/pull/4196) +- Add `PyTypeMethods::mro` and `PyTypeMethods::bases`. [#4197](https://github.com/PyO3/pyo3/pull/4197) +- Add `#[pyclass(ord)]` to implement ordering based on `PartialOrd`. [#4202](https://github.com/PyO3/pyo3/pull/4202) +- Implement `ToPyObject` and `IntoPy` for `PyBackedStr` and `PyBackedBytes`. [#4205](https://github.com/PyO3/pyo3/pull/4205) +- Add `#[pyclass(hash)]` option to implement `__hash__` in terms of the `Hash` implementation [#4206](https://github.com/PyO3/pyo3/pull/4206) +- Add `#[pyclass(eq)]` option to generate `__eq__` based on `PartialEq`, and `#[pyclass(eq_int)]` for simple enums to implement equality based on their discriminants. [#4210](https://github.com/PyO3/pyo3/pull/4210) +- Implement `From>` for `PyClassInitializer`. [#4214](https://github.com/PyO3/pyo3/pull/4214) +- Add `as_super` methods to `PyRef` and `PyRefMut` for accessing the base class by reference. [#4219](https://github.com/PyO3/pyo3/pull/4219) +- Implement `PartialEq` for `Bound<'py, PyString>`. [#4245](https://github.com/PyO3/pyo3/pull/4245) +- Implement `PyModuleMethods::filename` on PyPy. [#4249](https://github.com/PyO3/pyo3/pull/4249) +- Implement `PartialEq<[u8]>` for `Bound<'py, PyBytes>`. [#4250](https://github.com/PyO3/pyo3/pull/4250) +- Add `pyo3_ffi::c_str` macro to create `&'static CStr` on Rust versions which don't have 1.77's `c""` literals. [#4255](https://github.com/PyO3/pyo3/pull/4255) +- Support `bool` conversion with `numpy` 2.0's `numpy.bool` type [#4258](https://github.com/PyO3/pyo3/pull/4258) +- Add `PyAnyMethods::{bitnot, matmul, floor_div, rem, divmod}`. [#4264](https://github.com/PyO3/pyo3/pull/4264) + +### Changed + +- Change the type of `PySliceIndices::slicelength` and the `length` parameter of `PySlice::indices()`. [#3761](https://github.com/PyO3/pyo3/pull/3761) +- Deprecate implicit default for trailing optional arguments [#4078](https://github.com/PyO3/pyo3/pull/4078) +- `Clone`ing pointers into the Python heap has been moved behind the `py-clone` feature, as it must panic without the GIL being held as a soundness fix. [#4095](https://github.com/PyO3/pyo3/pull/4095) +- Add `#[track_caller]` to all `Py`, `Bound<'py, T>` and `Borrowed<'a, 'py, T>` methods which can panic. [#4098](https://github.com/PyO3/pyo3/pull/4098) +- Change `PyAnyMethods::dir` to be fallible and return `PyResult>` (and similar for `PyAny::dir`). [#4100](https://github.com/PyO3/pyo3/pull/4100) +- The global reference pool (to track pending reference count decrements) is now initialized lazily to avoid the overhead of taking a mutex upon function entry when the functionality is not actually used. [#4178](https://github.com/PyO3/pyo3/pull/4178) +- Emit error messages when using `weakref` or `dict` when compiling for `abi3` for Python older than 3.9. [#4194](https://github.com/PyO3/pyo3/pull/4194) +- Change `PyType::name` to always match Python `__name__`. [#4196](https://github.com/PyO3/pyo3/pull/4196) +- Remove CPython internal ffi call for complex number including: add, sub, mul, div, neg, abs, pow. Added PyAnyMethods::{abs, pos, neg} [#4201](https://github.com/PyO3/pyo3/pull/4201) +- Deprecate implicit integer comparison for simple enums in favor of `#[pyclass(eq_int)]`. [#4210](https://github.com/PyO3/pyo3/pull/4210) +- Set the `module=` attribute of declarative modules' child `#[pymodule]`s and `#[pyclass]`es. [#4213](https://github.com/PyO3/pyo3/pull/4213) +- Set the `module` option for complex enum variants from the value set on the complex enum `module`. [#4228](https://github.com/PyO3/pyo3/pull/4228) +- Respect the Python "limited API" when building for the `abi3` feature on PyPy or GraalPy. [#4237](https://github.com/PyO3/pyo3/pull/4237) +- Optimize code generated by `#[pyo3(get)]` on `#[pyclass]` fields. [#4254](https://github.com/PyO3/pyo3/pull/4254) +- `PyCFunction::new`, `PyCFunction::new_with_keywords` and `PyCFunction::new_closure` now take `&'static CStr` name and doc arguments (previously was `&'static str`). [#4255](https://github.com/PyO3/pyo3/pull/4255) +- The `experimental-declarative-modules` feature is now stabilized and available by default. [#4257](https://github.com/PyO3/pyo3/pull/4257) + +### Fixed + +- Fix panic when `PYO3_CROSS_LIB_DIR` is set to a missing path. [#4043](https://github.com/PyO3/pyo3/pull/4043) +- Fix a compile error when exporting an exception created with `create_exception!` living in a different Rust module using the `declarative-module` feature. [#4086](https://github.com/PyO3/pyo3/pull/4086) +- Fix FFI definitions of `PY_VECTORCALL_ARGUMENTS_OFFSET` and `PyVectorcall_NARGS` to fix a false-positive assertion. [#4104](https://github.com/PyO3/pyo3/pull/4104) +- Disable `PyUnicode_DATA` on PyPy: not exposed by PyPy. [#4116](https://github.com/PyO3/pyo3/pull/4116) +- Correctly handle `#[pyo3(from_py_with = ...)]` attribute on dunder (`__magic__`) method arguments instead of silently ignoring it. [#4117](https://github.com/PyO3/pyo3/pull/4117) +- Fix a compile error when declaring a standalone function or class method with a Python name that is a Rust keyword. [#4226](https://github.com/PyO3/pyo3/pull/4226) +- Fix declarative modules discarding doc comments on the `mod` node. [#4236](https://github.com/PyO3/pyo3/pull/4236) +- Fix `__dict__` attribute missing for `#[pyclass(dict)]` instances when building for `abi3` on Python 3.9. [#4251](https://github.com/PyO3/pyo3/pull/4251) + +## [0.21.2] - 2024-04-16 + +### Changed + +- Deprecate the `PySet::empty()` gil-ref constructor. [#4082](https://github.com/PyO3/pyo3/pull/4082) + +### Fixed + +- Fix compile error for `async fn` in `#[pymethods]` with a `&self` receiver and more than one additional argument. [#4035](https://github.com/PyO3/pyo3/pull/4035) +- Improve error message for wrong receiver type in `__traverse__`. [#4045](https://github.com/PyO3/pyo3/pull/4045) +- Fix compile error when exporting a `#[pyclass]` living in a different Rust module using the `experimental-declarative-modules` feature. [#4054](https://github.com/PyO3/pyo3/pull/4054) +- Fix `missing_docs` lint triggering on documented `#[pymodule]` functions. [#4067](https://github.com/PyO3/pyo3/pull/4067) +- Fix undefined symbol errors for extension modules on AIX (by linking `libpython`). [#4073](https://github.com/PyO3/pyo3/pull/4073) + +## [0.21.1] - 2024-04-01 + +### Added + +- Implement `Send` and `Sync` for `PyBackedStr` and `PyBackedBytes`. [#4007](https://github.com/PyO3/pyo3/pull/4007) +- Implement `Clone`, `Debug`, `PartialEq`, `Eq`, `PartialOrd`, `Ord` and `Hash` implementation for `PyBackedBytes` and `PyBackedStr`, and `Display` for `PyBackedStr`. [#4020](https://github.com/PyO3/pyo3/pull/4020) +- Add `import_exception_bound!` macro to import exception types without generating GIL Ref functionality for them. [#4027](https://github.com/PyO3/pyo3/pull/4027) + +### Changed + +- Emit deprecation warning for uses of GIL Refs as `#[setter]` function arguments. [#3998](https://github.com/PyO3/pyo3/pull/3998) +- Add `#[inline]` hints on many `Bound` and `Borrowed` methods. [#4024](https://github.com/PyO3/pyo3/pull/4024) + +### Fixed + +- Handle `#[pyo3(from_py_with = "")]` in `#[setter]` methods [#3995](https://github.com/PyO3/pyo3/pull/3995) +- Allow extraction of `&Bound` in `#[setter]` methods. [#3998](https://github.com/PyO3/pyo3/pull/3998) +- Fix some uncovered code blocks emitted by `#[pymodule]`, `#[pyfunction]` and `#[pyclass]` macros. [#4009](https://github.com/PyO3/pyo3/pull/4009) +- Fix typo in the panic message when a class referenced in `pyo3::import_exception!` does not exist. [#4012](https://github.com/PyO3/pyo3/pull/4012) +- Fix compile error when using an async `#[pymethod]` with a receiver and additional arguments. [#4015](https://github.com/PyO3/pyo3/pull/4015) + + +## [0.21.0] - 2024-03-25 + +### Added + +- Add support for GraalPy (24.0 and up). [#3247](https://github.com/PyO3/pyo3/pull/3247) +- Add `PyMemoryView` type. [#3514](https://github.com/PyO3/pyo3/pull/3514) +- Allow `async fn` in for `#[pyfunction]` and `#[pymethods]`, with the `experimental-async` feature. [#3540](https://github.com/PyO3/pyo3/pull/3540) [#3588](https://github.com/PyO3/pyo3/pull/3588) [#3599](https://github.com/PyO3/pyo3/pull/3599) [#3931](https://github.com/PyO3/pyo3/pull/3931) +- Implement `PyTypeInfo` for `PyEllipsis`, `PyNone` and `PyNotImplemented`. [#3577](https://github.com/PyO3/pyo3/pull/3577) +- Support `#[pyclass]` on enums that have non-unit variants. [#3582](https://github.com/PyO3/pyo3/pull/3582) +- Support `chrono` feature with `abi3` feature. [#3664](https://github.com/PyO3/pyo3/pull/3664) +- `FromPyObject`, `IntoPy` and `ToPyObject` are implemented on `std::duration::Duration` [#3670](https://github.com/PyO3/pyo3/pull/3670) +- Add `PyString::to_cow`. Add `Py::to_str`, `Py::to_cow`, and `Py::to_string_lossy`, as ways to access Python string data safely beyond the GIL lifetime. [#3677](https://github.com/PyO3/pyo3/pull/3677) +- Add `Bound` and `Borrowed` smart pointers as a new API for accessing Python objects. [#3686](https://github.com/PyO3/pyo3/pull/3686) +- Add `PyNativeType::as_borrowed` to convert "GIL refs" to the new `Bound` smart pointer. [#3692](https://github.com/PyO3/pyo3/pull/3692) +- Add `FromPyObject::extract_bound` method, to migrate `FromPyObject` implementations to the Bound API. [#3706](https://github.com/PyO3/pyo3/pull/3706) +- Add `gil-refs` feature to allow continued use of the deprecated GIL Refs APIs. [#3707](https://github.com/PyO3/pyo3/pull/3707) +- Add methods to `PyAnyMethods` for binary operators (`add`, `sub`, etc.) [#3712](https://github.com/PyO3/pyo3/pull/3712) +- Add `chrono-tz` feature allowing conversion between `chrono_tz::Tz` and `zoneinfo.ZoneInfo` [#3730](https://github.com/PyO3/pyo3/pull/3730) +- Add FFI definition `PyType_GetModuleByDef`. [#3734](https://github.com/PyO3/pyo3/pull/3734) +- Conversion between `std::time::SystemTime` and `datetime.datetime` [#3736](https://github.com/PyO3/pyo3/pull/3736) +- Add `Py::as_any` and `Py::into_any`. [#3785](https://github.com/PyO3/pyo3/pull/3785) +- Add `PyStringMethods::encode_utf8`. [#3801](https://github.com/PyO3/pyo3/pull/3801) +- Add `PyBackedStr` and `PyBackedBytes`, as alternatives to `&str` and `&bytes` where a Python object owns the data. [#3802](https://github.com/PyO3/pyo3/pull/3802) [#3991](https://github.com/PyO3/pyo3/pull/3991) +- Allow `#[pymodule]` macro on Rust `mod` blocks, with the `experimental-declarative-modules` feature. [#3815](https://github.com/PyO3/pyo3/pull/3815) +- Implement `ExactSizeIterator` for `set` and `frozenset` iterators on `abi3` feature. [#3849](https://github.com/PyO3/pyo3/pull/3849) +- Add `Py::drop_ref` to explicitly drop a `Py`` and immediately decrease the Python reference count if the GIL is already held. [#3871](https://github.com/PyO3/pyo3/pull/3871) +- Allow `#[pymodule]` macro on single argument functions that take `&Bound<'_, PyModule>`. [#3905](https://github.com/PyO3/pyo3/pull/3905) +- Implement `FromPyObject` for `Cow`. [#3928](https://github.com/PyO3/pyo3/pull/3928) +- Implement `Default` for `GILOnceCell`. [#3971](https://github.com/PyO3/pyo3/pull/3971) +- Add `PyDictMethods::into_mapping`, `PyListMethods::into_sequence` and `PyTupleMethods::into_sequence`. [#3982](https://github.com/PyO3/pyo3/pull/3982) + +### Changed + +- `PyDict::from_sequence` now takes a single argument of type `&PyAny` (previously took two arguments `Python` and `PyObject`). [#3532](https://github.com/PyO3/pyo3/pull/3532) +- Deprecate `Py::is_ellipsis` and `PyAny::is_ellipsis` in favour of `any.is(py.Ellipsis())`. [#3577](https://github.com/PyO3/pyo3/pull/3577) +- Split some `PyTypeInfo` functionality into new traits `HasPyGilRef` and `PyTypeCheck`. [#3600](https://github.com/PyO3/pyo3/pull/3600) +- Deprecate `PyTryFrom` and `PyTryInto` traits in favor of `any.downcast()` via the `PyTypeCheck` and `PyTypeInfo` traits. [#3601](https://github.com/PyO3/pyo3/pull/3601) +- Allow async methods to accept `&self`/`&mut self` [#3609](https://github.com/PyO3/pyo3/pull/3609) +- `FromPyObject` for set types now also accept `frozenset` objects as input. [#3632](https://github.com/PyO3/pyo3/pull/3632) +- `FromPyObject` for `bool` now also accepts NumPy's `bool_` as input. [#3638](https://github.com/PyO3/pyo3/pull/3638) +- Add `AsRefSource` associated type to `PyNativeType`. [#3653](https://github.com/PyO3/pyo3/pull/3653) +- Rename `.is_true` to `.is_truthy` on `PyAny` and `Py` to clarify that the test is not based on identity with or equality to the True singleton. [#3657](https://github.com/PyO3/pyo3/pull/3657) +- `PyType::name` is now `PyType::qualname` whereas `PyType::name` efficiently accesses the full name which includes the module name. [#3660](https://github.com/PyO3/pyo3/pull/3660) +- The `Iter(A)NextOutput` types are now deprecated and `__(a)next__` can directly return anything which can be converted into Python objects, i.e. awaitables do not need to be wrapped into `IterANextOutput` or `Option` any more. `Option` can still be used as well and returning `None` will trigger the fast path for `__next__`, stopping iteration without having to raise a `StopIteration` exception. [#3661](https://github.com/PyO3/pyo3/pull/3661) +- Implement `FromPyObject` on `chrono::DateTime` for all `Tz`, not just `FixedOffset` and `Utc`. [#3663](https://github.com/PyO3/pyo3/pull/3663) +- Add lifetime parameter to `PyTzInfoAccess` trait. For the deprecated gil-ref API, the trait is now implemented for `&'py PyTime` and `&'py PyDateTime` instead of `PyTime` and `PyDate`. [#3679](https://github.com/PyO3/pyo3/pull/3679) +- Calls to `__traverse__` become no-ops for unsendable pyclasses if on the wrong thread, thereby avoiding hard aborts at the cost of potential leakage. [#3689](https://github.com/PyO3/pyo3/pull/3689) +- Include `PyNativeType` in `pyo3::prelude`. [#3692](https://github.com/PyO3/pyo3/pull/3692) +- Improve performance of `extract::` (and other integer types) by avoiding call to `__index__()` converting the value to an integer for 3.10+. Gives performance improvement of around 30% for successful extraction. [#3742](https://github.com/PyO3/pyo3/pull/3742) +- Relax bound of `FromPyObject` for `Py` to just `T: PyTypeCheck`. [#3776](https://github.com/PyO3/pyo3/pull/3776) +- `PySet` and `PyFrozenSet` iterators now always iterate the equivalent of `iter(set)`. (A "fast path" with no noticeable performance benefit was removed.) [#3849](https://github.com/PyO3/pyo3/pull/3849) +- Move implementations of `FromPyObject` for `&str`, `Cow`, `&[u8]` and `Cow<[u8]>` onto a temporary trait `FromPyObjectBound` when `gil-refs` feature is deactivated. [#3928](https://github.com/PyO3/pyo3/pull/3928) +- Deprecate `GILPool`, `Python::with_pool`, and `Python::new_pool`. [#3947](https://github.com/PyO3/pyo3/pull/3947) + +### Removed + +- Remove all functionality deprecated in PyO3 0.19. [#3603](https://github.com/PyO3/pyo3/pull/3603) + +### Fixed + +- Match PyPy 7.3.14 in removing PyPy-only symbol `Py_MAX_NDIMS` in favour of `PyBUF_MAX_NDIM`. [#3757](https://github.com/PyO3/pyo3/pull/3757) +- Fix segmentation fault using `datetime` types when an invalid `datetime` module is on sys.path. [#3818](https://github.com/PyO3/pyo3/pull/3818) +- Fix `non_local_definitions` lint warning triggered by many PyO3 macros. [#3901](https://github.com/PyO3/pyo3/pull/3901) +- Disable `PyCode` and `PyCode_Type` on PyPy: `PyCode_Type` is not exposed by PyPy. [#3934](https://github.com/PyO3/pyo3/pull/3934) + +## [0.21.0-beta.0] - 2024-03-10 + +Prerelease of PyO3 0.21. See [the GitHub diff](https://github.com/pyo3/pyo3/compare/v0.21.0-beta.0...v0.21.0) for what changed between 0.21.0-beta.0 and the final release. + +## [0.20.3] - 2024-02-23 + +### Packaging + +- Add `portable-atomic` dependency. [#3619](https://github.com/PyO3/pyo3/pull/3619) +- Check maximum version of Python at build time and for versions not yet supported require opt-in to the `abi3` stable ABI by the environment variable `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1`. [#3821](https://github.com/PyO3/pyo3/pull/3821) + +### Fixed + +- Use `portable-atomic` to support platforms without 64-bit atomics. [#3619](https://github.com/PyO3/pyo3/pull/3619) +- Fix compilation failure with `either` feature enabled without `experimental-inspect` enabled. [#3834](https://github.com/PyO3/pyo3/pull/3834) + ## [0.20.2] - 2024-01-04 ### Packaging @@ -66,7 +807,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h ### Changed - Change `PySet::discard` to return `PyResult` (previously returned nothing). [#3281](https://github.com/PyO3/pyo3/pull/3281) -- Optimize implmentation of `IntoPy` for Rust tuples to Python tuples. [#3321](https://github.com/PyO3/pyo3/pull/3321) +- Optimize implementation of `IntoPy` for Rust tuples to Python tuples. [#3321](https://github.com/PyO3/pyo3/pull/3321) - Change `PyDict::get_item` to no longer suppress arbitrary exceptions (the return type is now `PyResult>` instead of `Option<&PyAny>`), and deprecate `PyDict::get_item_with_error`. [#3330](https://github.com/PyO3/pyo3/pull/3330) - Deprecate FFI definitions which are deprecated in Python 3.12. [#3336](https://github.com/PyO3/pyo3/pull/3336) - `AsPyPointer` is now an `unsafe trait`. [#3358](https://github.com/PyO3/pyo3/pull/3358) @@ -173,7 +914,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Add `#[pyo3(from_item_all)]` when deriving `FromPyObject` to specify `get_item` as getter for all fields. [#3120](https://github.com/PyO3/pyo3/pull/3120) - Add `pyo3::exceptions::PyBaseExceptionGroup` for Python 3.11, and corresponding FFI definition `PyExc_BaseExceptionGroup`. [#3141](https://github.com/PyO3/pyo3/pull/3141) - Accept `#[new]` with `#[classmethod]` to create a constructor which receives a (subtype's) class/`PyType` as its first argument. [#3157](https://github.com/PyO3/pyo3/pull/3157) -- Add `PyClass::get` and `Py::get` for GIL-indepedent access to classes with `#[pyclass(frozen)]`. [#3158](https://github.com/PyO3/pyo3/pull/3158) +- Add `PyClass::get` and `Py::get` for GIL-independent access to classes with `#[pyclass(frozen)]`. [#3158](https://github.com/PyO3/pyo3/pull/3158) - Add `PyAny::is_exact_instance` and `PyAny::is_exact_instance_of`. [#3161](https://github.com/PyO3/pyo3/pull/3161) ### Changed @@ -600,7 +1341,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Respect Rust privacy rules for items wrapped with `wrap_pyfunction` and `wrap_pymodule`. [#2081](https://github.com/PyO3/pyo3/pull/2081) - Add modulo argument to `__ipow__` magic method. [#2083](https://github.com/PyO3/pyo3/pull/2083) - Fix FFI definition for `_PyCFunctionFast`. [#2126](https://github.com/PyO3/pyo3/pull/2126) -- `PyDateTimeAPI` and `PyDateTime_TimeZone_UTC` are are now unsafe functions instead of statics. [#2126](https://github.com/PyO3/pyo3/pull/2126) +- `PyDateTimeAPI` and `PyDateTime_TimeZone_UTC` are now unsafe functions instead of statics. [#2126](https://github.com/PyO3/pyo3/pull/2126) - `PyDateTimeAPI` does not implicitly call `PyDateTime_IMPORT` anymore to reflect the original Python API more closely. Before the first call to `PyDateTime_IMPORT` a null pointer is returned. Therefore before calling any of the following FFI functions `PyDateTime_IMPORT` must be called to avoid undefined behavior: [#2126](https://github.com/PyO3/pyo3/pull/2126) - `PyDateTime_TimeZone_UTC` - `PyDate_Check` @@ -1131,8 +1872,8 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Change return type of `PyTuple::slice` and `PyTuple::split_from` from `Py` to `&PyTuple`. [#970](https://github.com/PyO3/pyo3/pull/970) - Change return type of `PyTuple::as_slice` to `&[&PyAny]`. [#971](https://github.com/PyO3/pyo3/pull/971) - Rename `PyTypeInfo::type_object` to `type_object_raw`, and add `Python` argument. [#975](https://github.com/PyO3/pyo3/pull/975) -- Update `num-complex` optional dependendency from `0.2` to `0.3`. [#977](https://github.com/PyO3/pyo3/pull/977) -- Update `num-bigint` optional dependendency from `0.2` to `0.3`. [#978](https://github.com/PyO3/pyo3/pull/978) +- Update `num-complex` optional dependency from `0.2` to `0.3`. [#977](https://github.com/PyO3/pyo3/pull/977) +- Update `num-bigint` optional dependency from `0.2` to `0.3`. [#978](https://github.com/PyO3/pyo3/pull/978) - `#[pyproto]` is re-implemented without specialization. [#961](https://github.com/PyO3/pyo3/pull/961) - `PyClassAlloc::alloc` is renamed to `PyClassAlloc::new`. [#990](https://github.com/PyO3/pyo3/pull/990) - `#[pyproto]` methods can now have return value `T` or `PyResult` (previously only `PyResult` was supported). [#996](https://github.com/PyO3/pyo3/pull/996) @@ -1465,7 +2206,7 @@ Yanked - `IntoPyDictPointer` was replace by `IntoPyDict` which doesn't convert `PyDict` itself anymore and returns a `PyDict` instead of `*mut PyObject`. - `PyTuple::new` now takes an `IntoIterator` instead of a slice - Updated to syn 0.15 -- Splitted `PyTypeObject` into `PyTypeObject` without the create method and `PyTypeCreate` with requires `PyObjectAlloc + PyTypeInfo + Sized`. +- Split `PyTypeObject` into `PyTypeObject` without the create method and `PyTypeCreate` with requires `PyObjectAlloc + PyTypeInfo + Sized`. - Ran `cargo edition --fix` which prefixed path with `crate::` for rust 2018 - Renamed `async` to `pyasync` as async will be a keyword in the 2018 edition. - Starting to use `NonNull<*mut PyObject>` for Py and PyObject by ijl [#260](https://github.com/PyO3/pyo3/pull/260) @@ -1538,7 +2279,7 @@ Yanked - `proc_macro` has been stabilized on nightly ([rust-lang/rust#52081](https://github.com/rust-lang/rust/pull/52081)). This means that we can remove the `proc_macro` feature, but now we need the `use_extern_macros` from the 2018 edition instead. - All proc macro are now prefixed with `py` and live in the prelude. This means you can use `#[pyclass]`, `#[pymethods]`, `#[pyproto]`, `#[pyfunction]` and `#[pymodinit]` directly, at least after a `use pyo3::prelude::*`. They were also moved into a module called `proc_macro`. You shouldn't use `#[pyo3::proc_macro::pyclass]` or other longer paths in attributes because `proc_macro_path_invoc` isn't going to be stabilized soon. - Renamed the `base` option in the `pyclass` macro to `extends`. -- `#[pymodinit]` uses the function name as module name, unless the name is overrriden with `#[pymodinit(name)]` +- `#[pymodinit]` uses the function name as module name, unless the name is overridden with `#[pymodinit(name)]` - The guide is now properly versioned. ## [0.2.7] - 2018-05-18 @@ -1612,7 +2353,7 @@ Yanked - Added inheritance support #15 - Added weakref support #56 - Added subclass support #64 -- Added `self.__dict__` supoort #68 +- Added `self.__dict__` support #68 - Added `pyo3::prelude` module #70 - Better `Iterator` support for PyTuple, PyList, PyDict #75 - Introduce IntoPyDictPointer similar to IntoPyTuple #69 @@ -1628,7 +2369,31 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.20.2...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.27.0...HEAD +[0.27.0]: https://github.com/pyo3/pyo3/compare/v0.26.0...v0.27.0 +[0.26.0]: https://github.com/pyo3/pyo3/compare/v0.25.1...v0.26.0 +[0.25.1]: https://github.com/pyo3/pyo3/compare/v0.25.0...v0.25.1 +[0.25.0]: https://github.com/pyo3/pyo3/compare/v0.24.2...v0.25.0 +[0.24.2]: https://github.com/pyo3/pyo3/compare/v0.24.1...v0.24.2 +[0.24.1]: https://github.com/pyo3/pyo3/compare/v0.24.0...v0.24.1 +[0.24.0]: https://github.com/pyo3/pyo3/compare/v0.23.5...v0.24.0 +[0.23.5]: https://github.com/pyo3/pyo3/compare/v0.23.4...v0.23.5 +[0.23.4]: https://github.com/pyo3/pyo3/compare/v0.23.3...v0.23.4 +[0.23.3]: https://github.com/pyo3/pyo3/compare/v0.23.2...v0.23.3 +[0.23.2]: https://github.com/pyo3/pyo3/compare/v0.23.1...v0.23.2 +[0.23.1]: https://github.com/pyo3/pyo3/compare/v0.23.0...v0.23.1 +[0.23.0]: https://github.com/pyo3/pyo3/compare/v0.22.5...v0.23.0 +[0.22.5]: https://github.com/pyo3/pyo3/compare/v0.22.4...v0.22.5 +[0.22.4]: https://github.com/pyo3/pyo3/compare/v0.22.3...v0.22.4 +[0.22.3]: https://github.com/pyo3/pyo3/compare/v0.22.2...v0.22.3 +[0.22.2]: https://github.com/pyo3/pyo3/compare/v0.22.1...v0.22.2 +[0.22.1]: https://github.com/pyo3/pyo3/compare/v0.22.0...v0.22.1 +[0.22.0]: https://github.com/pyo3/pyo3/compare/v0.21.2...v0.22.0 +[0.21.2]: https://github.com/pyo3/pyo3/compare/v0.21.1...v0.21.2 +[0.21.1]: https://github.com/pyo3/pyo3/compare/v0.21.0...v0.21.1 +[0.21.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0 +[0.21.0-beta.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0-beta.0 +[0.20.3]: https://github.com/pyo3/pyo3/compare/v0.20.2...v0.20.3 [0.20.2]: https://github.com/pyo3/pyo3/compare/v0.20.1...v0.20.2 [0.20.1]: https://github.com/pyo3/pyo3/compare/v0.20.0...v0.20.1 [0.20.0]: https://github.com/pyo3/pyo3/compare/v0.19.2...v0.20.0 @@ -1674,7 +2439,7 @@ Yanked [0.9.2]: https://github.com/pyo3/pyo3/compare/v0.9.1...v0.9.2 [0.9.1]: https://github.com/pyo3/pyo3/compare/v0.9.0...v0.9.1 [0.9.0]: https://github.com/pyo3/pyo3/compare/v0.8.5...v0.9.0 -[0.8.4]: https://github.com/pyo3/pyo3/compare/v0.8.4...v0.8.5 +[0.8.5]: https://github.com/pyo3/pyo3/compare/v0.8.4...v0.8.5 [0.8.4]: https://github.com/pyo3/pyo3/compare/v0.8.3...v0.8.4 [0.8.3]: https://github.com/pyo3/pyo3/compare/v0.8.2...v0.8.3 [0.8.2]: https://github.com/pyo3/pyo3/compare/v0.8.1...v0.8.2 @@ -1683,7 +2448,8 @@ Yanked [0.7.0]: https://github.com/pyo3/pyo3/compare/v0.6.0...v0.7.0 [0.6.0]: https://github.com/pyo3/pyo3/compare/v0.5.3...v0.6.0 [0.5.3]: https://github.com/pyo3/pyo3/compare/v0.5.2...v0.5.3 -[0.5.2]: https://github.com/pyo3/pyo3/compare/v0.5.0...v0.5.2 +[0.5.2]: https://github.com/pyo3/pyo3/compare/v0.5.1...v0.5.2 +[0.5.1]: https://github.com/pyo3/pyo3/compare/v0.5.0...v0.5.1 [0.5.0]: https://github.com/pyo3/pyo3/compare/v0.4.1...v0.5.0 [0.4.1]: https://github.com/pyo3/pyo3/compare/v0.4.0...v0.4.1 [0.4.0]: https://github.com/pyo3/pyo3/compare/v0.3.2...v0.4.0 diff --git a/Cargo.toml b/Cargo.toml index 5386b76f573..3c2e5372fbb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.21.0-dev" +version = "0.27.0" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -10,46 +10,67 @@ repository = "https://github.com/pyo3/pyo3" documentation = "https://docs.rs/crate/pyo3/" categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" -exclude = ["/.gitignore", ".cargo/config", "/codecov.yml", "/Makefile", "/pyproject.toml", "/noxfile.py", "/.github", "/tests/test_compile_error.rs", "/tests/ui"] +exclude = [ + "/.gitignore", + ".cargo/config", + "/codecov.yml", + "/Makefile", + "/pyproject.toml", + "/noxfile.py", + "/.github", + "/tests/test_compile_error.rs", + "/tests/ui", +] edition = "2021" -rust-version = "1.56" +rust-version.workspace = true [dependencies] -cfg-if = "1.0" libc = "0.2.62" -parking_lot = ">= 0.11, < 0.13" -memoffset = "0.9" -portable-atomic = "1.0" +once_cell = "1.21" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.0-dev" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.27.0" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.21.0-dev", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.27.0", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } # support crate for multiple-pymethods feature -inventory = { version = "0.3.0", optional = true } +inventory = { version = "0.3.5", optional = true } # crate integrations that can be added using the eponymous features -anyhow = { version = "1.0", optional = true } +anyhow = { version = "1.0.1", optional = true } +bigdecimal = { version = "0.4.7", optional = true } +bytes = { version = "1.10", optional = true } chrono = { version = "0.4.25", default-features = false, optional = true } -chrono-tz = { version = ">= 0.6, < 0.9", default-features = false, optional = true } +chrono-tz = { version = ">= 0.10, < 0.11", default-features = false, optional = true } either = { version = "1.9", optional = true } -eyre = { version = ">= 0.4, < 0.7", optional = true } -hashbrown = { version = ">= 0.9, < 0.15", optional = true } -indexmap = { version = ">= 1.6, < 3", optional = true } -num-bigint = { version = "0.4", optional = true } -num-complex = { version = ">= 0.2, < 0.5", optional = true } -rust_decimal = { version = "1.0.0", default-features = false, optional = true } +eyre = { version = ">= 0.6.8, < 0.7", optional = true } +hashbrown = { version = ">= 0.15.0, < 0.17", optional = true, default-features = false } +indexmap = { version = ">= 2.5.0, < 3", optional = true } +jiff-02 = { package = "jiff", version = "0.2", optional = true } +num-bigint = { version = "0.4.4", optional = true } +num-complex = { version = ">= 0.4.6, < 0.5", optional = true } +num-rational = { version = "0.4.1", optional = true } +num-traits = { version = "0.2.16", optional = true } +ordered-float = { version = "5.0.0", default-features = false, optional = true } +rust_decimal = { version = "1.15", default-features = false, optional = true } +time = { version = "0.3.38", default-features = false, optional = true } serde = { version = "1.0", optional = true } smallvec = { version = "1.0", optional = true } +uuid = { version = "1.11.0", optional = true } +lock_api = { version = "0.4", optional = true } +parking_lot = { version = "0.12", optional = true } +iana-time-zone = { version = "0.1", optional = true, features = ["fallback"]} + +[target.'cfg(not(target_has_atomic = "64"))'.dependencies] +portable-atomic = "1.0" [dev-dependencies] assert_approx_eq = "1.1.0" chrono = "0.4.25" -chrono-tz = ">= 0.6, < 0.9" +chrono-tz = ">= 0.10, < 0.11" # Required for "and $N others" normalization trybuild = ">=1.0.70" proptest = { version = "1.0", default-features = false, features = ["std"] } @@ -58,16 +79,23 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.61" rayon = "1.6.1" futures = "0.3.28" +tempfile = "3.12.0" +static_assertions = "1.1.0" +uuid = { version = "1.10.0", features = ["v4"] } +parking_lot = { version = "0.12.3", features = ["arc_lock"] } [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.27.0", features = ["resolve-config"] } [features] default = ["macros"] +# Enables support for `async fn` for `#[pyfunction]` and `#[pymethods]`. +experimental-async = ["macros", "pyo3-macros/experimental-async"] + # Enables pyo3::inspect module and additional type information on FromPyObject # and IntoPy traits -experimental-inspect = [] +experimental-inspect = ["pyo3-macros/experimental-inspect"] # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc. macros = ["pyo3-macros", "indoc", "unindent"] @@ -89,16 +117,29 @@ abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38", "pyo3-ffi/abi3-py38"] abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"] abi3-py310 = ["abi3-py311", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"] abi3-py311 = ["abi3-py312", "pyo3-build-config/abi3-py311", "pyo3-ffi/abi3-py311"] -abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312", "pyo3-ffi/abi3-py312"] +abi3-py312 = ["abi3-py313", "pyo3-build-config/abi3-py312", "pyo3-ffi/abi3-py312"] +abi3-py313 = ["abi3-py314", "pyo3-build-config/abi3-py313", "pyo3-ffi/abi3-py313"] +abi3-py314 = ["abi3", "pyo3-build-config/abi3-py314", "pyo3-ffi/abi3-py314"] # Automatically generates `python3.dll` import libraries for Windows targets. generate-import-lib = ["pyo3-ffi/generate-import-lib"] -# Changes `Python::with_gil` to automatically initialize the Python interpreter if needed. +# Changes `Python::attach` to automatically initialize the Python interpreter if needed. auto-initialize = [] -# Allows use of the deprecated "GIL Refs" APIs. -gil-refs = [] +# Enables `Clone`ing references to Python objects `Py` which panics if the +# thread is not attached to the Python interpreter. +py-clone = [] + +# Adds `OnceExt` and `MutexExt` implementations to the `parking_lot` types +parking_lot = ["dep:parking_lot", "lock_api"] +arc_lock = ["lock_api", "lock_api/arc_lock", "parking_lot?/arc_lock"] + +num-bigint = ["dep:num-bigint", "dep:num-traits"] +bigdecimal = ["dep:bigdecimal", "num-bigint"] + +chrono-local = ["chrono/clock", "dep:iana-time-zone"] + # Optimizes PyObject to Vec conversion and so on. nightly = [] @@ -107,20 +148,33 @@ nightly = [] # This is mostly intended for testing purposes - activating *all* of these isn't particularly useful. full = [ "macros", - # "multiple-pymethods", # TODO re-add this when MSRV is greater than 1.62 + # "multiple-pymethods", # Not supported by wasm + "anyhow", + "arc_lock", + "bigdecimal", + "bytes", "chrono", + "chrono-local", "chrono-tz", - "num-bigint", - "num-complex", - "hashbrown", - "smallvec", - "serde", - "indexmap", "either", - "eyre", - "anyhow", + "experimental-async", "experimental-inspect", + "eyre", + "hashbrown", + "indexmap", + "jiff-02", + "lock_api", + "num-bigint", + "num-complex", + "num-rational", + "ordered-float", + "parking_lot", + "py-clone", "rust_decimal", + "serde", + "smallvec", + "time", + "uuid", ] [workspace] @@ -129,9 +183,11 @@ members = [ "pyo3-build-config", "pyo3-macros", "pyo3-macros-backend", + "pyo3-introspection", "pytests", "examples", ] +package.rust-version = "1.83" [package.metadata.docs.rs] no-default-features = true @@ -149,6 +205,9 @@ let_unit_value = "warn" manual_assert = "warn" manual_ok_or = "warn" todo = "warn" +# TODO: make this "warn" +# https://github.com/PyO3/pyo3/issues/5487 +undocumented_unsafe_blocks = "allow" unnecessary_wraps = "warn" useless_transmute = "warn" used_underscore_binding = "warn" @@ -156,9 +215,10 @@ used_underscore_binding = "warn" [workspace.lints.rust] elided_lifetimes_in_paths = "warn" invalid_doc_attributes = "warn" -rust_2018_idioms = "warn" +rust_2018_idioms = { level = "warn", priority = -1 } rust_2021_prelude_collisions = "warn" unused_lifetimes = "warn" +unsafe_op_in_unsafe_fn = "warn" [workspace.lints.rustdoc] broken_intra_doc_links = "warn" diff --git a/Contributing.md b/Contributing.md index 332645542d7..97dda500998 100644 --- a/Contributing.md +++ b/Contributing.md @@ -18,24 +18,52 @@ The following sections also contain specific ideas on where to start contributin ## Setting up a development environment To work and develop PyO3, you need Python & Rust installed on your system. + * We encourage the use of [rustup](https://rustup.rs/) to be able to select and choose specific toolchains based on the project. * [Pyenv](https://github.com/pyenv/pyenv) is also highly recommended for being able to choose a specific Python version. * [virtualenv](https://virtualenv.pypa.io/en/latest/) can also be used with or without Pyenv to use specific installed Python versions. * [`nox`][nox] is used to automate many of our CI tasks. -### Caveats +### Testing, linting, etc. with nox + +[`Nox`][nox] is used to automate many of our CI tasks and can be used locally to handle verfication tasks as you code. We recommend running these actions via nox to make use of our prefered configuration options. You can install nox into your global python with pip: `pip install nox` or (recommended) with [`pipx`][pipx] `pip install pipx`, `pipx install nox` + +The main nox commands we have implemented are: + +* `nox -s test` will run the full suite of recommended rust and python tests (>10 minutes) +* `nox -s test-rust -- skip-full` will run a short suite of rust tests (2-3 minutes) +* `nox -s ruff` will check python linting and apply standard formatting rules +* `nox -s rustfmt` will check basic rust linting and apply standard formatting rules +* `nox -s rumdl` will check the markdown in the guide +* `nox -s clippy` will run clippy to make recommendations on rust style +* `nox -s bench` will benchmark your rust code +* `nox -s codspeed` will run our suite of rust and python performance tests +* `nox -s coverage` will analyse test coverage and output `coverage.json` (alternatively: `nox -s coverage lcov` outputs `lcov.info`) +* `nox -s check-guide` will use [`lychee`][lychee] to check all the links in the guide and doc comments. + +Use `nox -l` to list the full set of subcommands you can run. + +#### UI Tests + +PyO3 uses [`trybuild`][trybuild] to develop UI tests to capture error messages from the Rust compiler for some of the macro functionality. + +Because there are several feature combinations for these UI tests, when updating them all (e.g. for a new Rust compiler version) it may be helpful to use the `update-ui-tests` nox session: + +```bash +nox -s update-ui-tests +``` -* When using pyenv on macOS, installing a Python version using `--enable-shared` is required to make it work. i.e `env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.7.12` +## Ways to help ### Help users identify bugs -The [PyO3 Gitter channel](https://gitter.im/PyO3/Lobby) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase. +The [PyO3 Discord server](https://discord.gg/33kcChzH7f) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase. Helping others often reveals bugs, documentation weaknesses, and missing APIs. It's a good idea to open GitHub issues for these immediately so the resolution can be designed and implemented! ### Implement issues ready for development -Issues where the solution is clear and work is not in progress use the [needs-implementer](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-implemeter) label. +Issues where the solution is clear and work is not in progress use the [needs-implementer](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-implementer) label. Don't be afraid if the solution is not clear to you! The core PyO3 contributors will be happy to mentor you through any questions you have to help you write the solution. @@ -57,19 +85,25 @@ nox -s docs -- open #### Doctests We use lots of code blocks in our docs. Run `cargo test --doc` when making changes to check that -the doctests still work, or `cargo test` to run all the tests including doctests. See +the doctests still work, or `cargo test` to run all the Rust tests including doctests. See https://doc.rust-lang.org/rustdoc/documentation-tests.html for a guide on doctests. #### Building the guide You can preview the user guide by building it locally with `mdbook`. -First, install [`mdbook`][mdbook] and [`nox`][nox]. Then, run +First, install [`mdbook`][mdbook], the [`mdbook-tabs`][mdbook-tabs] plugin and [`nox`][nox]. Then, run ```shell nox -s build-guide -- --open ``` +To check all links in the guide are valid, also install [`lychee`][lychee] and use the `check-guide` session instead: + +```shell +nox -s check-guide +``` + ### Help design the next PyO3 Issues which don't yet have a clear solution use the [needs-design](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-design) label. @@ -84,21 +118,18 @@ Everybody is welcome to submit comments on open PRs. Please help ensure new PyO3 Here are a few things to note when you are writing PRs. -### Continuous Integration - -The PyO3 repo uses GitHub Actions. PRs are blocked from merging if CI is not successful. +### Testing and Continuous Integration -Formatting, linting and tests are checked for all Rust and Python code. In addition, all warnings in Rust code are disallowed (using `RUSTFLAGS="-D warnings"`). +The PyO3 repo uses GitHub Actions. +PRs are blocked from merging if CI is not successful. +Formatting, linting and tests are checked for all Rust and Python code (the pipeline will abort early if formatting fails to save resources). +In addition, all warnings in Rust code are disallowed (using `RUSTFLAGS="-D warnings"`). Tests run with all supported Python versions with the latest stable Rust compiler, as well as for Python 3.9 with the minimum supported Rust version. If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI. -You can run these tests yourself with -```nox``` -and -```nox -l``` -lists further commands you can run. +You can run the CI pipeline components yourself with `nox`, see [the testing section above](#testing-linting-etc-with-nox). ### Documenting changes @@ -155,6 +186,20 @@ Below are guidelines on what compatibility all PRs are expected to deliver for e PyO3 supports all officially supported Python versions, as well as the latest PyPy3 release. All of these versions are tested in CI. +#### Adding support for new CPython versions + +If you plan to add support for a pre-release version of CPython, here's a (non-exhaustive) checklist: + + - [ ] Wait until the last alpha release (usually alpha7), since ABI is not guaranteed until the first beta release + - [ ] Add prerelease_ver-dev (e.g. `3.14-dev`) to `.github/workflows/ci.yml`, and bump version in `noxfile.py`, `pyo3-ffi/Cargo.toml` under `max-version` within `[package.metadata.cpython]`, and `max` within `pyo3-ffi/build.rs` +- [ ] Add a new abi3-prerelease feature for the version (e.g. `abi3-py314`) + - In `pyo3-build-config/Cargo.toml`, set abi3-most_current_stable to ["abi3-prerelease"] and abi3-prerelease to ["abi3"] + - In `pyo3-ffi/Cargo.toml`, set abi3-most_current_stable to ["abi3-prerelease", "pyo3-build-config/abi3-most_current_stable"] and abi3-prerelease to ["abi3", "pyo3-build-config/abi3-prerelease"] + - In `Cargo.toml`, set abi3-most_current_stable to ["abi3-prerelease", "pyo3-ffi/abi3-most_current_stable"] and abi3-prerelease to ["abi3", "pyo3-ffi/abi3-prerelease"] + - [ ] Use `#[cfg(Py_prerelease])` (e.g. `#[cfg(Py_3_14)]`) and `#[cfg(not(Py_prerelease]))` to indicate changes between the stable branches of CPython and the pre-release + - [ ] Do not add a Rust binding to any function, struct, or global variable prefixed with `_` in CPython's headers + - [ ] Ping @ngoldbaum and @davidhewitt for assistance + ### Rust PyO3 aims to make use of up-to-date Rust language features to keep the implementation as efficient as possible. @@ -171,15 +216,20 @@ First, there are Rust-based benchmarks located in the `pyo3-benches` subdirector nox -s bench -Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](pytests). +Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](https://github.com/PyO3/pyo3/tree/main/pytests). ## Code coverage You can view what code is and isn't covered by PyO3's tests. We aim to have 100% coverage - please check coverage and add tests if you notice a lack of coverage! -- First, generate a `lcov.info` file with +- First, ensure the llvm-cov cargo plugin is installed. You may need to run the plugin through cargo once before using it with `nox`. +```shell +cargo install cargo-llvm-cov +cargo llvm-cov +``` +- Then, generate an `lcov.info` file with ```shell -nox -s coverage +nox -s coverage -- lcov ``` You can install an IDE plugin to view the coverage. For example, if you use VSCode: - Add the [coverage-gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) plugin. @@ -198,7 +248,7 @@ You can install an IDE plugin to view the coverage. For example, if you use VSCo ## Sponsor this project -At the moment there is no official organisation that accepts sponsorship on PyO3's behalf. If you're seeking to provide significant funding to the PyO3 ecosystem, please reach out to us on [GitHub](https://github.com/PyO3/pyo3/issues/new) or [Gitter](https://gitter.im/PyO3/Lobby) and we can discuss. +At the moment there is no official organisation that accepts sponsorship on PyO3's behalf. If you're seeking to provide significant funding to the PyO3 ecosystem, please reach out to us on [GitHub](https://github.com/PyO3/pyo3/issues/new) or [Discord](https://discord.gg/33kcChzH7f) and we can discuss. In the meanwhile, some of our maintainers have personal GitHub sponsorship pages and would be grateful for your support: @@ -206,4 +256,8 @@ In the meanwhile, some of our maintainers have personal GitHub sponsorship pages - [messense](https://github.com/sponsors/messense) [mdbook]: https://rust-lang.github.io/mdBook/cli/index.html +[mdbook-tabs]: https://mdbook-plugins.rustforweb.org/tabs.html +[lychee]: https://github.com/lycheeverse/lychee [nox]: https://github.com/theacodes/nox +[pipx]: https://pipx.pypa.io/stable/ +[trybuild]: https://github.com/dtolnay/trybuild diff --git a/LICENSE-APACHE b/LICENSE-APACHE index fca31990733..72207b851d3 100644 --- a/LICENSE-APACHE +++ b/LICENSE-APACHE @@ -1,189 +1,178 @@ - Copyright (c) 2017-present PyO3 Project and Contributors. https://github.com/PyO3 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. +Copyright (c) 2017-present PyO3 Project and Contributors. https://github.com/PyO3 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md index 21e09157d8d..5b190c921fc 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # PyO3 [![actions status](https://img.shields.io/github/actions/workflow/status/PyO3/pyo3/ci.yml?branch=main&logo=github&style=)](https://github.com/PyO3/pyo3/actions) -[![benchmark](https://img.shields.io/badge/benchmark-✓-Green?logo=github)](https://pyo3.rs/dev/bench/) +[![benchmark](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/PyO3/pyo3) [![codecov](https://img.shields.io/codecov/c/gh/PyO3/pyo3?logo=codecov)](https://codecov.io/gh/PyO3/pyo3) [![crates.io](https://img.shields.io/crates/v/pyo3?logo=rust)](https://crates.io/crates/pyo3) -[![minimum rustc 1.56](https://img.shields.io/badge/rustc-1.56+-blue?logo=rust)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) -[![dev chat](https://img.shields.io/gitter/room/PyO3/Lobby?logo=gitter)](https://gitter.im/PyO3/Lobby) +[![minimum rustc 1.83](https://img.shields.io/badge/rustc-1.83+-blue?logo=rust)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) +[![discord server](https://img.shields.io/discord/1209263839632424990?logo=discord)](https://discord.gg/33kcChzH7f) [![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green?logo=github)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) [Rust](https://www.rust-lang.org/) bindings for [Python](https://www.python.org/), including tools for creating native Python extension modules. Running and interacting with Python code from a Rust binary is also supported. @@ -16,9 +16,12 @@ ## Usage -PyO3 supports the following software versions: - - Python 3.7 and up (CPython and PyPy) - - Rust 1.56 and up +Requires Rust 1.83 or greater. + +PyO3 supports the following Python distributions: + - CPython 3.7 or greater + - PyPy 7.3 (Python 3.11+) + - GraalPy 25.0 or greater (Python 3.12+) You can use PyO3 to write a native Python module in Rust, or to embed Python in a Rust binary. The following sections explain each of these in turn. @@ -68,27 +71,24 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.20.2", features = ["extension-module"] } +pyo3 = { version = "0.27.0", features = ["extension-module"] } ``` **`src/lib.rs`** ```rust -use pyo3::prelude::*; - -/// Formats the sum of two numbers as string. -#[pyfunction] -fn sum_as_string(a: usize, b: usize) -> PyResult { - Ok((a + b).to_string()) -} - -/// A Python module implemented in Rust. The name of this function must match +/// A Python module implemented in Rust. The name of this module must match /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to /// import the module. -#[pymodule] -fn string_sum(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; - Ok(()) +#[pyo3::pymodule] +mod string_sum { + use pyo3::prelude::*; + + /// Formats the sum of two numbers as string. + #[pyfunction] + fn sum_as_string(a: usize, b: usize) -> PyResult { + Ok((a + b).to_string()) + } } ``` @@ -118,7 +118,7 @@ maturin develop If you want to be able to run `cargo test` or use this project in a Cargo workspace and are running into linker issues, there are some workarounds in [the FAQ](https://pyo3.rs/latest/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror). -As well as with `maturin`, it is possible to build using [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](https://pyo3.rs/latest/building_and_distribution.html#manual-builds). Both offer more flexibility than `maturin` but require more configuration to get started. +As well as with `maturin`, it is possible to build using [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](https://pyo3.rs/latest/building-and-distribution.html#manual-builds). Both offer more flexibility than `maturin` but require more configuration to get started. ### Using Python from Rust @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.20.2" +version = "0.27.0" features = ["auto-initialize"] ``` @@ -148,13 +148,13 @@ use pyo3::prelude::*; use pyo3::types::IntoPyDict; fn main() -> PyResult<()> { - Python::with_gil(|py| { - let sys = py.import_bound("sys")?; + Python::attach(|py| { + let sys = py.import("sys")?; let version: String = sys.getattr("version")?.extract()?; - let locals = [("os", py.import_bound("os")?)].into_py_dict_bound(py); - let code = "os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"; - let user: String = py.eval_bound(code, None, Some(&locals))?.extract()?; + let locals = [("os", py.import("os")?)].into_py_dict(py)?; + let code = c"os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"; + let user: String = py.eval(code, None, Some(&locals))?.extract()?; println!("Hello {}, I'm Python {}", user, version); Ok(()) @@ -162,7 +162,7 @@ fn main() -> PyResult<()> { } ``` -The guide has [a section](https://pyo3.rs/latest/python_from_rust.html) with lots of examples +The guide has [a section](https://pyo3.rs/latest/python-from-rust.html) with lots of examples about this topic. ## Tools and libraries @@ -174,16 +174,24 @@ about this topic. - [dict-derive](https://github.com/gperinazzo/dict-derive) _Derive FromPyObject to automatically transform Python dicts into Rust structs_ - [pyo3-log](https://github.com/vorner/pyo3-log) _Bridge from Rust to Python logging_ - [pythonize](https://github.com/davidhewitt/pythonize) _Serde serializer for converting Rust objects to JSON-compatible Python objects_ -- [pyo3-asyncio](https://github.com/awestlake87/pyo3-asyncio) _Utilities for working with Python's Asyncio library and async functions_ +- [pyo3-async-runtimes](https://github.com/PyO3/pyo3-async-runtimes) _Utilities for interoperability with Python's Asyncio library and Rust's async runtimes._ - [rustimport](https://github.com/mityax/rustimport) _Directly import Rust files or crates from Python, without manual compilation step. Provides pyo3 integration by default and generates pyo3 binding code automatically._ +- [pyo3-arrow](https://crates.io/crates/pyo3-arrow) _Lightweight [Apache Arrow](https://arrow.apache.org/) integration for pyo3._ +- [pyo3-bytes](https://crates.io/crates/pyo3-bytes) _Integration between [`bytes`](https://crates.io/crates/bytes) and pyo3._ +- [pyo3-object_store](https://github.com/developmentseed/obstore/tree/main/pyo3-object_store) _Integration between [`object_store`](https://docs.rs/object_store) and [`pyo3`](https://github.com/PyO3/pyo3)._ ## Examples -- [autopy](https://github.com/autopilot-rs/autopy) _A simple, cross-platform GUI automation library for Python and Rust._ - - Contains an example of building wheels on TravisCI and appveyor using [cibuildwheel](https://github.com/pypa/cibuildwheel) -- [ballista-python](https://github.com/apache/arrow-ballista-python) _A Python library that binds to Apache Arrow distributed query engine Ballista._ +- [arro3](https://github.com/kylebarron/arro3) _A minimal Python library for Apache Arrow, connecting to the Rust arrow crate._ + - [arro3-compute](https://github.com/kylebarron/arro3/tree/main/arro3-compute) _`arro3-compute`_ + - [arro3-core](https://github.com/kylebarron/arro3/tree/main/arro3-core) _`arro3-core`_ + - [arro3-io](https://github.com/kylebarron/arro3/tree/main/arro3-io) _`arro3-io`_ - [bed-reader](https://github.com/fastlmm/bed-reader) _Read and write the PLINK BED format, simply and efficiently._ - Shows Rayon/ndarray::parallel (including capturing errors, controlling thread num), Python types to Rust generics, Github Actions +- [blake3-py](https://github.com/oconnor663/blake3-py) _Python bindings for the [BLAKE3](https://github.com/BLAKE3-team/BLAKE3) cryptographic hash function._ + - Parallelized [builds](https://github.com/oconnor663/blake3-py/blob/master/.github/workflows/dists.yml) on GitHub Actions for MacOS, Linux, Windows, including free-threaded 3.13t wheels. +- [cellular_raza](https://cellular-raza.com) _A cellular agent-based simulation framework for building complex models from a clean slate._ +- [connector-x](https://github.com/sfu-db/connector-x/tree/main/connectorx-python) _Fastest library to load data from DB to DataFrames in Rust and Python._ - [cryptography](https://github.com/pyca/cryptography/tree/main/src/rust) _Python cryptography library with some functionality in Rust._ - [css-inline](https://github.com/Stranger6667/css-inline/tree/master/bindings/python) _CSS inlining for Python implemented in Rust._ - [datafusion-python](https://github.com/apache/arrow-datafusion-python) _A Python library that binds to Apache Arrow in-memory query engine DataFusion._ @@ -191,35 +199,46 @@ about this topic. - [fastbloom](https://github.com/yankun1992/fastbloom) _A fast [bloom filter](https://github.com/yankun1992/fastbloom#BloomFilter) | [counting bloom filter](https://github.com/yankun1992/fastbloom#countingbloomfilter) implemented by Rust for Rust and Python!_ - [fastuuid](https://github.com/thedrow/fastuuid/) _Python bindings to Rust's UUID library._ - [feos](https://github.com/feos-org/feos) _Lightning fast thermodynamic modeling in Rust with fully developed Python interface._ +- [finalytics](https://github.com/Nnamdi-sys/finalytics) _Investment Analysis library in Rust | Python._ - [forust](https://github.com/jinlow/forust) _A lightweight gradient boosted decision tree library written in Rust._ +- [geo-index](https://github.com/kylebarron/geo-index) _A Rust crate and [Python library](https://github.com/kylebarron/geo-index/tree/main/python) for packed, immutable, zero-copy spatial indexes._ +- [granian](https://github.com/emmett-framework/granian) _A Rust HTTP server for Python applications._ - [haem](https://github.com/BooleanCat/haem) _A Python library for working on Bioinformatics problems._ +- [html2text-rs](https://github.com/deedy5/html2text_rs) _Python library for converting HTML to markup or plain text._ - [html-py-ever](https://github.com/PyO3/setuptools-rust/tree/main/examples/html-py-ever) _Using [html5ever](https://github.com/servo/html5ever) through [kuchiki](https://github.com/kuchiki-rs/kuchiki) to speed up html parsing and css-selecting._ -- [hyperjson](https://github.com/mre/hyperjson) _A hyper-fast Python module for reading/writing JSON data using Rust's serde-json._ -- [inline-python](https://github.com/fusion-engineering/inline-python) _Inline Python code directly in your Rust code._ +- [hudi-rs](https://github.com/apache/hudi-rs) _The native Rust implementation for Apache Hudi, with C++ & Python API bindings._ +- [inline-python](https://github.com/m-ou-se/inline-python) _Inline Python code directly in your Rust code._ - [johnnycanencrypt](https://github.com/kushaldas/johnnycanencrypt) OpenPGP library with Yubikey support. -- [jsonschema-rs](https://github.com/Stranger6667/jsonschema-rs/tree/master/bindings/python) _Fast JSON Schema validation library._ +- [jsonschema](https://github.com/Stranger6667/jsonschema/tree/master/crates/jsonschema-py) _A high-performance JSON Schema validator for Python._ - [mocpy](https://github.com/cds-astro/mocpy) _Astronomical Python library offering data structures for describing any arbitrary coverage regions on the unit sphere._ +- [obstore](https://github.com/developmentseed/obstore) _The simplest, highest-throughput Python interface to Amazon S3, Google Cloud Storage, Azure Storage, & other S3-compliant APIs, powered by Rust._ - [opendal](https://github.com/apache/opendal/tree/main/bindings/python) _A data access layer that allows users to easily and efficiently retrieve data from various storage services in a unified way._ - [orjson](https://github.com/ijl/orjson) _Fast Python JSON library._ - [ormsgpack](https://github.com/aviramha/ormsgpack) _Fast Python msgpack library._ -- [point-process](https://github.com/ManifoldFR/point-process-rust/tree/master/pylib) _High level API for pointprocesses as a Python library._ -- [polaroid](https://github.com/daggy1234/polaroid) _Hyper Fast and safe image manipulation library for Python written in Rust._ - [polars](https://github.com/pola-rs/polars) _Fast multi-threaded DataFrame library in Rust | Python | Node.js._ +- [pycrdt](https://github.com/jupyter-server/pycrdt) _Python bindings for the Rust CRDT implementation [Yrs](https://github.com/y-crdt/y-crdt)._ - [pydantic-core](https://github.com/pydantic/pydantic-core) _Core validation logic for pydantic written in Rust._ -- [pyheck](https://github.com/kevinheavey/pyheck) _Fast case conversion library, built by wrapping [heck](https://github.com/withoutboats/heck)._ - - Quite easy to follow as there's not much code. -- [pyre](https://github.com/Project-Dream-Weaver/pyre-http) _Fast Python HTTP server written in Rust._ -- [ril-py](https://github.com/Cryptex-github/ril-py) _A performant and high-level image processing library for Python written in Rust._ +- [primp](https://github.com/deedy5/primp) _The fastest python HTTP client that can impersonate web browsers by mimicking their headers and TLS/JA3/JA4/HTTP2 fingerprints._ +- [rateslib](https://github.com/attack68/rateslib) _A fixed income library for Python using Rust extensions._ - [river](https://github.com/online-ml/river) _Online machine learning in python, the computationally heavy statistics algorithms are implemented in Rust._ +- [robyn](https://github.com/sparckles/Robyn) A Super Fast Async Python Web Framework with a Rust runtime. - [rust-python-coverage](https://github.com/cjermain/rust-python-coverage) _Example PyO3 project with automated test coverage for Rust and Python._ +- [rnet](https://github.com/0x676e67/rnet) Asynchronous Python HTTP Client with Black Magic +- [sail](https://github.com/lakehq/sail) _Unifying stream, batch, and AI workloads with Apache Spark compatibility._ - [tiktoken](https://github.com/openai/tiktoken) _A fast BPE tokeniser for use with OpenAI's models._ - [tokenizers](https://github.com/huggingface/tokenizers/tree/main/bindings/python) _Python bindings to the Hugging Face tokenizers (NLP) written in Rust._ - [tzfpy](http://github.com/ringsaturn/tzfpy) _A fast package to convert longitude/latitude to timezone name._ - [utiles](https://github.com/jessekrubin/utiles) _Fast Python web-map tile utilities_ -- [wasmer-python](https://github.com/wasmerio/wasmer-python) _Python library to run WebAssembly binaries._ ## Articles and other media +- [(Video) Using Rust in Free-Threaded vs Regular Python 3.13](https://www.youtube.com/watch?v=J7phN_M4GLM) - Jun 4, 2025 +- [(Video) Techniques learned from five years finding the way for Rust in Python](https://www.youtube.com/watch?v=KTQn_PTHNCw) - Feb 26, 2025 +- [(Podcast) Bridging Python and Rust: An Interview with PyO3 Maintainer David Hewitt](https://www.youtube.com/watch?v=P47JUMSQagU) - Aug 30, 2024 +- [(Video) PyO3: From Python to Rust and Back Again](https://www.youtube.com/watch?v=UmL_CA-v3O8) - Jul 3, 2024 +- [Parsing Python ASTs 20x Faster with Rust](https://www.gauge.sh/blog/parsing-python-asts-20x-faster-with-rust) - Jun 17, 2024 +- [(Video) How Python Harnesses Rust through PyO3](https://www.youtube.com/watch?v=UilujdubqVU) - May 18, 2024 +- [(Video) Combining Rust and Python: The Best of Both Worlds?](https://www.youtube.com/watch?v=lyG6AKzu4ew) - Mar 1, 2024 - [(Video) Extending Python with Rust using PyO3](https://www.youtube.com/watch?v=T45ZEmSR1-s) - Dec 16, 2023 - [A Week of PyO3 + rust-numpy (How to Speed Up Your Data Pipeline X Times)](https://terencezl.github.io/blog/2023/06/06/a-week-of-pyo3-rust-numpy/) - Jun 6, 2023 - [(Podcast) PyO3 with David Hewitt](https://rustacean-station.org/episode/david-hewitt/) - May 19, 2023 @@ -230,14 +249,14 @@ about this topic. - [Calling Rust from Python using PyO3](https://saidvandeklundert.net/learn/2021-11-18-calling-rust-from-python-using-pyo3/) - Nov 18, 2021 - [davidhewitt's 2021 talk at Rust Manchester meetup](https://www.youtube.com/watch?v=-XyWG_klSAw&t=320s) - Aug 19, 2021 - [Incrementally porting a small Python project to Rust](https://blog.waleedkhan.name/port-python-to-rust/) - Apr 29, 2021 -- [Vortexa - Integrating Rust into Python](https://www.vortexa.com/insight/integrating-rust-into-python) - Apr 12, 2021 +- [Vortexa - Integrating Rust into Python](https://www.vortexa.com/blog/integrating-rust-into-python) - Apr 12, 2021 - [Writing and publishing a Python module in Rust](https://blog.yossarian.net/2020/08/02/Writing-and-publishing-a-python-module-in-rust) - Aug 2, 2020 ## Contributing Everyone is welcomed to contribute to PyO3! There are many ways to support the project, such as: -- help PyO3 users with issues on GitHub and Gitter +- help PyO3 users with issues on GitHub and [Discord](https://discord.gg/33kcChzH7f) - improve documentation - write features and bugfixes - publish blogs and examples of how to use PyO3 diff --git a/Releasing.md b/Releasing.md index 545783c598c..d3a1b4cf8c4 100644 --- a/Releasing.md +++ b/Releasing.md @@ -44,9 +44,10 @@ Wait a couple of days in case anyone wants to hold up the release to add bugfixe ## 4. Put live To put live: -- 1. run `nox -s publish` to put live on crates.io -- 2. publish the release on Github -- 3. merge the release PR +- 1. merge the release PR +- 2. publish a release on GitHub targeting the release branch + +CI will automatically push to `crates.io`. ## 5. Tidy the main branch diff --git a/branding/favicon/pyo3_16x16.png b/branding/favicon/pyo3_16x16.png new file mode 100644 index 00000000000..0d2d77eb151 Binary files /dev/null and b/branding/favicon/pyo3_16x16.png differ diff --git a/branding/favicon/pyo3_32x32.png b/branding/favicon/pyo3_32x32.png new file mode 100644 index 00000000000..ff1f97ae269 Binary files /dev/null and b/branding/favicon/pyo3_32x32.png differ diff --git a/branding/pyo3logo.png b/branding/pyo3logo.png new file mode 100644 index 00000000000..06cad61734e Binary files /dev/null and b/branding/pyo3logo.png differ diff --git a/branding/pyo3logo.svg b/branding/pyo3logo.svg new file mode 100644 index 00000000000..0315c63e56a --- /dev/null +++ b/branding/pyo3logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/branding/pyotr.png b/branding/pyotr.png new file mode 100644 index 00000000000..75ab9bb34b8 Binary files /dev/null and b/branding/pyotr.png differ diff --git a/branding/pyotr.svg b/branding/pyotr.svg new file mode 100644 index 00000000000..52d478e67c4 --- /dev/null +++ b/branding/pyotr.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/build.rs b/build.rs index e20206310db..25e3f54e331 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,9 @@ use std::env; use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result}; -use pyo3_build_config::{bail, print_feature_cfgs, InterpreterConfig}; +use pyo3_build_config::{ + add_python_framework_link_args, bail, print_feature_cfgs, InterpreterConfig, +}; fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> { if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() && !interpreter_config.shared { @@ -16,7 +18,7 @@ fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<( \n\ For more information, see \ https://pyo3.rs/v{pyo3_version}/\ - building_and_distribution.html#embedding-python-in-rust", + building-and-distribution.html#embedding-python-in-rust", pyo3_version = env::var("CARGO_PKG_VERSION").unwrap() ); } @@ -36,16 +38,19 @@ fn configure_pyo3() -> Result<()> { ensure_auto_initialize_ok(interpreter_config)?; for cfg in interpreter_config.build_script_outputs() { - println!("{}", cfg) + println!("{cfg}") } - // Emit cfgs like `thread_local_const_init` print_feature_cfgs(); + // Make `cargo test` etc work on macOS with Xcode bundled Python + add_python_framework_link_args(); + Ok(()) } fn main() { + pyo3_build_config::print_expected_cfgs(); if let Err(e) = configure_pyo3() { eprintln!("error: {}", e.report()); std::process::exit(1) diff --git a/codecov.yml b/codecov.yml index d30f0ff47dc..356facdc3ff 100644 --- a/codecov.yml +++ b/codecov.yml @@ -5,8 +5,12 @@ coverage: project: default: target: auto - # Allow a tiny drop of overall project coverage in PR to reduce spurious failures. - threshold: 0.25% + # Allow a tiny drop of overall project coverage in PR due to + # not all configurations being tested in PR runs. + # + # (Note that patch coverage will still be required to be at least + # the project coverage.) + threshold: 1% ignore: - tests/ diff --git a/emscripten/Makefile b/emscripten/Makefile index af224854c26..54094382c1b 100644 --- a/emscripten/Makefile +++ b/emscripten/Makefile @@ -4,7 +4,7 @@ CURDIR=$(abspath .) BUILDROOT ?= $(CURDIR)/builddir PYMAJORMINORMICRO ?= 3.11.0 -EMSCRIPTEN_VERSION=3.1.13 +EMSCRIPTEN_VERSION=3.1.68 export EMSDKDIR = $(BUILDROOT)/emsdk diff --git a/emscripten/pybuilddir.txt b/emscripten/pybuilddir.txt deleted file mode 100644 index 59f2a4a7546..00000000000 --- a/emscripten/pybuilddir.txt +++ /dev/null @@ -1 +0,0 @@ -build/lib.linux-x86_64-3.11 \ No newline at end of file diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e54b3b5cde2..68aa8be7a76 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -3,6 +3,7 @@ name = "pyo3-examples" version = "0.0.0" publish = false edition = "2021" +rust-version = "1.83" [dev-dependencies] pyo3 = { path = "..", features = ["auto-initialize", "extension-module"] } @@ -10,5 +11,5 @@ pyo3 = { path = "..", features = ["auto-initialize", "extension-module"] } [[example]] name = "decorator" path = "decorator/src/lib.rs" -crate_type = ["cdylib"] +crate-type = ["cdylib"] doc-scrape-examples = true diff --git a/examples/README.md b/examples/README.md index baaa57b650d..3c7cc301399 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,9 +9,11 @@ Below is a brief description of each of these: | `decorator` | A project showcasing the example from the [Emulating callable objects](https://pyo3.rs/latest/class/call.html) chapter of the guide. | | `maturin-starter` | A template project which is configured to use [`maturin`](https://github.com/PyO3/maturin) for development. | | `setuptools-rust-starter` | A template project which is configured to use [`setuptools_rust`](https://github.com/PyO3/setuptools-rust/) for development. | -| `word-count` | A quick performance comparison between word counter implementations written in each of Rust and Python. | | `plugin` | Illustrates how to use Python as a scripting language within a Rust application | -| `sequential` | Illustrates how to use pyo3-ffi to write subinterpreter-safe modules | + +Note that there are also other examples in the `pyo3-ffi/examples` +directory that illustrate how to create rust extensions using raw FFI calls into +the CPython C API instead of using PyO3's abstractions. ## Creating new projects from these examples diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 12b203c3bb4..10287310615 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.27.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/decorator/Cargo.toml b/examples/decorator/Cargo.toml index 3456302a9fd..ef7e5dc2e1c 100644 --- a/examples/decorator/Cargo.toml +++ b/examples/decorator/Cargo.toml @@ -2,6 +2,7 @@ name = "decorator" version = "0.1.0" edition = "2021" +rust-version = "1.83" [lib] name = "decorator" diff --git a/examples/decorator/src/lib.rs b/examples/decorator/src/lib.rs index fb2f2932dd2..69915ff8256 100644 --- a/examples/decorator/src/lib.rs +++ b/examples/decorator/src/lib.rs @@ -1,6 +1,6 @@ use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; -use std::cell::Cell; +use std::sync::atomic::{AtomicU64, Ordering}; /// A function decorator that keeps track how often it is called. /// @@ -9,8 +9,8 @@ use std::cell::Cell; pub struct PyCounter { // Keeps track of how many calls have gone through. // - // See the discussion at the end for why `Cell` is used. - count: Cell, + // See the discussion at the end for why `AtomicU64` is used. + count: AtomicU64, // This is the actual function being wrapped. wraps: Py, @@ -26,32 +26,30 @@ impl PyCounter { #[new] fn __new__(wraps: Py) -> Self { PyCounter { - count: Cell::new(0), + count: AtomicU64::new(0), wraps, } } #[getter] fn count(&self) -> u64 { - self.count.get() + self.count.load(Ordering::Relaxed) } #[pyo3(signature = (*args, **kwargs))] fn __call__( &self, py: Python<'_>, - args: &PyTuple, - kwargs: Option>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - let old_count = self.count.get(); - let new_count = old_count + 1; - self.count.set(new_count); + let new_count = self.count.fetch_add(1, Ordering::Relaxed); let name = self.wraps.getattr(py, "__name__")?; - println!("{} has been called {} time(s).", name, new_count); + println!("{name} has been called {new_count} time(s)."); // After doing something, we finally forward the call to the wrapped function - let ret = self.wraps.call_bound(py, args, kwargs.as_ref())?; + let ret = self.wraps.call(py, args, kwargs)?; // We could do something with the return value of // the function before returning it @@ -60,7 +58,7 @@ impl PyCounter { } #[pymodule] -pub fn decorator(_py: Python<'_>, module: &PyModule) -> PyResult<()> { +pub fn decorator(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; Ok(()) } diff --git a/examples/getitem/Cargo.toml b/examples/getitem/Cargo.toml index 17020b9bd05..c31047eba24 100644 --- a/examples/getitem/Cargo.toml +++ b/examples/getitem/Cargo.toml @@ -2,6 +2,7 @@ name = "getitem" version = "0.1.0" edition = "2021" +rust-version = "1.83" [lib] name = "getitem" diff --git a/examples/getitem/src/lib.rs b/examples/getitem/src/lib.rs index 90a3e9fc52f..5739b67eb31 100644 --- a/examples/getitem/src/lib.rs +++ b/examples/getitem/src/lib.rs @@ -2,12 +2,11 @@ use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; use pyo3::types::PySlice; -use std::os::raw::c_long; #[derive(FromPyObject)] enum IntOrSlice<'py> { Int(i32), - Slice(&'py PySlice), + Slice(Bound<'py, PySlice>), } #[pyclass] @@ -23,13 +22,13 @@ impl ExampleContainer { ExampleContainer { max_length: 100 } } - fn __getitem__(&self, key: &PyAny) -> PyResult { + fn __getitem__(&self, key: &Bound<'_, PyAny>) -> PyResult { if let Ok(position) = key.extract::() { return Ok(position); - } else if let Ok(slice) = key.downcast::() { + } else if let Ok(slice) = key.cast::() { // METHOD 1 - the use PySliceIndices to help with bounds checking and for cases when only start or end are provided // in this case the start/stop/step all filled in to give valid values based on the max_length given - let index = slice.indices(self.max_length as c_long).unwrap(); + let index = slice.indices(self.max_length as isize).unwrap(); let _delta = index.stop - index.start; // METHOD 2 - Do the getattr manually really only needed if you have some special cases for stop/_step not being present @@ -62,8 +61,11 @@ impl ExampleContainer { fn __setitem__(&self, idx: IntOrSlice, value: u32) -> PyResult<()> { match idx { IntOrSlice::Slice(slice) => { - let index = slice.indices(self.max_length as c_long).unwrap(); - println!("Got a slice! {}-{}, step: {}, value: {}", index.start, index.stop, index.step, value); + let index = slice.indices(self.max_length as isize).unwrap(); + println!( + "Got a slice! {}-{}, step: {}, value: {}", + index.start, index.stop, index.step, value + ); } IntOrSlice::Int(index) => { println!("Got an index! {} : value: {}", index, value); @@ -73,9 +75,8 @@ impl ExampleContainer { } } -#[pymodule] -#[pyo3(name = "getitem")] -fn example(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(name = "getitem")] +fn example(m: &Bound<'_, PyModule>) -> PyResult<()> { // ? -https://github.com/PyO3/maturin/issues/475 m.add_class::()?; Ok(()) diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 12b203c3bb4..10287310615 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.27.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/Cargo.toml b/examples/maturin-starter/Cargo.toml index 257908a4bb0..53e464bbffe 100644 --- a/examples/maturin-starter/Cargo.toml +++ b/examples/maturin-starter/Cargo.toml @@ -2,6 +2,7 @@ name = "maturin-starter" version = "0.1.0" edition = "2021" +rust-version = "1.83" [lib] name = "maturin_starter" diff --git a/examples/maturin-starter/src/lib.rs b/examples/maturin-starter/src/lib.rs index 96ace0f97d5..5e0be857391 100644 --- a/examples/maturin-starter/src/lib.rs +++ b/examples/maturin-starter/src/lib.rs @@ -20,7 +20,7 @@ impl ExampleClass { /// An example module implemented in Rust using PyO3. #[pymodule] -fn maturin_starter(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn maturin_starter(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_wrapped(wrap_pymodule!(submodule::submodule))?; @@ -28,7 +28,7 @@ fn maturin_starter(py: Python<'_>, m: &PyModule) -> PyResult<()> { // e.g. from maturin_starter.submodule import SubmoduleClass let sys = PyModule::import(py, "sys")?; - let sys_modules: &PyDict = sys.getattr("modules")?.downcast()?; + let sys_modules: Bound<'_, PyDict> = sys.getattr("modules")?.cast_into()?; sys_modules.set_item("maturin_starter.submodule", m.getattr("submodule")?)?; Ok(()) diff --git a/examples/maturin-starter/src/submodule.rs b/examples/maturin-starter/src/submodule.rs index 56540b2e469..f3eb174100b 100644 --- a/examples/maturin-starter/src/submodule.rs +++ b/examples/maturin-starter/src/submodule.rs @@ -16,7 +16,7 @@ impl SubmoduleClass { } #[pymodule] -pub fn submodule(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn submodule(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 72cfe2be91d..bb9d2c4f3b5 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.27.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/plugin/Cargo.toml b/examples/plugin/Cargo.toml index 08127b5003f..ab342767f2d 100644 --- a/examples/plugin/Cargo.toml +++ b/examples/plugin/Cargo.toml @@ -2,11 +2,11 @@ name = "plugin_example" version = "0.1.0" edition = "2021" - +rust-version = "1.83" [dependencies] -pyo3={path="../../", features=["macros"]} -plugin_api={path="plugin_api"} +pyo3 = { path = "../../", features = ["macros"] } +plugin_api = { path = "plugin_api" } [workspace] diff --git a/examples/plugin/plugin_api/src/lib.rs b/examples/plugin/plugin_api/src/lib.rs index 59aae55699d..580c85a8c8e 100644 --- a/examples/plugin/plugin_api/src/lib.rs +++ b/examples/plugin/plugin_api/src/lib.rs @@ -26,7 +26,7 @@ impl Gadget { /// A Python module for plugin interface types #[pymodule] -pub fn plugin_api(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn plugin_api(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/examples/plugin/src/main.rs b/examples/plugin/src/main.rs index b50b54548e5..677d9e37b53 100644 --- a/examples/plugin/src/main.rs +++ b/examples/plugin/src/main.rs @@ -7,13 +7,13 @@ fn main() -> Result<(), Box> { //"export" our API module to the python runtime pyo3::append_to_inittab!(pylib_module); //spawn runtime - pyo3::prepare_freethreaded_python(); + Python::initialize(); //import path for python let path = Path::new("./python_plugin/"); //do useful work - Python::with_gil(|py| { + Python::attach(|py| { //add the current directory to import path of Python (do not use this in production!) - let syspath: &PyList = py.import("sys")?.getattr("path")?.extract()?; + let syspath: Bound = py.import("sys")?.getattr("path")?.extract()?; syspath.insert(0, &path)?; println!("Import path is: {:?}", syspath); diff --git a/examples/sequential/Cargo.toml b/examples/sequential/Cargo.toml deleted file mode 100644 index 4500c69b597..00000000000 --- a/examples/sequential/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "sequential" -version = "0.1.0" -edition = "2021" - -[lib] -name = "sequential" -crate-type = ["cdylib", "lib"] - -[dependencies] -pyo3-ffi = { path = "../../pyo3-ffi", features = ["extension-module"] } - -[workspace] diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 78a656558f8..f6c8aca02a9 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.27.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/Cargo.toml b/examples/setuptools-rust-starter/Cargo.toml index 5777cbbcd78..e7ae56a7612 100644 --- a/examples/setuptools-rust-starter/Cargo.toml +++ b/examples/setuptools-rust-starter/Cargo.toml @@ -2,6 +2,7 @@ name = "setuptools-rust-starter" version = "0.1.0" edition = "2021" +rust-version = "1.83" [lib] name = "setuptools_rust_starter" diff --git a/examples/setuptools-rust-starter/noxfile.py b/examples/setuptools-rust-starter/noxfile.py index 9edab96254b..d3579912ea7 100644 --- a/examples/setuptools-rust-starter/noxfile.py +++ b/examples/setuptools-rust-starter/noxfile.py @@ -1,10 +1,11 @@ import nox +import sys @nox.session def python(session: nox.Session): - session.install("-rrequirements-dev.txt") - session.run_always( - "pip", "install", "-e", ".", "--no-build-isolation", env={"BUILD_DEBUG": "1"} - ) + if sys.version_info < (3, 9): + session.skip("Python 3.9 or later is required for setuptools-rust 1.11") + session.env["SETUPTOOLS_RUST_CARGO_PROFILE"] = "dev" + session.install(".[dev]") session.run("pytest") diff --git a/examples/setuptools-rust-starter/pyproject.toml b/examples/setuptools-rust-starter/pyproject.toml index d82653c1701..bc4c7d2bc98 100644 --- a/examples/setuptools-rust-starter/pyproject.toml +++ b/examples/setuptools-rust-starter/pyproject.toml @@ -1,2 +1,25 @@ [build-system] -requires = ["setuptools>=41.0.0", "wheel", "setuptools_rust>=1.0.0"] +requires = ["setuptools>=62.4", "setuptools_rust>=1.11"] + +[project] +name = "setuptools-rust-starter" +version = "0.1.0" +classifiers = [ + "License :: OSI Approved :: MIT License", + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Rust", + "Operating System :: POSIX", + "Operating System :: MacOS :: MacOS X", +] + +[project.optional-dependencies] +dev = ["pytest"] + +[tool.setuptools.packages.find] +include = ["setuptools_rust_starter"] + +[[tool.setuptools-rust.ext-modules]] +target = "setuptools_rust_starter._setuptools_rust_starter" +path = "Cargo.toml" diff --git a/examples/setuptools-rust-starter/setup.cfg b/examples/setuptools-rust-starter/setup.cfg deleted file mode 100644 index 95caeb757e6..00000000000 --- a/examples/setuptools-rust-starter/setup.cfg +++ /dev/null @@ -1,17 +0,0 @@ -[metadata] -name = setuptools-rust-starter -version = 0.1.0 -classifiers = - License :: OSI Approved :: MIT License - Development Status :: 3 - Alpha - Intended Audience :: Developers - Programming Language :: Python - Programming Language :: Rust - Operating System :: POSIX - Operating System :: MacOS :: MacOS X - -[options] -packages = - setuptools_rust_starter -include_package_data = True -zip_safe = False diff --git a/examples/setuptools-rust-starter/setup.py b/examples/setuptools-rust-starter/setup.py deleted file mode 100644 index 85d9e21d3a4..00000000000 --- a/examples/setuptools-rust-starter/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -import os - -from setuptools import setup -from setuptools_rust import RustExtension - -setup( - rust_extensions=[ - RustExtension( - "setuptools_rust_starter._setuptools_rust_starter", - debug=os.environ.get("BUILD_DEBUG") == "1", - ) - ], -) diff --git a/examples/setuptools-rust-starter/src/lib.rs b/examples/setuptools-rust-starter/src/lib.rs index fbfeccc1555..e4ff036d850 100644 --- a/examples/setuptools-rust-starter/src/lib.rs +++ b/examples/setuptools-rust-starter/src/lib.rs @@ -20,7 +20,7 @@ impl ExampleClass { /// An example module implemented in Rust using PyO3. #[pymodule] -fn _setuptools_rust_starter(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn _setuptools_rust_starter(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_wrapped(wrap_pymodule!(submodule::submodule))?; @@ -28,7 +28,7 @@ fn _setuptools_rust_starter(py: Python<'_>, m: &PyModule) -> PyResult<()> { // e.g. from setuptools_rust_starter.submodule import SubmoduleClass let sys = PyModule::import(py, "sys")?; - let sys_modules: &PyDict = sys.getattr("modules")?.downcast()?; + let sys_modules: Bound<'_, PyDict> = sys.getattr("modules")?.cast_into()?; sys_modules.set_item("setuptools_rust_starter.submodule", m.getattr("submodule")?)?; Ok(()) diff --git a/examples/setuptools-rust-starter/src/submodule.rs b/examples/setuptools-rust-starter/src/submodule.rs index 56540b2e469..f3eb174100b 100644 --- a/examples/setuptools-rust-starter/src/submodule.rs +++ b/examples/setuptools-rust-starter/src/submodule.rs @@ -16,7 +16,7 @@ impl SubmoduleClass { } #[pymodule] -pub fn submodule(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn submodule(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/examples/string-sum/Cargo.toml b/examples/string-sum/Cargo.toml deleted file mode 100644 index 4a48b221c60..00000000000 --- a/examples/string-sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "string_sum" -version = "0.1.0" -edition = "2021" - -[lib] -name = "string_sum" -crate-type = ["cdylib"] - -[dependencies] -pyo3-ffi = { path = "../../pyo3-ffi", features = ["extension-module"] } - -[workspace] diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 12b203c3bb4..10287310615 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.27.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/word-count/Cargo.toml b/examples/word-count/Cargo.toml index cfb3444d5fe..816e3dd9a34 100644 --- a/examples/word-count/Cargo.toml +++ b/examples/word-count/Cargo.toml @@ -2,6 +2,7 @@ name = "word-count" version = "0.1.0" edition = "2021" +rust-version = "1.83" [lib] name = "word_count" diff --git a/examples/word-count/noxfile.py b/examples/word-count/noxfile.py index d64f210f3e5..c9dd6fce159 100644 --- a/examples/word-count/noxfile.py +++ b/examples/word-count/noxfile.py @@ -12,6 +12,5 @@ def test(session: nox.Session): @nox.session def bench(session: nox.Session): - session.env["MATURIN_PEP517_ARGS"] = "--profile=dev" session.install(".[dev]") session.run("pytest", "--benchmark-enable") diff --git a/examples/word-count/src/lib.rs b/examples/word-count/src/lib.rs index b7d3a8033a6..baa9cf18e57 100644 --- a/examples/word-count/src/lib.rs +++ b/examples/word-count/src/lib.rs @@ -17,8 +17,8 @@ fn search_sequential(contents: &str, needle: &str) -> usize { } #[pyfunction] -fn search_sequential_allow_threads(py: Python<'_>, contents: &str, needle: &str) -> usize { - py.allow_threads(|| search_sequential(contents, needle)) +fn search_sequential_detached(py: Python<'_>, contents: &str, needle: &str) -> usize { + py.detach(|| search_sequential(contents, needle)) } /// Count the occurrences of needle in line, case insensitive @@ -33,10 +33,10 @@ fn count_line(line: &str, needle: &str) -> usize { } #[pymodule] -fn word_count(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn word_count(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(search, m)?)?; m.add_function(wrap_pyfunction!(search_sequential, m)?)?; - m.add_function(wrap_pyfunction!(search_sequential_allow_threads, m)?)?; + m.add_function(wrap_pyfunction!(search_sequential_detached, m)?)?; Ok(()) } diff --git a/examples/word-count/tests/test_word_count.py b/examples/word-count/tests/test_word_count.py index 5991e4ae1de..5b665d8e29c 100644 --- a/examples/word-count/tests/test_word_count.py +++ b/examples/word-count/tests/test_word_count.py @@ -50,12 +50,8 @@ def test_word_count_python_sequential(benchmark, contents): def run_rust_sequential_twice( executor: ThreadPoolExecutor, contents: str, needle: str ) -> int: - future_1 = executor.submit( - word_count.search_sequential_allow_threads, contents, needle - ) - future_2 = executor.submit( - word_count.search_sequential_allow_threads, contents, needle - ) + future_1 = executor.submit(word_count.search_sequential_detached, contents, needle) + future_2 = executor.submit(word_count.search_sequential_detached, contents, needle) result_1 = future_1.result() result_2 = future_2.result() return result_1 + result_2 diff --git a/examples/word-count/word_count/__init__.py b/examples/word-count/word_count/__init__.py index 8ce7a175471..4a7f1b5ee5e 100644 --- a/examples/word-count/word_count/__init__.py +++ b/examples/word-count/word_count/__init__.py @@ -1,10 +1,10 @@ -from .word_count import search, search_sequential, search_sequential_allow_threads +from .word_count import search, search_sequential, search_sequential_detached __all__ = [ "search_py", "search", "search_sequential", - "search_sequential_allow_threads", + "search_sequential_detached", ] diff --git a/guide/book.toml b/guide/book.toml index 31fa4bb1587..be682a64eab 100644 --- a/guide/book.toml +++ b/guide/book.toml @@ -6,7 +6,11 @@ author = "PyO3 Project and Contributors" [preprocessor.pyo3_version] command = "python3 guide/pyo3_version.py" +[preprocessor.tabs] + [output.html] git-repository-url = "https://github.com/PyO3/pyo3/tree/main/guide" edit-url-template = "https://github.com/PyO3/pyo3/edit/main/guide/%7Bpath%7D" -playground.runnable = false \ No newline at end of file +playground.runnable = false +additional-css = ["theme/tabs.css"] +additional-js = ["theme/tabs.js"] diff --git a/guide/pyclass_parameters.md b/guide/pyclass-parameters.md similarity index 59% rename from guide/pyclass_parameters.md rename to guide/pyclass-parameters.md index 35c54147df5..2c9efe5c259 100644 --- a/guide/pyclass_parameters.md +++ b/guide/pyclass-parameters.md @@ -2,21 +2,30 @@ | Parameter | Description | | :- | :- | +| `constructor` | This is currently only allowed on [variants of complex enums][params-constructor]. It allows customization of the generated class constructor for each variant. It uses the same syntax and supports the same options as the `signature` attribute of functions and methods. | | `crate = "some::path"` | Path to import the `pyo3` crate, if it's not accessible at `::pyo3`. | | `dict` | Gives instances of this class an empty `__dict__` to store custom attributes. | +| `eq` | Implements `__eq__` using the `PartialEq` implementation of the underlying Rust datatype. | +| `eq_int` | Implements `__eq__` using `__int__` for simple enums. | | `extends = BaseType` | Use a custom baseclass. Defaults to [`PyAny`][params-1] | | `freelist = N` | Implements a [free list][params-2] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | +| `from_py_object` | Implement `FromPyObject` for this pyclass. Requires the pyclass to be `Clone`. | | `frozen` | Declares that your pyclass is immutable. It removes the borrow checker overhead when retrieving a shared reference to the Rust struct, but disables the ability to get a mutable reference. | +| `generic` | Implements runtime parametrization for the class following [PEP 560](https://peps.python.org/pep-0560/). | | `get_all` | Generates getters for all fields of the pyclass. | +| `hash` | Implements `__hash__` using the `Hash` implementation of the underlying Rust datatype. *Requires `eq` and `frozen`* | +| `immutable_type` | Makes the type object immutable. Supported on 3.14+ with the `abi3` feature active, or 3.10+ otherwise. | | `mapping` | Inform PyO3 that this class is a [`Mapping`][params-mapping], and so leave its implementation of sequence C-API slots empty. | | `module = "module_name"` | Python code will see the class as being defined in this module. Defaults to `builtins`. | | `name = "python_name"` | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. | +| `ord` | Implements `__lt__`, `__gt__`, `__le__`, & `__ge__` using the `PartialOrd` implementation of the underlying Rust datatype. *Requires `eq`* | | `rename_all = "renaming_rule"` | Applies renaming rules to every getters and setters of a struct, or every variants of an enum. Possible values are: "camelCase", "kebab-case", "lowercase", "PascalCase", "SCREAMING-KEBAB-CASE", "SCREAMING_SNAKE_CASE", "snake_case", "UPPERCASE". | | `sequence` | Inform PyO3 that this class is a [`Sequence`][params-sequence], and so leave its C-API mapping length slot empty. | | `set_all` | Generates setters for all fields of the pyclass. | +| `skip_from_py_object` | Prevents this PyClass from participating in the `FromPyObject: PyClass + Clone` blanket implementation. This allows a custom `FromPyObject` impl, even if `self` is `Clone`. | +| `str` | Implements `__str__` using the `Display` implementation of the underlying Rust datatype or by passing an optional format string `str=""`. *Note: The optional format string is only allowed for structs. `name` and `rename_all` are incompatible with the optional format string. Additional details can be found in the discussion on this [PR](https://github.com/PyO3/pyo3/pull/4233).* | | `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. Enums cannot be subclassed. | -| `text_signature = "(arg1, arg2, ...)"` | Sets the text signature for the Python class' `__new__` method. | -| `unsendable` | Required if your struct is not [`Send`][params-3]. Rather than using `unsendable`, consider implementing your struct in a threadsafe way by e.g. substituting [`Rc`][params-4] with [`Arc`][params-5]. By using `unsendable`, your class will panic when accessed by another thread. Also note the Python's GC is multi-threaded and while unsendable classes will not be traversed on foreign threads to avoid UB, this can lead to memory leaks. | +| `unsendable` | Required if your struct is not [`Send`][params-3]. Rather than using `unsendable`, consider implementing your struct in a thread-safe way by e.g. substituting [`Rc`][params-4] with [`Arc`][params-5]. By using `unsendable`, your class will panic when accessed by another thread. Also note the Python's GC is multi-threaded and while unsendable classes will not be traversed on foreign threads to avoid UB, this can lead to memory leaks. | | `weakref` | Allows this class to be [weakly referenceable][params-6]. | All of these parameters can either be passed directly on the `#[pyclass(...)]` annotation, or as one or @@ -33,11 +42,12 @@ struct MyClass {} struct MyClass {} ``` -[params-1]: https://docs.rs/pyo3/latest/pyo3/struct.PyAny.html +[params-1]: https://docs.rs/pyo3/latest/pyo3/types/struct.PyAny.html [params-2]: https://en.wikipedia.org/wiki/Free_list [params-3]: https://doc.rust-lang.org/std/marker/trait.Send.html [params-4]: https://doc.rust-lang.org/std/rc/struct.Rc.html [params-5]: https://doc.rust-lang.org/std/sync/struct.Arc.html [params-6]: https://docs.python.org/3/library/weakref.html +[params-constructor]: https://pyo3.rs/latest/class.html#complex-enums [params-mapping]: https://pyo3.rs/latest/class/protocols.html#mapping--sequence-types [params-sequence]: https://pyo3.rs/latest/class/protocols.html#mapping--sequence-types diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 34775a0d185..2f863231984 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -4,41 +4,47 @@ --- -- [Getting started](getting_started.md) -- [Python modules](module.md) -- [Python functions](function.md) - - [Function signatures](function/signature.md) - - [Error handling](function/error_handling.md) -- [Python classes](class.md) - - [Class customizations](class/protocols.md) - - [Basic object customization](class/object.md) - - [Emulating numeric types](class/numeric.md) - - [Emulating callable objects](class/call.md) +- [Getting started](getting-started.md) +- [Using Rust from Python](rust-from-python.md) + - [Python modules](module.md) + - [Python functions](function.md) + - [Function signatures](function/signature.md) + - [Error handling](function/error-handling.md) + - [Python classes](class.md) + - [Class customizations](class/protocols.md) + - [Basic object customization](class/object.md) + - [Emulating numeric types](class/numeric.md) + - [Emulating callable objects](class/call.md) + - [Thread safety](class/thread-safety.md) +- [Calling Python from Rust](python-from-rust.md) + - [Python object types](types.md) + - [Python exceptions](exception.md) + - [Calling Python functions](python-from-rust/function-calls.md) + - [Executing existing Python code](python-from-rust/calling-existing-code.md) - [Type conversions](conversions.md) - [Mapping of Rust types to Python types](conversions/tables.md) - [Conversion traits](conversions/traits.md) -- [Python exceptions](exception.md) -- [Calling Python from Rust](python_from_rust.md) - [Using `async` and `await`](async-await.md) -- [GIL, mutability and object types](types.md) - [Parallelism](parallelism.md) +- [Supporting Free-Threaded Python](free-threading.md) - [Debugging](debugging.md) - [Features reference](features.md) -- [Memory management](memory.md) - [Performance](performance.md) +- [Type stub generation and introspection](type-stub.md) - [Advanced topics](advanced.md) -- [Building and distribution](building_and_distribution.md) - - [Supporting multiple Python versions](building_and_distribution/multiple_python_versions.md) +- [Building and distribution](building-and-distribution.md) + - [Supporting multiple Python versions](building-and-distribution/multiple-python-versions.md) - [Useful crates](ecosystem.md) - [Logging](ecosystem/logging.md) + - [Tracing](ecosystem/tracing.md) - [Using `async` and `await`](ecosystem/async-await.md) - [FAQ and troubleshooting](faq.md) --- [Appendix A: Migration guide](migration.md) -[Appendix B: Trait bounds](trait_bounds.md) -[Appendix C: Python typing hints](python_typing_hints.md) +[Appendix B: Trait bounds](trait-bounds.md) +[Appendix C: Python typing hints](python-typing-hints.md) [CHANGELOG](changelog.md) --- diff --git a/guide/src/advanced.md b/guide/src/advanced.md index 8264c14dd17..cd630c4f9da 100644 --- a/guide/src/advanced.md +++ b/guide/src/advanced.md @@ -4,11 +4,5 @@ PyO3 exposes much of Python's C API through the `ffi` module. -The C API is naturally unsafe and requires you to manage reference counts, errors and specific invariants yourself. Please refer to the [C API Reference Manual](https://docs.python.org/3/c-api/) and [The Rustonomicon](https://doc.rust-lang.org/nightly/nomicon/ffi.html) before using any function from that API. - -## Memory management - -PyO3's `&PyAny` "owned references" and `Py` smart pointers are used to -access memory stored in Python's heap. This memory sometimes lives for longer -than expected because of differences in Rust and Python's memory models. See -the chapter on [memory management](./memory.md) for more information. +The C API is naturally unsafe and requires you to manage reference counts, errors and specific invariants yourself. +Please refer to the [C API Reference Manual](https://docs.python.org/3/c-api/) and [The Rustonomicon](https://doc.rust-lang.org/nightly/nomicon/ffi.html) before using any function from that API. diff --git a/guide/src/async-await.md b/guide/src/async-await.md index c14b5d93d84..c6a6c9a00e9 100644 --- a/guide/src/async-await.md +++ b/guide/src/async-await.md @@ -4,14 +4,16 @@ `#[pyfunction]` and `#[pymethods]` attributes also support `async fn`. -```rust +```rust,no_run # #![allow(dead_code)] +# #[cfg(feature = "experimental-async")] { use std::{thread, time::Duration}; use futures::channel::oneshot; use pyo3::prelude::*; #[pyfunction] -async fn sleep(seconds: f64, result: Option) -> Option { +#[pyo3(signature=(seconds, result=None))] +async fn sleep(seconds: f64, result: Option>) -> Option> { let (tx, rx) = oneshot::channel(); thread::spawn(move || { thread::sleep(Duration::from_secs_f64(seconds)); @@ -20,32 +22,41 @@ async fn sleep(seconds: f64, result: Option) -> Option { rx.await.unwrap(); result } +# } ``` -*Python awaitables instantiated with this method can only be awaited in *asyncio* context. Other Python async runtime may be supported in the future.* +*Python awaitables instantiated with this method can only be awaited in `asyncio` context. Other Python async runtime may be supported in the future.* ## `Send + 'static` constraint Resulting future of an `async fn` decorated by `#[pyfunction]` must be `Send + 'static` to be embedded in a Python object. -As a consequence, `async fn` parameters and return types must also be `Send + 'static`, so it is not possible to have a signature like `async fn does_not_compile(arg: &PyAny, py: Python<'_>) -> &PyAny`. +As a consequence, `async fn` parameters and return types must also be `Send + 'static`, so it is not possible to have a signature like `async fn does_not_compile<'py>(arg: Bound<'py, PyAny>) -> Bound<'py, PyAny>`. -However, there is an exception for method receiver, so async methods can accept `&self`/`&mut self`. Note that this means that the class instance is borrowed for as long as the returned future is not completed, even across yield points and while waiting for I/O operations to complete. Hence, other methods cannot obtain exclusive borrows while the future is still being polled. This is the same as how async methods in Rust generally work but it is more problematic for Rust code interfacing with Python code due to pervasive shared mutability. This strongly suggests to prefer shared borrows `&self` to exclusive ones `&mut self` to avoid racy borrow check failures at runtime. +However, there is an exception for method receivers, so async methods can accept `&self`/ `&mut self`. +Note that this means that the class instance is borrowed for as long as the returned future is not completed, even across yield points and while waiting for I/O operations to complete. +Hence, other methods cannot obtain exclusive borrows while the future is still being polled. +This is the same as how async methods in Rust generally work but it is more problematic for Rust code interfacing with Python code due to pervasive shared mutability. +This strongly suggests to prefer shared borrows `&self` over exclusive ones `&mut self` to avoid racy borrow check failures at runtime. -## Implicit GIL holding +## Implicitly attached to the interpreter -Even if it is not possible to pass a `py: Python<'_>` parameter to `async fn`, the GIL is still held during the execution of the future – it's also the case for regular `fn` without `Python<'_>`/`&PyAny` parameter, yet the GIL is held. +Even if it is not possible to pass a `py: Python<'py>` token to an `async fn`, we're still attached to the interpreter during the execution of the future – the same as for a regular `fn` without `Python<'py>`/`Bound<'py, PyAny>` parameter -It is still possible to get a `Python` marker using [`Python::with_gil`]({{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.with_gil); because `with_gil` is reentrant and optimized, the cost will be negligible. +It is still possible to get a `Python` marker using [`Python::attach`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.attach); because `attach` is reentrant and optimized, the cost will be negligible. -## Release the GIL across `.await` +## Detaching from the interpreter across `.await` -There is currently no simple way to release the GIL when awaiting a future, *but solutions are currently in development*. +There is currently no simple way to detach from the interpreter when awaiting a future, *but solutions are currently in development*. Here is the advised workaround for now: ```rust,ignore -use std::{future::Future, pin::{Pin, pin}, task::{Context, Poll}}; +use std::{ + future::Future, + pin::{Pin, pin}, + task::{Context, Poll}, +}; use pyo3::prelude::*; struct AllowThreads(F); @@ -59,8 +70,8 @@ where fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let waker = cx.waker(); - Python::with_gil(|gil| { - gil.allow_threads(|| pin!(&mut self.0).poll(&mut Context::from_waker(waker))) + Python::attach(|py| { + py.detach(|| pin!(&mut self.0).poll(&mut Context::from_waker(waker))) }) } } @@ -68,10 +79,11 @@ where ## Cancellation -Cancellation on the Python side can be caught using [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) type, by annotating a function parameter with `#[pyo3(cancel_handle)]. +Cancellation on the Python side can be caught using [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) type, by annotating a function parameter with `#[pyo3(cancel_handle)]`. -```rust +```rust,no_run # #![allow(dead_code)] +# #[cfg(feature = "experimental-async")] { use futures::FutureExt; use pyo3::prelude::*; use pyo3::coroutine::CancelHandle; @@ -83,12 +95,14 @@ async fn cancellable(#[pyo3(cancel_handle)] mut cancel: CancelHandle) { _ = cancel.cancelled().fuse() => println!("cancelled"), } } +# } ``` ## The `Coroutine` type -To make a Rust future awaitable in Python, PyO3 defines a [`Coroutine`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.Coroutine.html) type, which implements the Python [coroutine protocol](https://docs.python.org/3/library/collections.abc.html#collections.abc.Coroutine). +To make a Rust future awaitable in Python, PyO3 defines a [`Coroutine`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.Coroutine.html) type, which implements the Python [coroutine protocol](https://docs.python.org/3/library/collections.abc.html#collections.abc.Coroutine). -Each `coroutine.send` call is translated to a `Future::poll` call. If a [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) parameter is declared, the exception passed to `coroutine.throw` call is stored in it and can be retrieved with [`CancelHandle::cancelled`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html#method.cancelled); otherwise, it cancels the Rust future, and the exception is reraised; +Each `coroutine.send` call is translated to a `Future::poll` call. +If a [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) parameter is declared, the exception passed to `coroutine.throw` call is stored in it and can be retrieved with [`CancelHandle::cancelled`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html#method.cancelled); otherwise, it cancels the Rust future, and the exception is reraised; *The type does not yet have a public constructor until the design is finalized.* diff --git a/guide/src/building_and_distribution.md b/guide/src/building-and-distribution.md similarity index 51% rename from guide/src/building_and_distribution.md rename to guide/src/building-and-distribution.md index 97e77d24c34..e12dc18803b 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building-and-distribution.md @@ -1,23 +1,30 @@ # Building and distribution -This chapter of the guide goes into detail on how to build and distribute projects using PyO3. The way to achieve this is very different depending on whether the project is a Python module implemented in Rust, or a Rust binary embedding Python. For both types of project there are also common problems such as the Python version to build for and the [linker](https://en.wikipedia.org/wiki/Linker_(computing)) arguments to use. +This chapter of the guide goes into detail on how to build and distribute projects using PyO3. +The way to achieve this is very different depending on whether the project is a Python module implemented in Rust, or a Rust binary embedding Python. +For both types of project there are also common problems such as the Python version to build for and the [linker](https://en.wikipedia.org/wiki/Linker_(computing)) arguments to use. -The material in this chapter is intended for users who have already read the PyO3 [README](./index.md). It covers in turn the choices that can be made for Python modules and for Rust binaries. There is also a section at the end about cross-compiling projects using PyO3. +The material in this chapter is intended for users who have already read the PyO3 [README](./index.md). +It covers in turn the choices that can be made for Python modules and for Rust binaries. +There is also a section at the end about cross-compiling projects using PyO3. -There is an additional sub-chapter dedicated to [supporting multiple Python versions](./building_and_distribution/multiple_python_versions.html). +There is an additional sub-chapter dedicated to [supporting multiple Python versions](./building-and-distribution/multiple-python-versions.md). ## Configuring the Python version -PyO3 uses a build script (backed by the [`pyo3-build-config`] crate) to determine the Python version and set the correct linker arguments. By default it will attempt to use the following in order: - - Any active Python virtualenv. - - The `python` executable (if it's a Python 3 interpreter). - - The `python3` executable. +PyO3 uses a build script (backed by the [`pyo3-build-config`] crate) to determine the Python version and set the correct linker arguments. +By default it will attempt to use the following in order: + +- Any active Python virtualenv. +- The `python` executable (if it's a Python 3 interpreter). +- The `python3` executable. You can override the Python interpreter by setting the `PYO3_PYTHON` environment variable, e.g. `PYO3_PYTHON=python3.7`, `PYO3_PYTHON=/usr/bin/python3.9`, or even a PyPy interpreter `PYO3_PYTHON=pypy3`. Once the Python interpreter is located, `pyo3-build-config` executes it to query the information in the `sysconfig` module which is needed to configure the rest of the compilation. -To validate the configuration which PyO3 will use, you can run a compilation with the environment variable `PYO3_PRINT_CONFIG=1` set. An example output of doing this is shown below: +To validate the configuration which PyO3 will use, you can run a compilation with the environment variable `PYO3_PRINT_CONFIG=1` set. +An example output of doing this is shown below: ```console $ PYO3_PRINT_CONFIG=1 cargo build @@ -51,27 +58,40 @@ The `PYO3_ENVIRONMENT_SIGNATURE` environment variable can be used to trigger reb If you save the above output config from `PYO3_PRINT_CONFIG` to a file, it is possible to manually override the contents and feed it back into PyO3 using the `PYO3_CONFIG_FILE` env var. -If your build environment is unusual enough that PyO3's regular configuration detection doesn't work, using a config file like this will give you the flexibility to make PyO3 work for you. To see the full set of options supported, see the documentation for the [`InterpreterConfig` struct](https://docs.rs/pyo3-build-config/{{#PYO3_DOCS_VERSION}}/pyo3_build_config/struct.InterpreterConfig.html). +If your build environment is unusual enough that PyO3's regular configuration detection doesn't work, using a config file like this will give you the flexibility to make PyO3 work for you. +To see the full set of options supported, see the documentation for the [`InterpreterConfig` struct](https://docs.rs/pyo3-build-config/{{#PYO3_DOCS_VERSION}}/pyo3_build_config/struct.InterpreterConfig.html). ## Building Python extension modules -Python extension modules need to be compiled differently depending on the OS (and architecture) that they are being compiled for. As well as multiple OSes (and architectures), there are also many different Python versions which are actively supported. Packages uploaded to [PyPI](https://pypi.org/) usually want to upload prebuilt "wheels" covering many OS/arch/version combinations so that users on all these different platforms don't have to compile the package themselves. Package vendors can opt-in to the "abi3" limited Python API which allows their wheels to be used on multiple Python versions, reducing the number of wheels they need to compile, but restricts the functionality they can use. +Python extension modules need to be compiled differently depending on the OS (and architecture) that they are being compiled for. +As well as multiple OSes (and architectures), there are also many different Python versions which are actively supported. +Packages uploaded to [PyPI](https://pypi.org/) usually want to upload prebuilt "wheels" covering many OS/arch/version combinations so that users on all these different platforms don't have to compile the package themselves. +Package vendors can opt-in to the "abi3" limited Python API which allows their wheels to be used on multiple Python versions, reducing the number of wheels they need to compile, but restricts the functionality they can use. -There are many ways to go about this: it is possible to use `cargo` to build the extension module (along with some manual work, which varies with OS). The PyO3 ecosystem has two packaging tools, [`maturin`] and [`setuptools-rust`], which abstract over the OS difference and also support building wheels for PyPI upload. +There are many ways to go about this: it is possible to use `cargo` to build the extension module (along with some manual work, which varies with OS). +The PyO3 ecosystem has two packaging tools, [`maturin`] and [`setuptools-rust`], which abstract over the OS difference and also support building wheels for PyPI upload. PyO3 has some Cargo features to configure projects for building Python extension modules: - - The `extension-module` feature, which must be enabled when building Python extension modules. - - The `abi3` feature and its version-specific `abi3-pyXY` companions, which are used to opt-in to the limited Python API in order to support multiple Python versions in a single wheel. -This section describes each of these packaging tools before describing how to build manually without them. It then proceeds with an explanation of the `extension-module` feature. Finally, there is a section describing PyO3's `abi3` features. +- The `extension-module` feature, which must be enabled when building Python extension modules. +- The `abi3` feature and its version-specific `abi3-pyXY` companions, which are used to opt-in to the limited Python API in order to support multiple Python versions in a single wheel. + +This section describes each of these packaging tools before describing how to build manually without them. +It then proceeds with an explanation of the `extension-module` feature. +Finally, there is a section describing PyO3's `abi3` features. ### Packaging tools The PyO3 ecosystem has two main choices to abstract the process of developing Python extension modules: -- [`maturin`] is a command-line tool to build, package and upload Python modules. It makes opinionated choices about project layout meaning it needs very little configuration. This makes it a great choice for users who are building a Python extension from scratch and don't need flexibility. -- [`setuptools-rust`] is an add-on for `setuptools` which adds extra keyword arguments to the `setup.py` configuration file. It requires more configuration than `maturin`, however this gives additional flexibility for users adding Rust to an existing Python package that can't satisfy `maturin`'s constraints. -Consult each project's documentation for full details on how to get started using them and how to upload wheels to PyPI. It should be noted that while `maturin` is able to build [manylinux](https://github.com/pypa/manylinux)-compliant wheels out-of-the-box, `setuptools-rust` requires a bit more effort, [relying on Docker](https://setuptools-rust.readthedocs.io/en/latest/building_wheels.html) for this purpose. +- [`maturin`] is a command-line tool to build, package and upload Python modules. + It makes opinionated choices about project layout meaning it needs very little configuration. + This makes it a great choice for users who are building a Python extension from scratch and don't need flexibility. +- [`setuptools-rust`] is an add-on for `setuptools` which adds extra keyword arguments to the `setup.py` configuration file. + It requires more configuration than `maturin`, however this gives additional flexibility for users adding Rust to an existing Python package that can't satisfy `maturin`'s constraints. + +Consult each project's documentation for full details on how to get started using them and how to upload wheels to PyPI. +It should be noted that while `maturin` is able to build [manylinux](https://github.com/pypa/manylinux)-compliant wheels out-of-the-box, `setuptools-rust` requires a bit more effort, [relying on Docker](https://setuptools-rust.readthedocs.io/en/latest/building_wheels.html) for this purpose. There are also [`maturin-starter`] and [`setuptools-rust-starter`] examples in the PyO3 repository. @@ -80,32 +100,40 @@ There are also [`maturin-starter`] and [`setuptools-rust-starter`] examples in t To build a PyO3-based Python extension manually, start by running `cargo build` as normal in a library project which uses PyO3's `extension-module` feature and has the [`cdylib` crate type](https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-crate-type-field). Once built, symlink (or copy) and rename the shared library from Cargo's `target/` directory to your desired output directory: + - on macOS, rename `libyour_module.dylib` to `your_module.so`. - on Windows, rename `libyour_module.dll` to `your_module.pyd`. - on Linux, rename `libyour_module.so` to `your_module.so`. You can then open a Python shell in the output directory and you'll be able to run `import your_module`. -If you're packaging your library for redistribution, you should indicated the Python interpreter your library is compiled for by including the [platform tag](#platform-tags) in its name. This prevents incompatible interpreters from trying to import your library. If you're compiling for PyPy you *must* include the platform tag, or PyPy will ignore the module. +If you're packaging your library for redistribution, you should indicate the Python interpreter your library is compiled for by including the [platform tag](#platform-tags) in its name. +This prevents incompatible interpreters from trying to import your library. +If you're compiling for PyPy you *must* include the platform tag, or PyPy will ignore the module. #### Bazel builds -To use PyO3 with bazel one needs to manually configure PyO3, PyO3-ffi and PyO3-macros. In particular, one needs to make sure that it is compiled with the right python flags for the version you intend to use. +To use PyO3 with bazel one needs to manually configure PyO3, PyO3-ffi and PyO3-macros. +In particular, one needs to make sure that it is compiled with the right python flags for the version you intend to use. For example see: -1. https://github.com/OliverFM/pytorch_with_gazelle -- for a minimal example of a repo that can use PyO3, PyTorch and Gazelle to generate python Build files. -2. https://github.com/TheButlah/rules_pyo3 -- which has more extensive support, but is outdated. + +1. [github.com/abrisco/rules_pyo3](https://github.com/abrisco/rules_pyo3) -- General rules for building extension modules. +2. [github.com/OliverFM/pytorch_with_gazelle](https://github.com/OliverFM/pytorch_with_gazelle) -- for a minimal example of a repo that can use PyO3, PyTorch and Gazelle to generate python Build files. +3. [github.com/TheButlah/rules_pyo3](https://github.com/TheButlah/rules_pyo3) -- is somewhat dated. #### Platform tags -Rather than using just the `.so` or `.pyd` extension suggested above (depending on OS), you can prefix the shared library extension with a platform tag to indicate the interpreter it is compatible with. You can query your interpreter's platform tag from the `sysconfig` module. Some example outputs of this are seen below: +Rather than using just the `.so` or `.pyd` extension suggested above (depending on OS), you can prefix the shared library extension with a platform tag to indicate the interpreter it is compatible with. +You can query your interpreter's platform tag from the `sysconfig` module. +Some example outputs of this are seen below: ```bash # CPython 3.10 on macOS .cpython-310-darwin.so -# PyPy 7.3 (Python 3.8) on Linux +# PyPy 7.3 (Python 3.9) on Linux $ python -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))' -.pypy38-pp73-x86_64-linux-gnu.so +.pypy39-pp73-x86_64-linux-gnu.so ``` So, for example, a valid module library name on CPython 3.10 for macOS is `your_module.cpython-310-darwin.so`, and its equivalent when compiled for PyPy 7.3 on Linux would be `your_module.pypy38-pp73-x86_64-linux-gnu.so`. @@ -142,7 +170,17 @@ rustflags = [ ] ``` -Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. These can be resolved with another addition to `.cargo/config.toml`: +Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. + +The easiest way to set the correct linker arguments is to add a `build.rs` with the following content: + +```rust,ignore +fn main() { + pyo3_build_config::add_python_framework_link_args(); +} +``` + +Alternatively it can be resolved with another addition to `.cargo/config.toml`: ```toml [build] @@ -151,56 +189,61 @@ rustflags = [ ] ``` -Alternatively, on rust >= 1.56, one can include in `build.rs`: - -```rust -fn main() { - println!( - "cargo:rustc-link-arg=-Wl,-rpath,/Library/Developer/CommandLineTools/Library/Frameworks" - ); -} -``` - For more discussion on and workarounds for MacOS linking problems [see this issue](https://github.com/PyO3/pyo3/issues/1800#issuecomment-906786649). -Finally, don't forget that on MacOS the `extension-module` feature will cause `cargo test` to fail without the `--no-default-features` flag (see [the FAQ](https://pyo3.rs/main/faq.html#i-cant-run-cargo-test-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror)). +Finally, don't forget that on MacOS the `extension-module` feature will cause `cargo test` to fail without the `--no-default-features` flag (see [the FAQ](https://pyo3.rs/main/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror)). ### The `extension-module` feature PyO3's `extension-module` feature is used to disable [linking](https://en.wikipedia.org/wiki/Linker_(computing)) to `libpython` on Unix targets. -This is necessary because by default PyO3 links to `libpython`. This makes binaries, tests, and examples "just work". However, Python extensions on Unix must not link to libpython for [manylinux](https://www.python.org/dev/peps/pep-0513/) compliance. +This is necessary because by default PyO3 links to `libpython`. +This makes binaries, tests, and examples "just work". +However, Python extensions on Unix must not link to libpython for [manylinux](https://www.python.org/dev/peps/pep-0513/) compliance. -The downside of not linking to `libpython` is that binaries, tests, and examples (which usually embed Python) will fail to build. If you have an extension module as well as other outputs in a single project, you need to use optional Cargo features to disable the `extension-module` when you're not building the extension module. See [the FAQ](faq.md#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror) for an example workaround. +The downside of not linking to `libpython` is that binaries, tests, and examples (which usually embed Python) will fail to build. +If you have an extension module as well as other outputs in a single project, you need to use optional Cargo features to disable the `extension-module` when you're not building the extension module. +See [the FAQ](faq.md#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror) for an example workaround. ### `Py_LIMITED_API`/`abi3` -By default, Python extension modules can only be used with the same Python version they were compiled against. For example, an extension module built for Python 3.5 can't be imported in Python 3.8. [PEP 384](https://www.python.org/dev/peps/pep-0384/) introduced the idea of the limited Python API, which would have a stable ABI enabling extension modules built with it to be used against multiple Python versions. This is also known as `abi3`. +By default, Python extension modules can only be used with the same Python version they were compiled against. +For example, an extension module built for Python 3.5 can't be imported in Python 3.8. [PEP 384](https://www.python.org/dev/peps/pep-0384/) introduced the idea of the limited Python API, which would have a stable ABI enabling extension modules built with it to be used against multiple Python versions. +This is also known as `abi3`. -The advantage of building extension modules using the limited Python API is that package vendors only need to build and distribute a single copy (for each OS / architecture), and users can install it on all Python versions from the [minimum version](#minimum-python-version-for-abi3) and up. The downside of this is that PyO3 can't use optimizations which rely on being compiled against a known exact Python version. It's up to you to decide whether this matters for your extension module. It's also possible to design your extension module such that you can distribute `abi3` wheels but allow users compiling from source to benefit from additional optimizations - see the [support for multiple python versions](./building_and_distribution/multiple_python_versions.html) section of this guide, in particular the `#[cfg(Py_LIMITED_API)]` flag. +The advantage of building extension modules using the limited Python API is that package vendors only need to build and distribute a single copy (for each OS / architecture), and users can install it on all Python versions from the [minimum version](#minimum-python-version-for-abi3) and up. +The downside of this is that PyO3 can't use optimizations which rely on being compiled against a known exact Python version. +It's up to you to decide whether this matters for your extension module. +It's also possible to design your extension module such that you can distribute `abi3` wheels but allow users compiling from source to benefit from additional optimizations - see the [support for multiple python versions](./building-and-distribution/multiple-python-versions.md) section of this guide, in particular the `#[cfg(Py_LIMITED_API)]` flag. There are three steps involved in making use of `abi3` when building Python packages as wheels: -1. Enable the `abi3` feature in `pyo3`. This ensures `pyo3` only calls Python C-API functions which are part of the stable API, and on Windows also ensures that the project links against the correct shared object (no special behavior is required on other platforms): +1. Enable the `abi3` feature in `pyo3`. + This ensures `pyo3` only calls Python C-API functions which are part of the stable API, and on Windows also ensures that the project links against the correct shared object (no special behavior is required on other platforms): -```toml -[dependencies] -pyo3 = { {{#PYO3_CRATE_VERSION}}, features = ["abi3"] } -``` + ```toml + [dependencies] + pyo3 = { {{#PYO3_CRATE_VERSION}}, features = ["abi3"] } + ``` -2. Ensure that the built shared objects are correctly marked as `abi3`. This is accomplished by telling your build system that you're using the limited API. [`maturin`] >= 0.9.0 and [`setuptools-rust`] >= 0.11.4 support `abi3` wheels. -See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https://github.com/PyO3/setuptools-rust/pull/82) for more. +2. Ensure that the built shared objects are correctly marked as `abi3`. + This is accomplished by telling your build system that you're using the limited API. [`maturin`] >= 0.9.0 and [`setuptools-rust`] >= 0.11.4 support `abi3` wheels. -3. Ensure that the `.whl` is correctly marked as `abi3`. For projects using `setuptools`, this is accomplished by passing `--py-limited-api=cp3x` (where `x` is the minimum Python version supported by the wheel, e.g. `--py-limited-api=cp35` for Python 3.5) to `setup.py bdist_wheel`. + See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https://github.com/PyO3/setuptools-rust/pull/82) for more. + +3. Ensure that the `.whl` is correctly marked as `abi3`. + For projects using `setuptools`, this is accomplished by passing `--py-limited-api=cp3x` (where `x` is the minimum Python version supported by the wheel, e.g. `--py-limited-api=cp35` for Python 3.5) to `setup.py bdist_wheel`. #### Minimum Python version for `abi3` Because a single `abi3` wheel can be used with many different Python versions, PyO3 has feature flags `abi3-py37`, `abi3-py38`, `abi3-py39` etc. to set the minimum required Python version for your `abi3` wheel. For example, if you set the `abi3-py37` feature, your extension wheel can be used on all Python 3 versions from Python 3.7 and up. `maturin` and `setuptools-rust` will give the wheel a name like `my-extension-1.0-cp37-abi3-manylinux2020_x86_64.whl`. -As your extension module may be run with multiple different Python versions you may occasionally find you need to check the Python version at runtime to customize behavior. See [the relevant section of this guide](./building_and_distribution/multiple_python_versions.html#checking-the-python-version-at-runtime) on supporting multiple Python versions at runtime. +As your extension module may be run with multiple different Python versions you may occasionally find you need to check the Python version at runtime to customize behavior. +See [the relevant section of this guide](./building-and-distribution/multiple-python-versions.md#checking-the-python-version-at-runtime) on supporting multiple Python versions at runtime. -PyO3 is only able to link your extension module to abi3 version up to and including your host Python version. E.g., if you set `abi3-py38` and try to compile the crate with a host of Python 3.7, the build will fail. +PyO3 is only able to link your extension module to abi3 version up to and including your host Python version. +E.g., if you set `abi3-py38` and try to compile the crate with a host of Python 3.7, the build will fail. > Note: If you set more that one of these `abi3` version feature flags the lowest version always wins. For example, with both `abi3-py37` and `abi3-py38` set, PyO3 would build a wheel which supports Python 3.7 and up. @@ -220,8 +263,8 @@ the automatic import library generation feature to work. #### Missing features -Due to limitations in the Python API, there are a few `pyo3` features that do -not work when compiling for `abi3`. These are: +Due to limitations in the Python API, there are a few `pyo3` features that do not work when compiling for `abi3`. +These are: - `#[pyo3(text_signature = "...")]` does not work on classes until Python 3.10 or greater. - The `dict` and `weakref` options on classes are not supported until Python 3.9 or greater. @@ -230,35 +273,49 @@ not work when compiling for `abi3`. These are: ## Embedding Python in Rust -If you want to embed the Python interpreter inside a Rust program, there are two modes in which this can be done: dynamically and statically. We'll cover each of these modes in the following sections. Each of them affect how you must distribute your program. Instead of learning how to do this yourself, you might want to consider using a project like [PyOxidizer] to ship your application and all of its dependencies in a single file. +If you want to embed the Python interpreter inside a Rust program, there are two modes in which this can be done: dynamically and statically. +We'll cover each of these modes in the following sections. +Each of them affect how you must distribute your program. +Instead of learning how to do this yourself, you might want to consider using a project like [PyOxidizer] to ship your application and all of its dependencies in a single file. -PyO3 automatically switches between the two linking modes depending on whether the Python distribution you have configured PyO3 to use ([see above](#python-version)) contains a shared library or a static library. The static library is most often seen in Python distributions compiled from source without the `--enable-shared` configuration option. For example, this is the default for `pyenv` on macOS. +PyO3 automatically switches between the two linking modes depending on whether the Python distribution you have configured PyO3 to use ([see above](#configuring-the-python-version)) contains a shared library or a static library. +The static library is most often seen in Python distributions compiled from source without the `--enable-shared` configuration option. ### Dynamically embedding the Python interpreter -Embedding the Python interpreter dynamically is much easier than doing so statically. This is done by linking your program against a Python shared library (such as `libpython.3.9.so` on UNIX, or `python39.dll` on Windows). The implementation of the Python interpreter resides inside the shared library. This means that when the OS runs your Rust program it also needs to be able to find the Python shared library. +Embedding the Python interpreter dynamically is much easier than doing so statically. +This is done by linking your program against a Python shared library (such as `libpython.3.9.so` on UNIX, or `python39.dll` on Windows). +The implementation of the Python interpreter resides inside the shared library. +This means that when the OS runs your Rust program it also needs to be able to find the Python shared library. -This mode of embedding works well for Rust tests which need access to the Python interpreter. It is also great for Rust software which is installed inside a Python virtualenv, because the virtualenv sets up appropriate environment variables to locate the correct Python shared library. +This mode of embedding works well for Rust tests which need access to the Python interpreter. +It is also great for Rust software which is installed inside a Python virtualenv, because the virtualenv sets up appropriate environment variables to locate the correct Python shared library. For distributing your program to non-technical users, you will have to consider including the Python shared library in your distribution as well as setting up wrapper scripts to set the right environment variables (such as `LD_LIBRARY_PATH` on UNIX, or `PATH` on Windows). -Note that PyPy cannot be embedded in Rust (or any other software). Support for this is tracked on the [PyPy issue tracker](https://foss.heptapod.net/pypy/pypy/-/issues/3286). +Note that PyPy cannot be embedded in Rust (or any other software). +Support for this is tracked on the [PyPy issue tracker](https://github.com/pypy/pypy/issues/3836). ### Statically embedding the Python interpreter -Embedding the Python interpreter statically means including the contents of a Python static library directly inside your Rust binary. This means that to distribute your program you only need to ship your binary file: it contains the Python interpreter inside the binary! +Embedding the Python interpreter statically means including the contents of a Python static library directly inside your Rust binary. +This means that to distribute your program you only need to ship your binary file: it contains the Python interpreter inside the binary! -On Windows static linking is almost never done, so Python distributions don't usually include a static library. The information below applies only to UNIX. +On Windows static linking is almost never done, so Python distributions don't usually include a static library. +The information below applies only to UNIX. The Python static library is usually called `libpython.a`. -Static linking has a lot of complications, listed below. For these reasons PyO3 does not yet have first-class support for this embedding mode. See [issue 416 on PyO3's GitHub](https://github.com/PyO3/pyo3/issues/416) for more information and to discuss any issues you encounter. +Static linking has a lot of complications, listed below. +For these reasons PyO3 does not yet have first-class support for this embedding mode. +See [issue 416 on PyO3's GitHub](https://github.com/PyO3/pyo3/issues/416) for more information and to discuss any issues you encounter. The [`auto-initialize`](features.md#auto-initialize) feature is deliberately disabled when embedding the interpreter statically because this is often unintentionally done by new users to PyO3 running test programs. Trying out PyO3 is much easier using dynamic embedding. The known complications are: - - To import compiled extension modules (such as other Rust extension modules, or those written in C), your binary must have the correct linker flags set during compilation to export the original contents of `libpython.a` so that extensions can use them (e.g. `-Wl,--export-dynamic`). - - The C compiler and flags which were used to create `libpython.a` must be compatible with your Rust compiler and flags, else you will experience compilation failures. + +- To import compiled extension modules (such as other Rust extension modules, or those written in C), your binary must have the correct linker flags set during compilation to export the original contents of `libpython.a` so that extensions can use them (e.g. `-Wl,--export-dynamic`). +- The C compiler and flags which were used to create `libpython.a` must be compatible with your Rust compiler and flags, else you will experience compilation failures. Significantly different compiler versions may see errors like this: @@ -272,41 +329,43 @@ The known complications are: /usr/bin/ld: /usr/lib/gcc/x86_64-linux-gnu/9/../../../x86_64-linux-gnu/libpython3.9.a(zlibmodule.o): relocation R_X86_64_32 against `.data' can not be used when making a PIE object; recompile with -fPIE ``` -If you encounter these or other complications when linking the interpreter statically, discuss them on [issue 416 on PyO3's GitHub](https://github.com/PyO3/pyo3/issues/416). It is hoped that eventually that discussion will contain enough information and solutions that PyO3 can offer first-class support for static embedding. +If you encounter these or other complications when linking the interpreter statically, discuss them on [issue 416 on PyO3's GitHub](https://github.com/PyO3/pyo3/issues/416). +It is hoped that eventually that discussion will contain enough information and solutions that PyO3 can offer first-class support for static embedding. ### Import your module when embedding the Python interpreter -When you run your Rust binary with an embedded interpreter, any `#[pymodule]` created modules won't be accessible to import unless added to a table called `PyImport_Inittab` before the embedded interpreter is initialized. This will cause Python statements in your embedded interpreter such as `import your_new_module` to fail. You can call the macro [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) with your module before initializing the Python interpreter to add the module function into that table. (The Python interpreter will be initialized by calling `prepare_freethreaded_python`, `with_embedded_python_interpreter`, or `Python::with_gil` with the [`auto-initialize`](features.md#auto-initialize) feature enabled.) +When you run your Rust binary with an embedded interpreter, any `#[pymodule]` created modules won't be accessible to import unless added to a table called `PyImport_Inittab` before the embedded interpreter is initialized. +This will cause Python statements in your embedded interpreter such as `import your_new_module` to fail. +You can call the macro [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) with your module before initializing the Python interpreter to add the module function into that table. (The Python interpreter will be initialized by calling `Python::initialize`, `with_embedded_python_interpreter`, or `Python::attach` with the [`auto-initialize`](features.md#auto-initialize) feature enabled.) ## Cross Compiling -Thanks to Rust's great cross-compilation support, cross-compiling using PyO3 is relatively straightforward. To get started, you'll need a few pieces of software: +Thanks to Rust's great cross-compilation support, cross-compiling using PyO3 is relatively straightforward. +To get started, you'll need a few pieces of software: -* A toolchain for your target. -* The appropriate options in your Cargo `.config` for the platform you're targeting and the toolchain you are using. -* A Python interpreter that's already been compiled for your target (optional when building "abi3" extension modules). -* A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#python-version) variable (optional when building "abi3" extension modules). +- A toolchain for your target. +- The appropriate options in your Cargo `.config` for the platform you're targeting and the toolchain you are using. +- A Python interpreter that's already been compiled for your target (optional when building "abi3" extension modules). +- A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#configuring-the-python-version) variable (optional when building "abi3" extension modules). -After you've obtained the above, you can build a cross-compiled PyO3 module by using Cargo's `--target` flag. PyO3's build script will detect that you are attempting a cross-compile based on your host machine and the desired target. +After you've obtained the above, you can build a cross-compiled PyO3 module by using Cargo's `--target` flag. +PyO3's build script will detect that you are attempting a cross-compile based on your host machine and the desired target. When cross-compiling, PyO3's build script cannot execute the target Python interpreter to query the configuration, so there are a few additional environment variables you may need to set: -* `PYO3_CROSS`: If present this variable forces PyO3 to configure as a cross-compilation. -* `PYO3_CROSS_LIB_DIR`: This variable can be set to the directory containing the target's libpython DSO and the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import libraries for the Windows target. This variable is only needed when the output binary must link to libpython explicitly (e.g. when targeting Windows and Android or embedding a Python interpreter), or when it is absolutely required to get the interpreter configuration from `_sysconfigdata*.py`. -* `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`. -* `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation. CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`. - -An experimental `pyo3` crate feature `generate-import-lib` enables the user to cross-compile -extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment -variable or providing any Windows Python library files. It uses an external [`python3-dll-a`] crate -to generate import libraries for the Python DLL for MinGW-w64 and MSVC compile targets. -`python3-dll-a` uses the binutils `dlltool` program to generate DLL import libraries for MinGW-w64 targets. -It is possible to override the default `dlltool` command name for the cross target -by setting `PYO3_MINGW_DLLTOOL` environment variable. -*Note*: MSVC targets require LLVM binutils or MSVC build tools to be available on the host system. -More specifically, `python3-dll-a` requires `llvm-dlltool` or `lib.exe` executable to be present in `PATH` when -targeting `*-pc-windows-msvc`. The Zig compiler executable can be used in place of `llvm-dlltool` when the `ZIG_COMMAND` -environment variable is set to the installed Zig program name (`"zig"` or `"python -m ziglang"`). +- `PYO3_CROSS`: If present this variable forces PyO3 to configure as a cross-compilation. +- `PYO3_CROSS_LIB_DIR`: This variable can be set to the directory containing the target's libpython DSO and the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import libraries for the Windows target. + This variable is only needed when the output binary must link to libpython explicitly (e.g. when targeting Windows and Android or embedding a Python interpreter), or when it is absolutely required to get the interpreter configuration from `_sysconfigdata*.py`. +- `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. + This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`. +- `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation. + CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`. + +An experimental `pyo3` crate feature `generate-import-lib` enables the user to cross-compile extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment variable or providing any Windows Python library files. +It uses an external [`python3-dll-a`] crate to generate import libraries for the Python DLL for MinGW-w64 and MSVC compile targets. `python3-dll-a` uses the binutils `dlltool` program to generate DLL import libraries for MinGW-w64 targets. +It is possible to override the default `dlltool` command name for the cross target by setting `PYO3_MINGW_DLLTOOL` environment variable. *Note*: MSVC targets require LLVM binutils or MSVC build tools to be available on the host system. +More specifically, `python3-dll-a` requires `llvm-dlltool` or `lib.exe` executable to be present in `PATH` when targeting `*-pc-windows-msvc`. +The Zig compiler executable can be used in place of `llvm-dlltool` when the `ZIG_COMMAND` environment variable is set to the installed Zig program name (`"zig"` or `"python -m ziglang"`). An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`): @@ -317,6 +376,7 @@ cargo build --target armv7-unknown-linux-gnueabihf ``` If there are multiple python versions at the cross lib directory and you cannot set a more precise location to include both the `libpython` DSO and `_sysconfigdata*.py` files, you can set the required version: + ```sh export PYO3_CROSS_PYTHON_VERSION=3.8 export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib" @@ -325,6 +385,7 @@ cargo build --target armv7-unknown-linux-gnueabihf ``` Or another example with the same sys root but building for Windows: + ```sh export PYO3_CROSS_PYTHON_VERSION=3.9 export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib" @@ -339,8 +400,9 @@ or when cross compiling extension modules for Windows and the experimental `gene crate feature is enabled. The following resources may also be useful for cross-compiling: - - [github.com/japaric/rust-cross](https://github.com/japaric/rust-cross) is a primer on cross compiling Rust. - - [github.com/rust-embedded/cross](https://github.com/rust-embedded/cross) uses Docker to make Rust cross-compilation easier. + +- [github.com/japaric/rust-cross](https://github.com/japaric/rust-cross) is a primer on cross compiling Rust. +- [github.com/rust-embedded/cross](https://github.com/rust-embedded/cross) uses Docker to make Rust cross-compilation easier. [`pyo3-build-config`]: https://github.com/PyO3/pyo3/tree/main/pyo3-build-config [`maturin-starter`]: https://github.com/PyO3/pyo3/tree/main/examples/maturin-starter diff --git a/guide/src/building_and_distribution/multiple_python_versions.md b/guide/src/building-and-distribution/multiple-python-versions.md similarity index 63% rename from guide/src/building_and_distribution/multiple_python_versions.md rename to guide/src/building-and-distribution/multiple-python-versions.md index 43203686fc8..961c13b7d9c 100644 --- a/guide/src/building_and_distribution/multiple_python_versions.md +++ b/guide/src/building-and-distribution/multiple-python-versions.md @@ -1,14 +1,19 @@ # Supporting multiple Python versions -PyO3 supports all actively-supported Python 3 and PyPy versions. As much as possible, this is done internally to PyO3 so that your crate's code does not need to adapt to the differences between each version. However, as Python features grow and change between versions, PyO3 cannot a completely identical API for every Python version. This may require you to add conditional compilation to your crate or runtime checks for the Python version. +PyO3 supports all actively-supported Python 3 and PyPy versions. +As much as possible, this is done internally to PyO3 so that your crate's code does not need to adapt to the differences between each version. +However, as Python features grow and change between versions, PyO3 cannot offer a completely identical API for every Python version. +This may require you to add conditional compilation to your crate or runtime checks for the Python version. This section of the guide first introduces the `pyo3-build-config` crate, which you can use as a `build-dependency` to add additional `#[cfg]` flags which allow you to support multiple Python versions at compile-time. -Second, we'll show how to check the Python version at runtime. This can be useful when building for multiple versions with the `abi3` feature, where the Python API compiled against is not always the same as the one in use. +Second, we'll show how to check the Python version at runtime. +This can be useful when building for multiple versions with the `abi3` feature, where the Python API compiled against is not always the same as the one in use. ## Conditional compilation for different Python versions -The `pyo3-build-config` exposes multiple [`#[cfg]` flags](https://doc.rust-lang.org/rust-by-example/attribute/cfg.html) which can be used to conditionally compile code for a given Python version. PyO3 itself depends on this crate, so by using it you can be sure that you are configured correctly for the Python version PyO3 is building against. +The `pyo3-build-config` exposes multiple [`#[cfg]` flags](https://doc.rust-lang.org/rust-by-example/attribute/cfg.html) which can be used to conditionally compile code for a given Python version. +PyO3 itself depends on this crate, so by using it you can be sure that you are configured correctly for the Python version PyO3 is building against. This allows us to write code like the following @@ -51,13 +56,15 @@ After these steps you are ready to annotate your code! ### Common usages of `pyo3-build-cfg` flags -The `#[cfg]` flags added by `pyo3-build-cfg` can be combined with all of Rust's logic in the `#[cfg]` attribute to create very precise conditional code generation. The following are some common patterns implemented using these flags: +The `#[cfg]` flags added by `pyo3-build-cfg` can be combined with all of Rust's logic in the `#[cfg]` attribute to create very precise conditional code generation. +The following are some common patterns implemented using these flags: ```text #[cfg(Py_3_7)] ``` -This `#[cfg]` marks code that will only be present on Python 3.7 and upwards. There are similar options `Py_3_8`, `Py_3_9`, `Py_3_10` and so on for each minor version. +This `#[cfg]` marks code that will only be present on Python 3.7 and upwards. +There are similar options `Py_3_8`, `Py_3_9`, `Py_3_10` and so on for each minor version. ```text #[cfg(not(Py_3_7))] @@ -69,13 +76,15 @@ This `#[cfg]` marks code that will only be present on Python versions before (bu #[cfg(not(Py_LIMITED_API))] ``` -This `#[cfg]` marks code that is only available when building for the unlimited Python API (i.e. PyO3's `abi3` feature is not enabled). This might be useful if you want to ship your extension module as an `abi3` wheel and also allow users to compile it from source to make use of optimizations only possible with the unlimited API. +This `#[cfg]` marks code that is only available when building for the unlimited Python API (i.e. PyO3's `abi3` feature is not enabled). +This might be useful if you want to ship your extension module as an `abi3` wheel and also allow users to compile it from source to make use of optimizations only possible with the unlimited API. ```text #[cfg(any(Py_3_9, not(Py_LIMITED_API)))] ``` -This `#[cfg]` marks code which is available when running Python 3.9 or newer, or when using the unlimited API with an older Python version. Patterns like this are commonly seen on Python APIs which were added to the limited Python API in a specific minor version. +This `#[cfg]` marks code which is available when running Python 3.9 or newer, or when using the unlimited API with an older Python version. +Patterns like this are commonly seen on Python APIs which were added to the limited Python API in a specific minor version. ```text #[cfg(PyPy)] @@ -85,24 +94,26 @@ This `#[cfg]` marks code which is running on PyPy. ## Checking the Python version at runtime -When building with PyO3's `abi3` feature, your extension module will be compiled against a specific [minimum version](../building_and_distribution.html#minimum-python-version-for-abi3) of Python, but may be running on newer Python versions. +When building with PyO3's `abi3` feature, your extension module will be compiled against a specific [minimum version](../building-and-distribution.md#minimum-python-version-for-abi3) of Python, but may be running on newer Python versions. -For example with PyO3's `abi3-py38` feature, your extension will be compiled as if it were for Python 3.8. If you were using `pyo3-build-config`, `#[cfg(Py_3_8)]` would be present. Your user could freely install and run your abi3 extension on Python 3.9. +For example with PyO3's `abi3-py38` feature, your extension will be compiled as if it were for Python 3.8. +If you were using `pyo3-build-config`, `#[cfg(Py_3_8)]` would be present. +Your user could freely install and run your abi3 extension on Python 3.9. There's no way to detect your user doing that at compile time, so instead you need to fall back to runtime checks. -PyO3 provides the APIs [`Python::version()`] and [`Python::version_info()`] to query the running Python version. This allows you to do the following, for example: - +PyO3 provides the APIs [`Python::version()`] and [`Python::version_info()`] to query the running Python version. +This allows you to do the following, for example: ```rust use pyo3::Python; -Python::with_gil(|py| { +Python::attach(|py| { // PyO3 supports Python 3.7 and up. assert!(py.version_info() >= (3, 7)); assert!(py.version_info() >= (3, 7, 0)); }); ``` -[`Python::version()`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.version -[`Python::version_info()`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.version_info +[`Python::version()`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.version +[`Python::version_info()`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.version_info diff --git a/guide/src/class.md b/guide/src/class.md index 6ca582ed963..ecc68f3a8a6 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -2,9 +2,11 @@ PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs. -The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` or `enum` to generate a Python type for it. They will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled, each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) `#[pymethods]` may also have implementations for Python magic methods such as `__str__`. +The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` or `enum` to generate a Python type for it. +They will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled, each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) `#[pymethods]` may also have implementations for Python magic methods such as `__str__`. -This chapter will discuss the functionality and configuration these attributes offer. Below is a list of links to the relevant section of this chapter for each: +This chapter will discuss the functionality and configuration these attributes offer. +Below is a list of links to the relevant section of this chapter for each: - [`#[pyclass]`](#defining-a-new-class) - [`#[pyo3(get, set)]`](#object-properties-using-pyo3get-set) @@ -16,12 +18,13 @@ This chapter will discuss the functionality and configuration these attributes o - [`#[classmethod]`](#class-methods) - [`#[classattr]`](#class-attributes) - [`#[args]`](#method-arguments) -- [Magic methods and slots](class/protocols.html) +- [Magic methods and slots](class/protocols.md) - [Classes as function arguments](#classes-as-function-arguments) ## Defining a new class To define a custom Python class, add the `#[pyclass]` attribute to a Rust struct or enum. + ```rust # #![allow(dead_code)] use pyo3::prelude::*; @@ -37,14 +40,16 @@ struct Number(i32); // PyO3 supports unit-only enums (which contain only unit variants) // These simple enums behave similarly to Python's enumerations (enum.Enum) -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum { Variant, OtherVariant = 30, // PyO3 supports custom discriminants. } // PyO3 supports custom discriminants in unit-only enums -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum HttpResponse { Ok = 200, NotFound = 404, @@ -52,44 +57,55 @@ enum HttpResponse { // ... } -// PyO3 also supports enums with non-unit variants -// These complex enums have sligtly different behavior from the simple enums above +// PyO3 also supports enums with Struct and Tuple variants +// These complex enums have slightly different behavior from the simple enums above // They are meant to work with instance checks and match statement patterns +// The variants can be mixed and matched +// Struct variants have named fields while tuple enums generate generic names for fields in order _0, _1, _2, ... +// Apart from this both types are functionally identical #[pyclass] enum Shape { Circle { radius: f64 }, Rectangle { width: f64, height: f64 }, - RegularPolygon { side_count: u32, radius: f64 }, - Nothing { }, + RegularPolygon(u32, f64), + Nothing(), } ``` -The above example generates implementations for [`PyTypeInfo`] and [`PyClass`] for `MyClass`, `Number`, `MyEnum`, `HttpResponse`, and `Shape`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter. +The above example generates implementations for [`PyTypeInfo`] and [`PyClass`] for `MyClass`, `Number`, `MyEnum`, `HttpResponse`, and `Shape`. +To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter. ### Restrictions -To integrate Rust types with Python, PyO3 needs to place some restrictions on the types which can be annotated with `#[pyclass]`. In particular, they must have no lifetime parameters, no generic parameters, and must implement `Send`. The reason for each of these is explained below. +To integrate Rust types with Python, PyO3 needs to place some restrictions on the types which can be annotated with `#[pyclass]`. +In particular, they must have no lifetime parameters, no generic parameters, and must be thread-safe. +The reason for each of these is explained below. #### No lifetime parameters -Rust lifetimes are used by the Rust compiler to reason about a program's memory safety. They are a compile-time only concept; there is no way to access Rust lifetimes at runtime from a dynamic language like Python. +Rust lifetimes are used by the Rust compiler to reason about a program's memory safety. +They are a compile-time only concept; there is no way to access Rust lifetimes at runtime from a dynamic language like Python. -As soon as Rust data is exposed to Python, there is no guarantee that the Rust compiler can make on how long the data will live. Python is a reference-counted language and those references can be held for an arbitrarily long time which is untraceable by the Rust compiler. The only possible way to express this correctly is to require that any `#[pyclass]` does not borrow data for any lifetime shorter than the `'static` lifetime, i.e. the `#[pyclass]` cannot have any lifetime parameters. +As soon as Rust data is exposed to Python, there is no guarantee that the Rust compiler can make on how long the data will live. +Python is a reference-counted language and those references can be held for an arbitrarily long time which is untraceable by the Rust compiler. +The only possible way to express this correctly is to require that any `#[pyclass]` does not borrow data for any lifetime shorter than the `'static` lifetime, i.e. the `#[pyclass]` cannot have any lifetime parameters. -When you need to share ownership of data between Python and Rust, instead of using borrowed references with lifetimes consider using reference-counted smart pointers such as [`Arc`] or [`Py`]. +When you need to share ownership of data between Python and Rust, instead of using borrowed references with lifetimes consider using reference-counted smart pointers such as [`Arc`] or [`Py`][`Py`]. #### No generic parameters -A Rust `struct Foo` with a generic parameter `T` generates new compiled implementations each time it is used with a different concrete type for `T`. These new implementations are generated by the compiler at each usage site. This is incompatible with wrapping `Foo` in Python, where there needs to be a single compiled implementation of `Foo` which is integrated with the Python interpreter. +A Rust `struct Foo` with a generic parameter `T` generates new compiled implementations each time it is used with a different concrete type for `T`. +These new implementations are generated by the compiler at each usage site. +This is incompatible with wrapping `Foo` in Python, where there needs to be a single compiled implementation of `Foo` which is integrated with the Python interpreter. -Currently, the best alternative is to write a macro which expands to a new #[pyclass] for each instantiation you want: +Currently, the best alternative is to write a macro which expands to a new `#[pyclass]` for each instantiation you want: ```rust # #![allow(dead_code)] use pyo3::prelude::*; struct GenericClass { - data: T + data: T, } macro_rules! create_interface { @@ -102,7 +118,9 @@ macro_rules! create_interface { impl $name { #[new] pub fn new(data: $type) -> Self { - Self { inner: GenericClass { data: data } } + Self { + inner: GenericClass { data: data }, + } } } }; @@ -112,15 +130,22 @@ create_interface!(IntClass, i64); create_interface!(FloatClass, String); ``` -#### Must be Send +#### Must be thread-safe + +Python objects are freely shared between threads by the Python interpreter. +This means that: + +- Python objects may be created and destroyed by different Python threads; therefore `#[pyclass]` objects must be `Send`. +- Python objects may be accessed by multiple Python threads simultaneously; therefore `#[pyclass]` objects must be `Sync`. -Because Python objects are freely shared between threads by the Python interpreter, there is no guarantee which thread will eventually drop the object. Therefore all types annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)). +For now, don't worry about these requirements; simple classes will already be thread-safe. +There is a [detailed discussion on thread-safety](./class/thread-safety.md) later in the guide. ## Constructor By default, it is not possible to create an instance of a custom class from Python code. -To declare a constructor, you need to define a method and annotate it with the `#[new]` -attribute. Only Python's `__new__` method can be specified, `__init__` is not available. +To declare a constructor, you need to define a method and annotate it with the `#[new]` attribute. +Only Python's `__new__` method can be specified, `__init__` is not available. ```rust # #![allow(dead_code)] @@ -172,41 +197,43 @@ For arguments, see the [`Method arguments`](#method-arguments) section below. ## Adding the class to a module -The next step is to create the module initializer and add our class to it: +The next step is to create the Python module and add our class to it: ```rust # #![allow(dead_code)] # use pyo3::prelude::*; +# fn main() {} # #[pyclass] # struct Number(i32); # #[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) +mod my_module { + #[pymodule_export] + use super::Number; } ``` -## PyCell and interior mutability +## Bound and interior mutability -You sometimes need to convert your `pyclass` into a Python object and access it -from Rust code (e.g., for testing it). -[`PyCell`] is the primary interface for that. +It is often useful to turn a `#[pyclass]` type `T` into a Python object and access it from Rust code. +The [`Py`] and [`Bound<'py, T>`] smart pointers are the ways to represent a Python object in PyO3's API. +More detail can be found about them [in the Python objects](./types.md#pyo3s-smart-pointers) section of the guide. -`PyCell` is always allocated in the Python heap, so Rust doesn't have ownership of it. -In other words, Rust code can only extract a `&PyCell`, not a `PyCell`. +Most Python objects do not offer exclusive (`&mut`) access (see the [section on Python's memory model](./python-from-rust.md#pythons-memory-model)). +However, Rust structs wrapped as Python objects (called `pyclass` types) often *do* need `&mut` access. +However, the Rust borrow checker cannot reason about `&mut` references once an object's ownership has been passed to the Python interpreter. -Thus, to mutate data behind `&PyCell` safely, PyO3 employs the -[Interior Mutability Pattern](https://doc.rust-lang.org/book/ch15-05-interior-mutability.html) -like [`RefCell`]. +To solve this, PyO3 does borrow checking at runtime using a scheme very similar to `std::cell::RefCell`. +This is known as [interior mutability](https://doc.rust-lang.org/book/ch15-05-interior-mutability.html). -Users who are familiar with `RefCell` can use `PyCell` just like `RefCell`. +Users who are familiar with `RefCell` can use `Py` and `Bound<'py, T>` just like `RefCell`. + +For users who are not very familiar with `RefCell`, here is a reminder of Rust's rules of borrowing: -For users who are not very familiar with `RefCell`, here is a reminder of Rust's rules of borrowing: - At any given time, you can have either (but not both of) one mutable reference or any number of immutable references. -- References must always be valid. +- References can never outlast the data they refer to. -`PyCell`, like `RefCell`, ensures these borrowing rules by tracking references at runtime. +`Py` and `Bound<'py, T>`, like `RefCell`, ensure these borrowing rules by tracking references at runtime. ```rust # use pyo3::prelude::*; @@ -215,8 +242,8 @@ struct MyClass { #[pyo3(get)] num: i32, } -Python::with_gil(|py| { - let obj = PyCell::new(py, MyClass { num: 3 }).unwrap(); +Python::attach(|py| { + let obj = Bound::new(py, MyClass { num: 3 }).unwrap(); { let obj_ref = obj.borrow(); // Get PyRef assert_eq!(obj_ref.num, 3); @@ -231,15 +258,13 @@ Python::with_gil(|py| { assert!(obj.try_borrow_mut().is_err()); } - // You can convert `&PyCell` to a Python object + // You can convert `Bound` to a Python object pyo3::py_run!(py, obj, "assert obj.num == 5"); }); ``` -`&PyCell` is bounded by the same lifetime as a [`GILGuard`]. -To make the object longer lived (for example, to store it in a struct on the -Rust side), you can use `Py`, which stores an object longer than the GIL -lifetime, and therefore needs a `Python<'_>` token to access. +A `Bound<'py, T>` is restricted to the Python lifetime `'py`. +To make the object longer lived (for example, to store it in a struct on the Rust side), use `Py`. `Py` needs a `Python<'_>` token to allow access: ```rust # use pyo3::prelude::*; @@ -249,23 +274,25 @@ struct MyClass { } fn return_myclass() -> Py { - Python::with_gil(|py| Py::new(py, MyClass { num: 1 }).unwrap()) + Python::attach(|py| Py::new(py, MyClass { num: 1 }).unwrap()) } let obj = return_myclass(); -Python::with_gil(|py| { - let cell = obj.as_ref(py); // Py::as_ref returns &PyCell - let obj_ref = cell.borrow(); // Get PyRef +Python::attach(move |py| { + let bound = obj.bind(py); // Py::bind returns &Bound<'py, MyClass> + let obj_ref = bound.borrow(); // Get PyRef assert_eq!(obj_ref.num, 1); }); ``` ### frozen classes: Opting out of interior mutability -As detailed above, runtime borrow checking is currently enabled by default. But a class can opt of out it by declaring itself `frozen`. It can still use interior mutability via standard Rust types like `RefCell` or `Mutex`, but it is not bound to the implementation provided by PyO3 and can choose the most appropriate strategy on field-by-field basis. +As detailed above, runtime borrow checking is currently enabled by default. +But a class can opt of out it by declaring itself `frozen`. +It can still use interior mutability via standard Rust types like `RefCell` or `Mutex`, but it is not bound to the implementation provided by PyO3 and can choose the most appropriate strategy on field-by-field basis. -Classes which are `frozen` and also `Sync`, e.g. they do use `Mutex` but not `RefCell`, can be accessed without needing the Python GIL via the `PyCell::get` and `Py::get` methods: +Classes which are `frozen` and also `Sync`, e.g. they do use `Mutex` but not `RefCell`, can be accessed without needing a `Python` token via the `Bound::get` and `Py::get` methods: ```rust use std::sync::atomic::{AtomicUsize, Ordering}; @@ -276,7 +303,7 @@ struct FrozenCounter { value: AtomicUsize, } -let py_counter: Py = Python::with_gil(|py| { +let py_counter: Py = Python::attach(|py| { let counter = FrozenCounter { value: AtomicUsize::new(0), }; @@ -285,13 +312,16 @@ let py_counter: Py = Python::with_gil(|py| { }); py_counter.get().value.fetch_add(1, Ordering::Relaxed); + +Python::attach(move |_py| drop(py_counter)); ``` -Frozen classes are likely to become the default thereby guiding the PyO3 ecosystem towards a more deliberate application of interior mutability. Eventually, this should enable further optimizations of PyO3's internals and avoid downstream code paying the cost of interior mutability when it is not actually required. +Frozen classes are likely to become the default thereby guiding the PyO3 ecosystem towards a more deliberate application of interior mutability. +Eventually, this should enable further optimizations of PyO3's internals and avoid downstream code paying the cost of interior mutability when it is not actually required. ## Customizing the class -{{#include ../pyclass_parameters.md}} +{{#include ../pyclass-parameters.md}} These parameters are covered in various sections of this guide. @@ -311,22 +341,18 @@ Consult the table below to determine which type your constructor should return: ## Inheritance -By default, `object`, i.e. `PyAny` is used as the base class. To override this default, -use the `extends` parameter for `pyclass` with the full path to the base class. -Currently, only classes defined in Rust and builtins provided by PyO3 can be inherited -from; inheriting from other classes defined in Python is not yet supported -([#991](https://github.com/PyO3/pyo3/issues/991)). - +By default, `object`, i.e. `PyAny` is used as the base class. +To override this default, use the `extends` parameter for `pyclass` with the full path to the base class. +Currently, only classes defined in Rust and builtins provided by PyO3 can be inherited from; inheriting from other classes defined in Python is not yet supported ([#991](https://github.com/PyO3/pyo3/issues/991)). For convenience, `(T, U)` implements `Into>` where `U` is the base class of `T`. But for a more deeply nested inheritance, you have to return `PyClassInitializer` explicitly. -To get a parent class from a child, use [`PyRef`] instead of `&self` for methods, -or [`PyRefMut`] instead of `&mut self`. -Then you can access a parent class by `self_.as_ref()` as `&Self::BaseClass`, -or by `self_.into_super()` as `PyRef`. +To get a parent class from a child, use [`PyRef`] instead of `&self` for methods, or [`PyRefMut`] instead of `&mut self`. +Then you can access a parent class by `self_.as_super()` as `&PyRef`, or by `self_.into_super()` as `PyRef` (and similar for the `PyRefMut` case). +For convenience, `self_.as_ref()` can also be used to get `&Self::BaseClass` directly; however, this approach does not let you access base classes higher in the inheritance hierarchy, for which you would need to chain multiple `as_super` or `into_super` calls. ```rust # use pyo3::prelude::*; @@ -343,7 +369,7 @@ impl BaseClass { BaseClass { val1: 10 } } - pub fn method(&self) -> PyResult { + pub fn method1(&self) -> PyResult { Ok(self.val1) } } @@ -361,8 +387,8 @@ impl SubClass { } fn method2(self_: PyRef<'_, Self>) -> PyResult { - let super_ = self_.as_ref(); // Get &BaseClass - super_.method().map(|x| x * self_.val2) + let super_ = self_.as_super(); // Get &PyRef + super_.method1().map(|x| x * self_.val2) } } @@ -379,26 +405,49 @@ impl SubSubClass { } fn method3(self_: PyRef<'_, Self>) -> PyResult { + let base = self_.as_super().as_super(); // Get &PyRef<'_, BaseClass> + base.method1().map(|x| x * self_.val3) + } + + fn method4(self_: PyRef<'_, Self>) -> PyResult { let v = self_.val3; let super_ = self_.into_super(); // Get PyRef<'_, SubClass> SubClass::method2(super_).map(|x| x * v) } + fn get_values(self_: PyRef<'_, Self>) -> (usize, usize, usize) { + let val1 = self_.as_super().as_super().val1; + let val2 = self_.as_super().val2; + (val1, val2, self_.val3) + } + + fn double_values(mut self_: PyRefMut<'_, Self>) { + self_.as_super().as_super().val1 *= 2; + self_.as_super().val2 *= 2; + self_.val3 *= 2; + } + #[staticmethod] - fn factory_method(py: Python<'_>, val: usize) -> PyResult { + fn factory_method(py: Python<'_>, val: usize) -> PyResult> { let base = PyClassInitializer::from(BaseClass::new()); let sub = base.add_subclass(SubClass { val2: val }); if val % 2 == 0 { - Ok(Py::new(py, sub)?.to_object(py)) + Ok(Py::new(py, sub)?.into_any()) } else { let sub_sub = sub.add_subclass(SubSubClass { val3: val }); - Ok(Py::new(py, sub_sub)?.to_object(py)) + Ok(Py::new(py, sub_sub)?.into_any()) } } } -# Python::with_gil(|py| { -# let subsub = pyo3::PyCell::new(py, SubSubClass::new()).unwrap(); -# pyo3::py_run!(py, subsub, "assert subsub.method3() == 3000"); +# Python::attach(|py| { +# let subsub = pyo3::Py::new(py, SubSubClass::new()).unwrap(); +# pyo3::py_run!(py, subsub, "assert subsub.method1() == 10"); +# pyo3::py_run!(py, subsub, "assert subsub.method2() == 150"); +# pyo3::py_run!(py, subsub, "assert subsub.method3() == 200"); +# pyo3::py_run!(py, subsub, "assert subsub.method4() == 3000"); +# pyo3::py_run!(py, subsub, "assert subsub.get_values() == (10, 15, 20)"); +# pyo3::py_run!(py, subsub, "assert subsub.double_values() == None"); +# pyo3::py_run!(py, subsub, "assert subsub.get_values() == (20, 30, 40)"); # let subsub = SubSubClass::factory_method(py, 2).unwrap(); # let subsubsub = SubSubClass::factory_method(py, 3).unwrap(); # let cls = py.get_type::(); @@ -411,8 +460,8 @@ You can inherit native types such as `PyDict`, if they implement [`PySizedLayout`]({{#PYO3_DOCS_URL}}/pyo3/type_object/trait.PySizedLayout.html). This is not supported when building for the Python limited API (aka the `abi3` feature of PyO3). -However, because of some technical problems, we don't currently provide safe upcasting methods for types -that inherit native types. Even in such cases, you can unsafely get a base class by raw pointer conversion. +To convert between the Rust type and its native base class, you can take `slf` as a Python object. +To access the Rust fields use `slf.borrow()` or `slf.borrow_mut()`, and to access the base class use `slf.cast::()`. ```rust # #[cfg(not(Py_LIMITED_API))] { @@ -433,21 +482,21 @@ impl DictWithCounter { Self::default() } - fn set(mut self_: PyRefMut<'_, Self>, key: String, value: &PyAny) -> PyResult<()> { - self_.counter.entry(key.clone()).or_insert(0); - let py = self_.py(); - let dict: &PyDict = unsafe { py.from_borrowed_ptr_or_err(self_.as_ptr())? }; + fn set(slf: &Bound<'_, Self>, key: String, value: Bound<'_, PyAny>) -> PyResult<()> { + slf.borrow_mut().counter.entry(key.clone()).or_insert(0); + let dict = slf.cast::()?; dict.set_item(key, value) } } -# Python::with_gil(|py| { -# let cnt = pyo3::PyCell::new(py, DictWithCounter::new()).unwrap(); +# Python::attach(|py| { +# let cnt = pyo3::Py::new(py, DictWithCounter::new()).unwrap(); # pyo3::py_run!(py, cnt, "cnt.set('abc', 10); assert cnt['abc'] == 10") # }); # } ``` If `SubClass` does not provide a base class initialization, the compilation fails. + ```rust,compile_fail # use pyo3::prelude::*; @@ -490,13 +539,13 @@ struct MyDict { impl MyDict { #[new] #[pyo3(signature = (*args, **kwargs))] - fn new(args: &PyAny, kwargs: Option<&PyAny>) -> Self { + fn new(args: &Bound<'_, PyAny>, kwargs: Option<&Bound<'_, PyAny>>) -> Self { Self { private: 0 } } // some custom methods that use `private` here... } -# Python::with_gil(|py| { +# Python::attach(|py| { # let cls = py.get_type::(); # pyo3::py_run!(py, cls, "cls(a=1, b=2)") # }); @@ -509,6 +558,7 @@ initial items, such as `MyDict(item_sequence)` or `MyDict(a=1, b=2)`. ## Object properties PyO3 supports two ways to add properties to your `#[pyclass]`: + - For simple struct fields with no side effects, a `#[pyo3(get, set)]` attribute can be added directly to the field definition in the `#[pyclass]`. - For properties which require computation you can define `#[getter]` and `#[setter]` functions in the [`#[pymethods]`](#instance-methods) block. @@ -520,6 +570,7 @@ For simple cases where a member variable is just read and written with no side e ```rust # use pyo3::prelude::*; +# #[allow(dead_code)] #[pyclass] struct MyClass { #[pyo3(get, set)] @@ -527,15 +578,18 @@ struct MyClass { } ``` -The above would make the `num` field available for reading and writing as a `self.num` Python property. To expose the property with a different name to the field, specify this alongside the rest of the options, e.g. `#[pyo3(get, set, name = "custom_name")]`. +The above would make the `num` field available for reading and writing as a `self.num` Python property. +To expose the property with a different name to the field, specify this alongside the rest of the options, e.g. `#[pyo3(get, set, name = "custom_name")]`. Properties can be readonly or writeonly by using just `#[pyo3(get)]` or `#[pyo3(set)]` respectively. To use these annotations, your field type must implement some conversion traits: -- For `get` the field type must implement both `IntoPy` and `Clone`. + +- For `get` the field type `T` must implement either `&T: IntoPyObject` or `T: IntoPyObject + Clone`. - For `set` the field type must implement `FromPyObject`. -For example, implementations of those traits are provided for the `Cell` type, if the inner type also implements the trait. This means you can use `#[pyo3(get, set)]` on fields wrapped in a `Cell`. +For example, implementations of those traits are provided for the `Cell` type, if the inner type also implements the trait. +This means you can use `#[pyo3(get, set)]` on fields wrapped in a `Cell`. ### Object properties using `#[getter]` and `#[setter]` @@ -559,14 +613,11 @@ impl MyClass { } ``` -A getter or setter's function name is used as the property name by default. There are several -ways how to override the name. +A getter or setter's function name is used as the property name by default. +There are several ways how to override the name. -If a function name starts with `get_` or `set_` for getter or setter respectively, -the descriptor name becomes the function name with this prefix removed. This is also useful in case of -Rust keywords like `type` -([raw identifiers](https://doc.rust-lang.org/edition-guide/rust-2018/module-system/raw-identifiers.html) -can be used since Rust 2018). +If a function name starts with `get_` or `set_` for getter or setter respectively, the descriptor name becomes the function name with this prefix removed. +This is also useful in case of Rust keywords like `type` ([raw identifiers](https://doc.rust-lang.org/edition-guide/rust-2018/module-system/raw-identifiers.html) can be used since Rust 2018). ```rust # use pyo3::prelude::*; @@ -617,19 +668,16 @@ impl MyClass { In this case, the property `number` is defined and available from Python code as `self.number`. -Attributes defined by `#[setter]` or `#[pyo3(set)]` will always raise `AttributeError` on `del` -operations. Support for defining custom `del` behavior is tracked in -[#1778](https://github.com/PyO3/pyo3/issues/1778). +Attributes defined by `#[setter]` or `#[pyo3(set)]` will always raise `AttributeError` on `del` operations. +Support for defining custom `del` behavior is tracked in [#1778](https://github.com/PyO3/pyo3/issues/1778). ## Instance methods -To define a Python compatible method, an `impl` block for your struct has to be annotated with the -`#[pymethods]` attribute. PyO3 generates Python compatible wrappers for all functions in this -block with some variations, like descriptors, class method static methods, etc. +To define a Python compatible method, an `impl` block for your struct has to be annotated with the `#[pymethods]` attribute. +PyO3 generates Python compatible wrappers for all functions in this block with some variations, like descriptors, class method static methods, etc. -Since Rust allows any number of `impl` blocks, you can easily split methods -between those accessible to Python (and Rust) and those accessible only to Rust. However to have multiple -`#[pymethods]`-annotated `impl` blocks for the same struct you must enable the [`multiple-pymethods`] feature of PyO3. +Since Rust allows any number of `impl` blocks, you can easily split methods between those accessible to Python (and Rust) and those accessible only to Rust. +However to have multiple `#[pymethods]`-annotated `impl` blocks for the same struct you must enable the [`multiple-pymethods`] feature of PyO3. ```rust # use pyo3::prelude::*; @@ -650,8 +698,9 @@ impl MyClass { } ``` -Calls to these methods are protected by the GIL, so both `&self` and `&mut self` can be used. -The return type must be `PyResult` or `T` for some `T` that implements `IntoPy`; +Both `&self` and `&mut self` can be used, due to the use of [runtime borrow checking](#bound-and-interior-mutability). + +The return type must be `PyResult` or `T` for some `T` that implements `IntoPyObject`; the latter is allowed if the method cannot raise Python exceptions. A `Python` parameter can be specified as part of method signature, in this case the `py` argument @@ -699,21 +748,22 @@ impl MyClass { Declares a class method callable from Python. -* The first parameter is the type object of the class on which the method is called. +- The first parameter is the type object of the class on which the method is called. This may be the type object of a derived class. -* The first parameter implicitly has type `&PyType`. -* For details on `parameter-list`, see the documentation of `Method arguments` section. -* The return type must be `PyResult` or `T` for some `T` that implements `IntoPy`. +- The first parameter implicitly has type `&Bound<'_, PyType>`. +- For details on `parameter-list`, see the documentation of `Method arguments` section. +- The return type must be `PyResult` or `T` for some `T` that implements `IntoPyObject`. ### Constructors which accept a class argument To create a constructor which takes a positional class argument, you can combine the `#[classmethod]` and `#[new]` modifiers: + ```rust # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyType; # #[pyclass] -# struct BaseClass(PyObject); +# struct BaseClass(Py); # #[pymethods] impl BaseClass { @@ -729,9 +779,8 @@ impl BaseClass { ## Static methods -To create a static method for a custom class, the method needs to be annotated with the -`#[staticmethod]` attribute. The return type must be `T` or `PyResult` for some `T` that implements -`IntoPy`. +To create a static method for a custom class, the method needs to be annotated with the `#[staticmethod]` attribute. +The return type must be `T` or `PyResult` for some `T` that implements `IntoPyObject`. ```rust # use pyo3::prelude::*; @@ -754,7 +803,7 @@ impl MyClass { To create a class attribute (also called [class variable][classattr]), a method without any arguments can be annotated with the `#[classattr]` attribute. -```rust +```rust,no_run # use pyo3::prelude::*; # #[pyclass] # struct MyClass {} @@ -766,7 +815,7 @@ impl MyClass { } } -Python::with_gil(|py| { +Python::attach(|py| { let my_class = py.get_type::(); pyo3::py_run!(py, my_class, "assert my_class.my_attribute == 'hello'") }); @@ -775,10 +824,12 @@ Python::with_gil(|py| { > Note: if the method has a `Result` return type and returns an `Err`, PyO3 will panic during class creation. +> Note: `#[classattr]` does not work with [`#[pyo3(warn(...))]`](./function.md#warn) attribute. + If the class attribute is defined with `const` code only, one can also annotate associated constants: -```rust +```rust,no_run # use pyo3::prelude::*; # #[pyclass] # struct MyClass {} @@ -791,9 +842,15 @@ impl MyClass { ## Classes as function arguments -Free functions defined using `#[pyfunction]` interact with classes through the same mechanisms as the self parameters of instance methods, i.e. they can take GIL-bound references, GIL-bound reference wrappers or GIL-indepedent references: +Class objects can be used as arguments to `#[pyfunction]`s and `#[pymethods]` in the same way as the self parameters of instance methods, i.e. they can be passed as: -```rust +- `Py` or `Bound<'py, T>` smart pointers to the class Python object, +- `&T` or `&mut T` references to the Rust data contained in the Python object, or +- `PyRef` and `PyRefMut` reference wrappers. + +Examples of each of these below: + +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; #[pyclass] @@ -801,29 +858,30 @@ struct MyClass { my_field: i32, } -// Take a GIL-bound reference when the underlying `PyCell` is irrelevant. +// Take a reference to Rust data when the Python object is irrelevant. #[pyfunction] fn increment_field(my_class: &mut MyClass) { my_class.my_field += 1; } -// Take a GIL-bound reference wrapper when borrowing should be automatic, -// but interaction with the underlying `PyCell` is desired. +// Take a reference wrapper when borrowing should be automatic, +// but access to the Python object is still needed #[pyfunction] -fn print_field(my_class: PyRef<'_, MyClass>) { +fn print_field_and_return_me(my_class: PyRef<'_, MyClass>) -> PyRef<'_, MyClass> { println!("{}", my_class.my_field); + my_class } -// Take a GIL-bound reference to the underlying cell -// when borrowing needs to be managed manually. +// Take (a reference to) a Python object smart pointer when borrowing needs to be managed manually. #[pyfunction] -fn increment_then_print_field(my_class: &PyCell) { +fn increment_then_print_field(my_class: &Bound<'_, MyClass>) { my_class.borrow_mut().my_field += 1; println!("{}", my_class.borrow().my_field); } -// Take a GIL-indepedent reference when you want to store the reference elsewhere. +// When the Python object smart pointer needs to be stored elsewhere prefer `Py` over `Bound<'py, T>` +// to avoid the lifetime restrictions. #[pyfunction] fn print_refcnt(my_class: Py, py: Python<'_>) { println!("{}", my_class.get_refcnt(py)); @@ -832,7 +890,7 @@ fn print_refcnt(my_class: Py, py: Python<'_>) { Classes can also be passed by value if they can be cloned, i.e. they automatically implement `FromPyObject` if they implement `Clone`, e.g. via `#[derive(Clone)]`: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; #[pyclass] @@ -842,7 +900,7 @@ struct MyClass { } #[pyfunction] -fn dissamble_clone(my_class: MyClass) { +fn disassemble_clone(my_class: MyClass) { let MyClass { mut my_field } = my_class; *my_field += 1; } @@ -852,11 +910,13 @@ Note that `#[derive(FromPyObject)]` on a class is usually not useful as it tries ## Method arguments -Similar to `#[pyfunction]`, the `#[pyo3(signature = (...))]` attribute can be used to specify the way that `#[pymethods]` accept arguments. Consult the documentation for [`function signatures`](./function/signature.md) to see the parameters this attribute accepts. +Similar to `#[pyfunction]`, the `#[pyo3(signature = (...))]` attribute can be used to specify the way that `#[pymethods]` accept arguments. +Consult the documentation for [`function signatures`](./function/signature.md) to see the parameters this attribute accepts. -The following example defines a class `MyClass` with a method `method`. This method has a signature that sets default values for `num` and `name`, and indicates that `py_args` should collect all extra positional arguments and `py_kwargs` all extra keyword arguments: +The following example defines a class `MyClass` with a method `method`. +This method has a signature that sets default values for `num` and `name`, and indicates that `py_args` should collect all extra positional arguments and `py_kwargs` all extra keyword arguments: -```rust +```rust,no_run # use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; # @@ -876,9 +936,9 @@ impl MyClass { fn method( &mut self, num: i32, - py_args: &PyTuple, + py_args: &Bound<'_, PyTuple>, name: &str, - py_kwargs: Option<&PyDict>, + py_kwargs: Option<&Bound<'_, PyDict>>, ) -> String { let num_before = self.num; self.num = num; @@ -901,9 +961,7 @@ py_args=('World', 666), py_kwargs=Some({'x': 44, 'y': 55}), name=Hello, num=44, py_args=(), py_kwargs=None, name=World, num=-1, num_before=44 ``` -## Making class method signatures available to Python - -The [`text_signature = "..."`](./function.md#text_signature) option for `#[pyfunction]` also works for `#[pymethods]`: +The [`#[pyo3(text_signature = "...")`](./function/signature.md#overriding-the-generated-signature) option for `#[pyfunction]` also works for `#[pymethods]`. ```rust # #![allow(dead_code)] @@ -939,7 +997,7 @@ impl MyClass { } # # fn main() -> PyResult<()> { -# Python::with_gil(|py| { +# Python::attach(|py| { # let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; # let module = PyModule::new(py, "my_module")?; # module.add_class::()?; @@ -950,7 +1008,7 @@ impl MyClass { # assert_eq!(doc, ""); # # let sig: String = inspect -# .call1((class,))? +# .call1((&class,))? # .call_method0("__str__")? # .extract()?; # assert_eq!(sig, "(c, d)"); @@ -958,7 +1016,7 @@ impl MyClass { # let doc: String = class.getattr("__doc__")?.extract()?; # assert_eq!(doc, ""); # -# inspect.call1((class,)).expect_err("`text_signature` on classes is not compatible with compilation in `abi3` mode until Python 3.10 or greater"); +# inspect.call1((&class,)).expect_err("`text_signature` on classes is not compatible with compilation in `abi3` mode until Python 3.10 or greater"); # } # # { @@ -1005,25 +1063,68 @@ impl MyClass { Note that `text_signature` on `#[new]` is not compatible with compilation in `abi3` mode until Python 3.10 or greater. -## #[pyclass] enums +### Method receivers and lifetime elision + +PyO3 supports writing instance methods using the normal method receivers for shared `&self` and unique `&mut self` references. +This interacts with [lifetime elision][lifetime-elision] insofar as the lifetime of a such a receiver is assigned to all elided output lifetime parameters. + +This is a good default for general Rust code where return values are more likely to borrow from the receiver than from the other arguments, if they contain any lifetimes at all. +However, when returning bound references `Bound<'py, T>` in PyO3-based code, the Python lifetime `'py` should usually be derived from a `py: Python<'py>` token passed as an argument instead of the receiver. + +Specifically, signatures like + +```rust,ignore +fn frobnicate(&self, py: Python) -> Bound; +``` + +will not work as they are inferred as + +```rust,ignore +fn frobnicate<'a, 'py>(&'a self, py: Python<'py>) -> Bound<'a, Foo>; +``` + +instead of the intended + +```rust,ignore +fn frobnicate<'a, 'py>(&'a self, py: Python<'py>) -> Bound<'py, Foo>; +``` + +and should usually be written as + +```rust,ignore +fn frobnicate<'py>(&self, py: Python<'py>) -> Bound<'py, Foo>; +``` + +The same problem does not exist for `#[pyfunction]`s as the special case for receiver lifetimes does not apply and indeed a signature like + +```rust,ignore +fn frobnicate(bar: &Bar, py: Python) -> Bound; +``` + +will yield compiler error [E0106 "missing lifetime specifier"][compiler-error-e0106]. + +## `#[pyclass]` enums Enum support in PyO3 comes in two flavors, depending on what kind of variants the enum has: simple and complex. ### Simple enums -A simple enum (a.k.a. C-like enum) has only unit variants. +A simple enum (a.k.a. +C-like enum) has only unit variants. -PyO3 adds a class attribute for each variant, so you can access them in Python without defining `#[new]`. PyO3 also provides default implementations of `__richcmp__` and `__int__`, so they can be compared using `==`: +PyO3 adds a class attribute for each variant, so you can access them in Python without defining `#[new]`. +PyO3 also provides default implementations of `__richcmp__` and `__int__`, so they can be compared using `==`: ```rust # use pyo3::prelude::*; -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum { Variant, OtherVariant, } -Python::with_gil(|py| { +Python::attach(|py| { let x = Py::new(py, MyEnum::Variant).unwrap(); let y = Py::new(py, MyEnum::OtherVariant).unwrap(); let cls = py.get_type::(); @@ -1039,20 +1140,19 @@ You can also convert your simple enums into `int`: ```rust # use pyo3::prelude::*; -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum { Variant, OtherVariant = 10, } -Python::with_gil(|py| { +Python::attach(|py| { let cls = py.get_type::(); let x = MyEnum::Variant as i32; // The exact value is assigned by the compiler. pyo3::py_run!(py, cls x, r#" assert int(cls.Variant) == x assert int(cls.OtherVariant) == 10 - assert cls.OtherVariant == 10 # You can also compare against int. - assert 10 == cls.OtherVariant "#) }) ``` @@ -1061,13 +1161,14 @@ PyO3 also provides `__repr__` for enums: ```rust # use pyo3::prelude::*; -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum{ Variant, OtherVariant, } -Python::with_gil(|py| { +Python::attach(|py| { let cls = py.get_type::(); let x = Py::new(py, MyEnum::Variant).unwrap(); pyo3::py_run!(py, cls x, r#" @@ -1077,11 +1178,13 @@ Python::with_gil(|py| { }) ``` -All methods defined by PyO3 can be overridden. For example here's how you override `__repr__`: +All methods defined by PyO3 can be overridden. +For example here's how you override `__repr__`: ```rust # use pyo3::prelude::*; -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] enum MyEnum { Answer = 42, } @@ -1093,7 +1196,7 @@ impl MyEnum { } } -Python::with_gil(|py| { +Python::attach(|py| { let cls = py.get_type::(); pyo3::py_run!(py, cls, "assert repr(cls.Answer) == '42'") }) @@ -1103,13 +1206,14 @@ Enums and their variants can also be renamed using `#[pyo3(name)]`. ```rust # use pyo3::prelude::*; -#[pyclass(name = "RenamedEnum")] +#[pyclass(eq, eq_int, name = "RenamedEnum")] +#[derive(PartialEq)] enum MyEnum { #[pyo3(name = "UPPERCASE")] Variant, } -Python::with_gil(|py| { +Python::attach(|py| { let x = Py::new(py, MyEnum::Variant).unwrap(); let cls = py.get_type::(); pyo3::py_run!(py, x cls, r#" @@ -1119,6 +1223,32 @@ Python::with_gil(|py| { }) ``` +Ordering of enum variants is optionally added using `#[pyo3(ord)]`. +*Note: Implementation of the `PartialOrd` trait is required when passing the `ord` argument. If not implemented, a compile time error is raised.* + +```rust +# use pyo3::prelude::*; +#[pyclass(eq, ord)] +#[derive(PartialEq, PartialOrd)] +enum MyEnum{ + A, + B, + C, +} + +Python::attach(|py| { + let cls = py.get_type::(); + let a = Py::new(py, MyEnum::A).unwrap(); + let b = Py::new(py, MyEnum::B).unwrap(); + let c = Py::new(py, MyEnum::C).unwrap(); + pyo3::py_run!(py, cls a b c, r#" + assert (a < b) == True + assert (c <= b) == False + assert (c > a) == True + "#) +}) +``` + You may not use enums as a base class or let enums inherit from other classes. ```rust,compile_fail @@ -1147,9 +1277,11 @@ enum BadSubclass { An enum is complex if it has any non-unit (struct or tuple) variants. -Currently PyO3 supports only struct variants in a complex enum. Support for unit and tuple variants is planned. +PyO3 supports only struct and tuple variants in a complex enum. +Unit variants aren't supported at present (the recommendation is to use an empty tuple enum instead). -PyO3 adds a class attribute for each variant, which may be used to construct values and in match patterns. PyO3 also provides getter methods for all fields of each variant. +PyO3 adds a class attribute for each variant, which may be used to construct values and in match patterns. +PyO3 also provides getter methods for all fields of each variant. ```rust # use pyo3::prelude::*; @@ -1157,14 +1289,14 @@ PyO3 adds a class attribute for each variant, which may be used to construct val enum Shape { Circle { radius: f64 }, Rectangle { width: f64, height: f64 }, - RegularPolygon { side_count: u32, radius: f64 }, + RegularPolygon(u32, f64), Nothing { }, } # #[cfg(Py_3_10)] -Python::with_gil(|py| { - let circle = Shape::Circle { radius: 10.0 }.into_py(py); - let square = Shape::RegularPolygon { side_count: 4, radius: 10.0 }.into_py(py); +Python::attach(|py| { + let circle = Shape::Circle { radius: 10.0 }.into_pyobject(py)?; + let square = Shape::RegularPolygon(4, 10.0).into_pyobject(py)?; let cls = py.get_type::(); pyo3::py_run!(py, circle square cls, r#" assert isinstance(circle, cls) @@ -1173,8 +1305,8 @@ Python::with_gil(|py| { assert isinstance(square, cls) assert isinstance(square, cls.RegularPolygon) - assert square.side_count == 4 - assert square.radius == 10.0 + assert square[0] == 4 # Gets _0 field + assert square[1] == 10.0 # Gets _1 field def count_vertices(cls, shape): match shape: @@ -1182,18 +1314,22 @@ Python::with_gil(|py| { return 0 case cls.Rectangle(): return 4 - case cls.RegularPolygon(side_count=n): + case cls.RegularPolygon(n): return n case cls.Nothing(): return 0 assert count_vertices(cls, circle) == 0 assert count_vertices(cls, square) == 4 - "#) + "#); +# Ok::<_, PyErr>(()) }) +# .unwrap(); ``` -WARNING: `Py::new` and `.into_py` are currently inconsistent. Note how the constructed value is _not_ an instance of the specific variant. For this reason, constructing values is only recommended using `.into_py`. +WARNING: `Py::new` and `.into_pyobject` are currently inconsistent. +Note how the constructed value is _not_ an instance of the specific variant. +For this reason, constructing values is only recommended using `.into_pyobject`. ```rust # use pyo3::prelude::*; @@ -1202,7 +1338,7 @@ enum MyEnum { Variant { i: i32 }, } -Python::with_gil(|py| { +Python::attach(|py| { let x = Py::new(py, MyEnum::Variant { i: 42 }).unwrap(); let cls = py.get_type::(); pyo3::py_run!(py, x cls, r#" @@ -1212,34 +1348,86 @@ Python::with_gil(|py| { }) ``` +The constructor of each generated class can be customized using the `#[pyo3(constructor = (...))]` attribute. +This uses the same syntax as the [`#[pyo3(signature = (...))]`](function/signature.md) attribute on function and methods and supports the same options. +To apply this attribute simply place it on top of a variant in a `#[pyclass]` complex enum as shown below: + +```rust +# use pyo3::prelude::*; +#[pyclass] +enum Shape { + #[pyo3(constructor = (radius=1.0))] + Circle { radius: f64 }, + #[pyo3(constructor = (*, width, height))] + Rectangle { width: f64, height: f64 }, + #[pyo3(constructor = (side_count, radius=1.0))] + RegularPolygon { side_count: u32, radius: f64 }, + Nothing { }, +} + +# #[cfg(Py_3_10)] +Python::attach(|py| { + let cls = py.get_type::(); + pyo3::py_run!(py, cls, r#" + circle = cls.Circle() + assert isinstance(circle, cls) + assert isinstance(circle, cls.Circle) + assert circle.radius == 1.0 + + square = cls.Rectangle(width = 1, height = 1) + assert isinstance(square, cls) + assert isinstance(square, cls.Rectangle) + assert square.width == 1 + assert square.height == 1 + + hexagon = cls.RegularPolygon(6) + assert isinstance(hexagon, cls) + assert isinstance(hexagon, cls.RegularPolygon) + assert hexagon.side_count == 6 + assert hexagon.radius == 1 + "#) +}) +``` + ## Implementation details The `#[pyclass]` macros rely on a lot of conditional code generation: each `#[pyclass]` can optionally have a `#[pymethods]` block. -To support this flexibility the `#[pyclass]` macro expands to a blob of boilerplate code which sets up the structure for ["dtolnay specialization"](https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md). This implementation pattern enables the Rust compiler to use `#[pymethods]` implementations when they are present, and fall back to default (empty) definitions when they are not. +To support this flexibility the `#[pyclass]` macro expands to a blob of boilerplate code which sets up the structure for ["dtolnay specialization"](https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md). +This implementation pattern enables the Rust compiler to use `#[pymethods]` implementations when they are present, and fall back to default (empty) definitions when they are not. -This simple technique works for the case when there is zero or one implementations. To support multiple `#[pymethods]` for a `#[pyclass]` (in the [`multiple-pymethods`] feature), a registry mechanism provided by the [`inventory`](https://github.com/dtolnay/inventory) crate is used instead. This collects `impl`s at library load time, but isn't supported on all platforms. See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) for more details. +This simple technique works for the case when there is zero or one implementations. +To support multiple `#[pymethods]` for a `#[pyclass]` (in the [`multiple-pymethods`] feature), a registry mechanism provided by the [`inventory`](https://github.com/dtolnay/inventory) crate is used instead. +This collects `impl`s at library load time, but isn't supported on all platforms. See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) for more details. -The `#[pyclass]` macro expands to roughly the code seen below. The `PyClassImplCollector` is the type used internally by PyO3 for dtolnay specialization: +The `#[pyclass]` macro expands to roughly the code seen below. +The `PyClassImplCollector` is the type used internally by PyO3 for dtolnay specialization: ```rust # #[cfg(not(feature = "multiple-pymethods"))] { # use pyo3::prelude::*; // Note: the implementation differs slightly with the `multiple-pymethods` feature enabled. +# #[allow(dead_code)] struct MyClass { # #[allow(dead_code)] num: i32, } -unsafe impl pyo3::type_object::HasPyGilRef for MyClass { - type AsRefTarget = pyo3::PyCell; -} + +impl pyo3::types::DerefToPyAny for MyClass {} + unsafe impl pyo3::type_object::PyTypeInfo for MyClass { const NAME: &'static str = "MyClass"; const MODULE: ::std::option::Option<&'static str> = ::std::option::Option::None; + #[inline] fn type_object_raw(py: pyo3::Python<'_>) -> *mut pyo3::ffi::PyTypeObject { ::lazy_type_object() - .get_or_init(py) + .get_or_try_init(py) + .unwrap_or_else(|e| pyo3::impl_::pyclass::type_object_init_failed( + py, + e, + ::NAME + )) .as_type_ptr() } } @@ -1248,35 +1436,11 @@ impl pyo3::PyClass for MyClass { type Frozen = pyo3::pyclass::boolean_struct::False; } -impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a MyClass -{ - type Holder = ::std::option::Option>; - - #[inline] - fn extract(obj: &'py pyo3::PyAny, holder: &'a mut Self::Holder) -> pyo3::PyResult { - pyo3::impl_::extract_argument::extract_pyclass_ref(obj, holder) - } -} - -impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a mut MyClass -{ - type Holder = ::std::option::Option>; - - #[inline] - fn extract(obj: &'py pyo3::PyAny, holder: &'a mut Self::Holder) -> pyo3::PyResult { - pyo3::impl_::extract_argument::extract_pyclass_ref_mut(obj, holder) - } -} - -impl pyo3::IntoPy for MyClass { - fn into_py(self, py: pyo3::Python<'_>) -> pyo3::PyObject { - pyo3::IntoPy::into_py(pyo3::Py::new(py, self).unwrap(), py) - } -} - impl pyo3::impl_::pyclass::PyClassImpl for MyClass { const IS_BASETYPE: bool = false; const IS_SUBCLASS: bool = false; + const IS_MAPPING: bool = false; + const IS_SEQUENCE: bool = false; type BaseType = PyAny; type ThreadChecker = pyo3::impl_::pyclass::SendablePyClass; type PyClassMutability = <::PyClassMutability as pyo3::impl_::pycell::PyClassMutability>::MutableChild; @@ -1284,6 +1448,9 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { type WeakRef = pyo3::impl_::pyclass::PyClassDummySlot; type BaseNativeType = pyo3::PyAny; + const RAW_DOC: &'static std::ffi::CStr = c"..."; + const DOC: &'static std::ffi::CStr = c"..."; + fn items_iter() -> pyo3::impl_::pyclass::PyClassItemsIter { use pyo3::impl_::pyclass::*; let collector = PyClassImplCollector::::new(); @@ -1296,38 +1463,29 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { static TYPE_OBJECT: LazyTypeObject = LazyTypeObject::new(); &TYPE_OBJECT } - - fn doc(py: Python<'_>) -> pyo3::PyResult<&'static ::std::ffi::CStr> { - use pyo3::impl_::pyclass::*; - static DOC: pyo3::sync::GILOnceCell<::std::borrow::Cow<'static, ::std::ffi::CStr>> = pyo3::sync::GILOnceCell::new(); - DOC.get_or_try_init(py, || { - let collector = PyClassImplCollector::::new(); - build_pyclass_doc(::NAME, "", None.or_else(|| collector.new_text_signature())) - }).map(::std::ops::Deref::deref) - } } -# Python::with_gil(|py| { +# Python::attach(|py| { # let cls = py.get_type::(); # pyo3::py_run!(py, cls, "assert cls.__name__ == 'MyClass'") # }); # } ``` - -[`GILGuard`]: {{#PYO3_DOCS_URL}}/pyo3/struct.GILGuard.html [`PyTypeInfo`]: {{#PYO3_DOCS_URL}}/pyo3/type_object/trait.PyTypeInfo.html -[`Py`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html -[`PyCell`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyCell.html +[`Py`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html +[`Bound<'py, T>`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html [`PyClass`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass/trait.PyClass.html [`PyRef`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html [`PyRefMut`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRefMut.html [`PyClassInitializer`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass_init/struct.PyClassInitializer.html [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html -[`RefCell`]: https://doc.rust-lang.org/std/cell/struct.RefCell.html [classattr]: https://docs.python.org/3/tutorial/classes.html#class-and-instance-variables [`multiple-pymethods`]: features.md#multiple-pymethods + +[lifetime-elision]: https://doc.rust-lang.org/reference/lifetime-elision.html +[compiler-error-e0106]: https://doc.rust-lang.org/error_codes/E0106.html diff --git a/guide/src/class/call.md b/guide/src/class/call.md index 3b20986239b..0f08bb53ce7 100644 --- a/guide/src/class/call.md +++ b/guide/src/class/call.md @@ -6,11 +6,10 @@ This allows instances of a class to behave similar to functions. This method's signature must look like `__call__(, ...) -> object` - here, any argument list can be defined as for normal pymethods -### Example: Implementing a call counter +## Example: Implementing a call counter -The following pyclass is a basic decorator - its constructor takes a Python object -as argument and calls that object when called. An equivalent Python implementation -is linked at the end. +The following pyclass is a basic decorator - its constructor takes a Python object as argument and calls that object when called. +An equivalent Python implementation is linked at the end. An example crate containing this pyclass can be found [here](https://github.com/PyO3/pyo3/tree/main/examples/decorator) @@ -66,7 +65,7 @@ def Counter(wraps): return call ``` -### What is the `Cell` for? +### What is the `AtomicU64` for? A [previous implementation] used a normal `u64`, which meant it required a `&mut self` receiver to update the count: @@ -75,8 +74,8 @@ A [previous implementation] used a normal `u64`, which meant it required a `&mut fn __call__( &mut self, py: Python<'_>, - args: &PyTuple, - kwargs: Option<&PyDict>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { self.count += 1; let name = self.wraps.getattr(py, "__name__")?; @@ -92,9 +91,9 @@ fn __call__( } ``` -The problem with this is that the `&mut self` receiver means PyO3 has to borrow it exclusively, - and hold this borrow across the`self.wraps.call(py, args, kwargs)` call. This call returns control to the user's Python code - which is free to call arbitrary things, *including* the decorated function. If that happens PyO3 is unable to create a second unique borrow and will be forced to raise an exception. +The problem with this is that the `&mut self` receiver means PyO3 has to borrow it exclusively, and hold this borrow across the `self.wraps.call(py, args, kwargs)` call. +This call returns control to the user's Python code which is free to call arbitrary things, *including* the decorated function. +If that happens PyO3 is unable to create a second unique borrow and will be forced to raise an exception. As a result, something innocent like this will raise an exception: @@ -108,14 +107,18 @@ say_hello() # RuntimeError: Already borrowed ``` -The implementation in this chapter fixes that by never borrowing exclusively; all the methods take `&self` as receivers, of which multiple may exist simultaneously. This requires a shared counter and the easiest way to do that is to use [`Cell`], so that's what is used here. +The implementation in this chapter fixes that by never borrowing exclusively; all the methods take `&self` as receivers, of which multiple may exist simultaneously. +This requires a shared counter and the most straightforward way to implement thread-safe interior mutability (e.g. the type does not need to accept `&mut self` to modify the "interior" state) for a `u64` is to use [`AtomicU64`], so that's what is used here. This shows the dangers of running arbitrary Python code - note that "running arbitrary Python code" can be far more subtle than the example above: + - Python's asynchronous executor may park the current thread in the middle of Python code, even in Python code that *you* control, and let other Python code run. - Dropping arbitrary Python objects may invoke destructors defined in Python (`__del__` methods). - Calling Python's C-api (most PyO3 apis call C-api functions internally) may raise exceptions, which may allow Python code in signal handlers to run. +- On the free-threaded build, users might use Python's `threading` module to work with your types simultaneously from multiple OS threads. -This is especially important if you are writing unsafe code; Python code must never be able to cause undefined behavior. You must ensure that your Rust code is in a consistent state before doing any of the above things. +This is especially important if you are writing unsafe code; Python code must never be able to cause undefined behavior. +You must ensure that your Rust code is in a consistent state before doing any of the above things. -[previous implementation]: https://github.com/PyO3/pyo3/discussions/2598 "Thread Safe Decorator · Discussion #2598 · PyO3/pyo3" -[`Cell`]: https://doc.rust-lang.org/std/cell/struct.Cell.html "Cell in std::cell - Rust" +[previous implementation]: "Thread Safe Decorator · Discussion #2598 · PyO3/pyo3" +[`AtomicU64`]: "AtomicU64 in std::sync::atomic - Rust" diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index c6b6a65b711..d7801b45f1a 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -2,14 +2,17 @@ At this point we have a `Number` class that we can't actually do any math on! -Before proceeding, we should think about how we want to handle overflows. There are three obvious solutions: -- We can have infinite precision just like Python's `int`. However that would be quite boring - we'd - be reinventing the wheel. +Before proceeding, we should think about how we want to handle overflows. +There are three obvious solutions: + +- We can have infinite precision just like Python's `int`. + However that would be quite boring - we'd be reinventing the wheel. - We can raise exceptions whenever `Number` overflows, but that makes the API painful to use. -- We can wrap around the boundary of `i32`. This is the approach we'll take here. To do that we'll just forward to `i32`'s - `wrapping_*` methods. +- We can wrap around the boundary of `i32`. + This is the approach we'll take here. + To do that we'll just forward to `i32`'s `wrapping_*` methods. -### Fixing our constructor +## Fixing our constructor Let's address the first overflow, in `Number`'s constructor: @@ -26,29 +29,28 @@ Traceback (most recent call last): OverflowError: Python int too large to convert to C long ``` -Instead of relying on the default [`FromPyObject`] extraction to parse arguments, we can specify our -own extraction function, using the `#[pyo3(from_py_with = "...")]` attribute. Unfortunately PyO3 -doesn't provide a way to wrap Python integers out of the box, but we can do a Python call to mask it -and cast it to an `i32`. +Instead of relying on the default [`FromPyObject`] extraction to parse arguments, we can specify our own extraction function, using the `#[pyo3(from_py_with = ...)]` attribute. +Unfortunately PyO3 doesn't provide a way to wrap Python integers out of the box, but we can do a Python call to mask it and cast it to an `i32`. -```rust +```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; -fn wrap(obj: &PyAny) -> Result { +fn wrap(obj: &Bound<'_, PyAny>) -> PyResult { let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; let val: u32 = val.extract()?; // 👇 This intentionally overflows! Ok(val as i32) } ``` + We also add documentation, via `///` comments, which are visible to Python users. -```rust +```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; -fn wrap(obj: &PyAny) -> Result { +fn wrap(obj: &Bound<'_, PyAny>) -> PyResult { let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; let val: u32 = val.extract()?; Ok(val as i32) @@ -62,15 +64,15 @@ struct Number(i32); #[pymethods] impl Number { #[new] - fn new(#[pyo3(from_py_with = "wrap")] value: i32) -> Self { + fn new(#[pyo3(from_py_with = wrap)] value: i32) -> Self { Self(value) } } ``` - With that out of the way, let's implement some operators: -```rust + +```rust,no_run use pyo3::exceptions::{PyZeroDivisionError, PyValueError}; # use pyo3::prelude::*; @@ -124,7 +126,7 @@ impl Number { ### Unary arithmetic operations -```rust +```rust,no_run # use pyo3::prelude::*; # # #[pyclass] @@ -150,9 +152,9 @@ impl Number { } ``` -### Support for the `complex()`, `int()` and `float()` built-in functions. +### Support for the `complex()`, `int()` and `float()` built-in functions -```rust +```rust,no_run # use pyo3::prelude::*; # # #[pyclass] @@ -171,7 +173,7 @@ impl Number { } fn __complex__<'py>(&self, py: Python<'py>) -> Bound<'py, PyComplex> { - PyComplex::from_doubles_bound(py, self.0 as f64, 0.0) + PyComplex::from_doubles(py, self.0 as f64, 0.0) } } ``` @@ -206,14 +208,13 @@ assert hash_djb2('l50_50') == Number(-1152549421) ```rust use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; -use std::convert::TryInto; use pyo3::exceptions::{PyValueError, PyZeroDivisionError}; use pyo3::prelude::*; use pyo3::class::basic::CompareOp; -use pyo3::types::PyComplex; +use pyo3::types::{PyComplex, PyString}; -fn wrap(obj: &PyAny) -> Result { +fn wrap(obj: &Bound<'_, PyAny>) -> PyResult { let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; let val: u32 = val.extract()?; Ok(val as i32) @@ -226,13 +227,13 @@ struct Number(i32); #[pymethods] impl Number { #[new] - fn new(#[pyo3(from_py_with = "wrap")] value: i32) -> Self { + fn new(#[pyo3(from_py_with = wrap)] value: i32) -> Self { Self(value) } - fn __repr__(slf: &PyCell) -> PyResult { + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { // Get the class name dynamically in case `Number` is subclassed - let class_name: String = slf.get_type().qualname()?; + let class_name: Bound<'_, PyString> = slf.get_type().qualname()?; Ok(format!("{}({})", class_name, slf.borrow().0)) } @@ -322,16 +323,16 @@ impl Number { } fn __complex__<'py>(&self, py: Python<'py>) -> Bound<'py, PyComplex> { - PyComplex::from_doubles_bound(py, self.0 as f64, 0.0) + PyComplex::from_doubles(py, self.0 as f64, 0.0) } } #[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) +mod my_module { + #[pymodule_export] + use super::Number; } -# const SCRIPT: &'static str = r#" +# const SCRIPT: &'static std::ffi::CStr = cr#" # def hash_djb2(s: str): # n = Number(0) # five = Number(5) @@ -386,11 +387,11 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { # use pyo3::PyTypeInfo; # # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { -# let globals = PyModule::import(py, "__main__")?.dict().as_borrowed(); -# globals.set_item("Number", Number::type_object_bound(py))?; +# Python::attach(|py| -> PyResult<()> { +# let globals = PyModule::import(py, "__main__")?.dict(); +# globals.set_item("Number", Number::type_object(py))?; # -# py.run_bound(SCRIPT, Some(&globals), None)?; +# py.run(SCRIPT, Some(&globals), None)?; # Ok(()) # }) # } @@ -398,31 +399,35 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { ## Appendix: Writing some unsafe code -At the beginning of this chapter we said that PyO3 doesn't provide a way to wrap Python integers out -of the box but that's a half truth. There's not a PyO3 API for it, but there's a Python C API -function that does: +At the beginning of this chapter we said that PyO3 doesn't provide a way to wrap Python integers out of the box but that's a half truth. +There's not a PyO3 API for it, but there's a Python C API function that does: ```c unsigned long PyLong_AsUnsignedLongMask(PyObject *obj) ``` -We can call this function from Rust by using [`pyo3::ffi::PyLong_AsUnsignedLongMask`]. This is an *unsafe* -function, which means we have to use an unsafe block to call it and take responsibility for upholding -the contracts of this function. Let's review those contracts: -- The GIL must be held. If it's not, calling this function causes a data race. +We can call this function from Rust by using [`pyo3::ffi::PyLong_AsUnsignedLongMask`]. +This is an *unsafe* function, which means we have to use an unsafe block to call it and take responsibility for upholding the contracts of this function. +Let's review those contracts: + +- We must be attached to the interpreter. + If we're not, calling this function causes a data race. - The pointer must be valid, i.e. it must be properly aligned and point to a valid Python object. -Let's create that helper function. The signature has to be `fn(&PyAny) -> PyResult`. -- `&PyAny` represents a checked borrowed reference, so the pointer derived from it is valid (and not null). -- Whenever we have borrowed references to Python objects in scope, it is guaranteed that the GIL is held. This reference is also where we can get a [`Python`] token to use in our call to [`PyErr::take`]. +Let's create that helper function. +The signature has to be `fn(&Bound<'_, PyAny>) -> PyResult`. -```rust +- `&Bound<'_, PyAny>` represents a checked bound reference, so the pointer derived from it is valid (and not null). +- Whenever we have bound references to Python objects in scope, it is guaranteed that we're attached to the interpreter. + This reference is also where we can get a [`Python`] token to use in our call to [`PyErr::take`]. + +```rust,no_run # #![allow(dead_code)] -use std::os::raw::c_ulong; +use std::ffi::c_ulong; use pyo3::prelude::*; use pyo3::ffi; -fn wrap(obj: &PyAny) -> Result { +fn wrap(obj: &Bound<'_, PyAny>) -> Result { let py: Python<'_> = obj.py(); unsafe { @@ -441,6 +446,6 @@ fn wrap(obj: &PyAny) -> Result { ``` [`PyErr::take`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/struct.PyErr.html#method.take -[`Python`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html +[`Python`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html [`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html [`pyo3::ffi::PyLong_AsUnsignedLongMask`]: {{#PYO3_DOCS_URL}}/pyo3/ffi/fn.PyLong_AsUnsignedLongMask.html diff --git a/guide/src/class/object.md b/guide/src/class/object.md index 5d4f53a17d4..12f3dd7f06b 100644 --- a/guide/src/class/object.md +++ b/guide/src/class/object.md @@ -2,8 +2,9 @@ Recall the `Number` class from the previous chapter: -```rust +```rust,no_run # #![allow(dead_code)] +# fn main() {} use pyo3::prelude::*; #[pyclass] @@ -18,9 +19,9 @@ impl Number { } #[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) +mod my_module { + #[pymodule_export] + use super::Number; } ``` @@ -38,13 +39,13 @@ print(n) ``` -### String representations +## String representations -It can't even print an user-readable representation of itself! We can fix that by defining the -`__repr__` and `__str__` methods inside a `#[pymethods]` block. We do this by accessing the value -contained inside `Number`. +It can't even print an user-readable representation of itself! +We can fix that by defining the `__repr__` and `__str__` methods inside a `#[pymethods]` block. +We do this by accessing the value contained inside `Number`. -```rust +```rust,no_run # use pyo3::prelude::*; # # #[pyclass] @@ -70,26 +71,69 @@ impl Number { } ``` -#### Accessing the class name +To automatically generate the `__str__` implementation using a `Display` trait implementation, pass the `str` argument to `pyclass`. -In the `__repr__`, we used a hard-coded class name. This is sometimes not ideal, -because if the class is subclassed in Python, we would like the repr to reflect -the subclass name. This is typically done in Python code by accessing -`self.__class__.__name__`. In order to be able to access the Python type information -*and* the Rust struct, we need to use a `PyCell` as the `self` argument. +```rust,no_run +# use std::fmt::{Display, Formatter}; +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +#[pyclass(str)] +struct Coordinate { + x: i32, + y: i32, + z: i32, +} -```rust +impl Display for Coordinate { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "({}, {}, {})", self.x, self.y, self.z) + } +} +``` + +For convenience, a shorthand format string can be passed to `str` as `str=""` for **structs only**. It expands and is passed into the `format!` macro in the following ways: + +- `"{x}"` -> `"{}", self.x` +- `"{0}"` -> `"{}", self.0` +- `"{x:?}"` -> `"{:?}", self.x` + +*Note: Depending upon the format string you use, this may require implementation of the `Display` or `Debug` traits for the given Rust types.* +*Note: the pyclass args `name` and `rename_all` are incompatible with the shorthand format string and will raise a compile time error.* + +```rust,no_run +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +#[pyclass(str="({x}, {y}, {z})")] +struct Coordinate { + x: i32, + y: i32, + z: i32, +} +``` + +### Accessing the class name + +In the `__repr__`, we used a hard-coded class name. +This is sometimes not ideal, because if the class is subclassed in Python, we would like the repr to reflect the subclass name. +This is typically done in Python code by accessing `self.__class__.__name__`. +In order to be able to access the Python type information *and* the Rust struct, we need to use a `Bound` as the `self` argument. + +```rust,no_run # use pyo3::prelude::*; +# use pyo3::types::PyString; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # #[pymethods] impl Number { - fn __repr__(slf: &PyCell) -> PyResult { + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { // This is the equivalent of `self.__class__.__name__` in Python. - let class_name: String = slf.get_type().qualname()?; - // To access fields of the Rust struct, we need to borrow the `PyCell`. + let class_name: Bound<'_, PyString> = slf.get_type().qualname()?; + // To access fields of the Rust struct, we need to borrow from the Bound object. Ok(format!("{}({})", class_name, slf.borrow().0)) } } @@ -97,11 +141,12 @@ impl Number { ### Hashing +Let's also implement hashing. +We'll just hash the `i32`. +For that we need a [`Hasher`]. +The one provided by `std` is [`DefaultHasher`], which uses the [SipHash] algorithm. -Let's also implement hashing. We'll just hash the `i32`. For that we need a [`Hasher`]. The one -provided by `std` is [`DefaultHasher`], which uses the [SipHash] algorithm. - -```rust +```rust,no_run use std::collections::hash_map::DefaultHasher; // Required to call the `.hash` and `.finish` methods, which are defined on traits. @@ -109,6 +154,7 @@ use std::hash::{Hash, Hasher}; # use pyo3::prelude::*; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # @@ -122,6 +168,22 @@ impl Number { } ``` +To implement `__hash__` using the Rust [`Hash`] trait implementation, the `hash` option can be used. +This option is only available for `frozen` classes to prevent accidental hash changes from mutating the object. +If you need an `__hash__` implementation for a mutable class, use the manual method from above. +This option also requires `eq`: According to the [Python docs](https://docs.python.org/3/reference/datamodel.html#object.__hash__) "If a class does not define an `__eq__()` method it should not define a `__hash__()` operation either" + +```rust,no_run +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +#[pyclass(frozen, eq, hash)] +#[derive(PartialEq, Hash)] +struct Number(i32); +``` + + + > **Note**: When implementing `__hash__` and comparisons, it is important that the following property holds: > > ```text @@ -136,7 +198,7 @@ impl Number { > Types which should not be hashable can override this by setting `__hash__` to None. > This is the same mechanism as for a pure-Python class. This is done like so: > -> ```rust +> ```rust,no_run > # use pyo3::prelude::*; > #[pyclass] > struct NotHashable {} @@ -148,17 +210,20 @@ impl Number { > } > ``` + + ### Comparisons -PyO3 supports the usual magic comparison methods available in Python such as `__eq__`, `__lt__` -and so on. It is also possible to support all six operations at once with `__richcmp__`. +PyO3 supports the usual magic comparison methods available in Python such as `__eq__`, `__lt__` and so on. +It is also possible to support all six operations at once with `__richcmp__`. This method will be called with a value of `CompareOp` depending on the operation. -```rust +```rust,no_run use pyo3::class::basic::CompareOp; # use pyo3::prelude::*; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # @@ -180,11 +245,12 @@ impl Number { If you obtain the result by comparing two Rust values, as in this example, you can take a shortcut using `CompareOp::matches`: -```rust +```rust,no_run use pyo3::class::basic::CompareOp; # use pyo3::prelude::*; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # @@ -201,7 +267,6 @@ the given `CompareOp`. Alternatively, you can implement just equality using `__eq__`: - ```rust # use pyo3::prelude::*; # @@ -216,9 +281,9 @@ impl Number { } # fn main() -> PyResult<()> { -# Python::with_gil(|py| { -# let x = PyCell::new(py, Number(4))?; -# let y = PyCell::new(py, Number(4))?; +# Python::attach(|py| { +# let x = &Bound::new(py, Number(4))?; +# let y = &Bound::new(py, Number(4))?; # assert!(x.eq(y)?); # assert!(!x.ne(y)?); # Ok(()) @@ -226,13 +291,36 @@ impl Number { # } ``` +To implement `__eq__` using the Rust [`PartialEq`] trait implementation, the `eq` option can be used. + +```rust,no_run +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +#[pyclass(eq)] +#[derive(PartialEq)] +struct Number(i32); +``` + +To implement `__lt__`, `__le__`, `__gt__`, & `__ge__` using the Rust `PartialOrd` trait implementation, the `ord` option can be used. *Note: Requires `eq`.* + +```rust,no_run +# use pyo3::prelude::*; +# +# #[allow(dead_code)] +#[pyclass(eq, ord)] +#[derive(PartialEq, PartialOrd)] +struct Number(i32); +``` + ### Truthyness We'll consider `Number` to be `True` if it is nonzero: -```rust +```rust,no_run # use pyo3::prelude::*; # +# #[allow(dead_code)] # #[pyclass] # struct Number(i32); # @@ -246,12 +334,14 @@ impl Number { ### Final code -```rust +```rust,no_run +# fn main() {} use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; use pyo3::prelude::*; use pyo3::class::basic::CompareOp; +use pyo3::types::PyString; #[pyclass] struct Number(i32); @@ -263,8 +353,8 @@ impl Number { Self(value) } - fn __repr__(slf: &PyCell) -> PyResult { - let class_name: String = slf.get_type().qualname()?; + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { + let class_name: Bound<'_, PyString> = slf.get_type().qualname()?; Ok(format!("{}({})", class_name, slf.borrow().0)) } @@ -295,9 +385,9 @@ impl Number { } #[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) +mod my_module { + #[pymodule_export] + use super::Number; } ``` @@ -305,3 +395,4 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { [`Hasher`]: https://doc.rust-lang.org/std/hash/trait.Hasher.html [`DefaultHasher`]: https://doc.rust-lang.org/std/collections/hash_map/struct.DefaultHasher.html [SipHash]: https://en.wikipedia.org/wiki/SipHash +[`PartialEq`]: https://doc.rust-lang.org/stable/std/cmp/trait.PartialEq.html diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 411978f0567..aa99ea95554 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -1,50 +1,64 @@ -# Magic methods and slots +# Class customizations -Python's object model defines several protocols for different object behavior, such as the sequence, mapping, and number protocols. You may be familiar with implementing these protocols in Python classes by "magic" methods, such as `__str__` or `__repr__`. Because of the double-underscores surrounding their name, these are also known as "dunder" methods. +Python's object model defines several protocols for different object behavior, such as the sequence, mapping, and number protocols. +Python classes support these protocols by implementing "magic" methods, such as `__str__` or `__repr__`. +Because of the double-underscores surrounding their name, these are also known as "dunder" methods. -In the Python C-API which PyO3 is implemented upon, many of these magic methods have to be placed into special "slots" on the class type object, as covered in the previous section. +PyO3 makes it possible for every magic method to be implemented in `#[pymethods]` just as they would be done in a regular Python class, with a few notable differences: -If a function name in `#[pymethods]` is a recognised magic method, it will be automatically placed into the correct slot in the Python type object. The function name is taken from the usual rules for naming `#[pymethods]`: the `#[pyo3(name = "...")]` attribute is used if present, otherwise the Rust function name is used. +- `__new__` and `__init__` are replaced by the [`#[new]` attribute](../class.md#constructor). +- `__del__` is not yet supported, but may be in the future. +- `__buffer__` and `__release_buffer__` are currently not supported and instead PyO3 supports [`__getbuffer__` and `__releasebuffer__`](#buffer-objects) methods (these predate [PEP 688](https://peps.python.org/pep-0688/#python-level-buffer-protocol)), again this may change in the future. +- PyO3 adds [`__traverse__` and `__clear__`](#garbage-collector-integration) methods for controlling garbage collection. +- The Python C-API which PyO3 is implemented upon requires many magic methods to have a specific function signature in C and be placed into special "slots" on the class type object. + This limits the allowed argument and return types for these methods. + They are listed in detail in the section below. -The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). Some of the slots do not have a magic method in Python, which leads to a few additional magic methods defined only in PyO3: - - Magic methods for garbage collection - - Magic methods for the buffer protocol +If a magic method is not on the list above (for example `__init_subclass__`), then it should just work in PyO3. +If this is not the case, please file a bug report. + +## Magic Methods handled by PyO3 + +If a function name in `#[pymethods]` is a magic method which is known to need special handling, it will be automatically placed into the correct slot in the Python type object. +The function name is taken from the usual rules for naming `#[pymethods]`: the `#[pyo3(name = "...")]` attribute is used if present, otherwise the Rust function name is used. + +The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). When PyO3 handles a magic method, a couple of changes apply compared to other `#[pymethods]`: - - The Rust function signature is restricted to match the magic method. - - The `#[pyo3(signature = (...)]` and `#[pyo3(text_signature = "...")]` attributes are not allowed. -The following sections list of all magic methods PyO3 currently handles. The +- The Rust function signature is restricted to match the magic method. +- The `#[pyo3(signature = (...)]` and `#[pyo3(text_signature = "...")]` attributes are not allowed. + +The following sections list all magic methods for which PyO3 implements the necessary special handling. The given signatures should be interpreted as follows: - - All methods take a receiver as first argument, shown as ``. It can be - `&self`, `&mut self` or a `PyCell` reference like `self_: PyRef<'_, Self>` and - `self_: PyRefMut<'_, Self>`, as described [here](../class.md#inheritance). - - An optional `Python<'py>` argument is always allowed as the first argument. - - Return values can be optionally wrapped in `PyResult`. - - `object` means that any type is allowed that can be extracted from a Python + +- All methods take a receiver as first argument, shown as ``. + It can be `&self`, `&mut self` or a `Bound` reference like `self_: PyRef<'_, Self>` and `self_: PyRefMut<'_, Self>`, as described [here](../class.md#inheritance). +- An optional `Python<'py>` argument is always allowed as the first argument. +- Return values can be optionally wrapped in `PyResult`. +- `object` means that any type is allowed that can be extracted from a Python object (if argument) or converted to a Python object (if return value). - - Other types must match what's given, e.g. `pyo3::basic::CompareOp` for +- Other types must match what's given, e.g. `pyo3::basic::CompareOp` for `__richcmp__`'s second argument. - - For the comparison and arithmetic methods, extraction errors are not +- For the comparison and arithmetic methods, extraction errors are not propagated as exceptions, but lead to a return of `NotImplemented`. - - For some magic methods, the return values are not restricted by PyO3, but - checked by the Python interpreter. For example, `__str__` needs to return a - string object. This is indicated by `object (Python type)`. - +- For some magic methods, the return values are not restricted by PyO3, but checked by the Python interpreter. + For example, `__str__` needs to return a string object. This is indicated by `object (Python type)`. ### Basic object customization - - `__str__() -> object (str)` - - `__repr__() -> object (str)` +- `__str__() -> object (str)` +- `__repr__() -> object (str)` - - `__hash__() -> isize` +- `__hash__() -> isize` Objects that compare equal must have the same hash value. Any type up to 64 bits may be returned instead of `isize`, PyO3 will convert to an isize automatically (wrapping unsigned types like `u64` and `usize`).
Disabling Python's default hash + By default, all `#[pyclass]` types have a default hash implementation from Python. Types which should not be hashable can override this by setting `__hash__` to `None`. This is the same mechanism as for a pure-Python class. This is done like so: - ```rust + ```rust,no_run # use pyo3::prelude::*; # #[pyclass] @@ -53,17 +67,18 @@ given signatures should be interpreted as follows: #[pymethods] impl NotHashable { #[classattr] - const __hash__: Option = None; + const __hash__: Option> = None; } ``` +
- - `__lt__(, object) -> object` - - `__le__(, object) -> object` - - `__eq__(, object) -> object` - - `__ne__(, object) -> object` - - `__gt__(, object) -> object` - - `__ge__(, object) -> object` +- `__lt__(, object) -> object` +- `__le__(, object) -> object` +- `__eq__(, object) -> object` +- `__ne__(, object) -> object` +- `__gt__(, object) -> object` +- `__ge__(, object) -> object` The implementations of Python's "rich comparison" operators `<`, `<=`, `==`, `!=`, `>` and `>=` respectively. @@ -73,7 +88,7 @@ given signatures should be interpreted as follows: The return type will normally be `bool` or `PyResult`, however any Python object can be returned. - - `__richcmp__(, object, pyo3::basic::CompareOp) -> object` +- `__richcmp__(, object, pyo3::basic::CompareOp) -> object` Implements Python comparison operations (`==`, `!=`, `<`, `<=`, `>`, and `>=`) in a single method. The `CompareOp` argument indicates the comparison operation being performed. You can use @@ -89,21 +104,23 @@ given signatures should be interpreted as follows: If you want to leave some operations unimplemented, you can return `py.NotImplemented()` for some of the operations: - ```rust + ```rust,no_run use pyo3::class::basic::CompareOp; + use pyo3::types::PyNotImplemented; # use pyo3::prelude::*; + # use pyo3::BoundObject; # # #[pyclass] # struct Number(i32); # #[pymethods] impl Number { - fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> PyObject { + fn __richcmp__<'py>(&self, other: &Self, op: CompareOp, py: Python<'py>) -> PyResult> { match op { - CompareOp::Eq => (self.0 == other.0).into_py(py), - CompareOp::Ne => (self.0 != other.0).into_py(py), - _ => py.NotImplemented(), + CompareOp::Eq => Ok((self.0 == other.0).into_pyobject(py)?.into_any()), + CompareOp::Ne => Ok((self.0 != other.0).into_pyobject(py)?.into_any()), + _ => Ok(PyNotImplemented::get(py).into_any()), } } } @@ -113,8 +130,8 @@ given signatures should be interpreted as follows: signature, the generated code will automatically `return NotImplemented`. - - `__getattr__(, object) -> object` - - `__getattribute__(, object) -> object` +- `__getattr__(, object) -> object` +- `__getattribute__(, object) -> object`
Differences between `__getattr__` and `__getattribute__` As in Python, `__getattr__` is only called if the attribute is not found @@ -124,35 +141,37 @@ given signatures should be interpreted as follows: infinite recursion, and use `baseclass.__getattribute__()`.
- - `__setattr__(, value: object) -> ()` - - `__delattr__(, object) -> ()` +- `__setattr__(, value: object) -> ()` +- `__delattr__(, object) -> ()` Overrides attribute access. - - `__bool__() -> bool` +- `__bool__() -> bool` Determines the "truthyness" of an object. - - `__call__(, ...) -> object` - here, any argument list can be defined +- `__call__(, ...) -> object` - here, any argument list can be defined as for normal `pymethods` ### Iterable objects Iterators can be defined using these methods: - - `__iter__() -> object` - - `__next__() -> Option or IterNextOutput` ([see details](#returning-a-value-from-iteration)) +- `__iter__() -> object` +- `__next__() -> Option or IterNextOutput` ([see details](#returning-a-value-from-iteration)) Returning `None` from `__next__` indicates that that there are no further items. Example: -```rust +```rust,no_run use pyo3::prelude::*; +use std::sync::Mutex; + #[pyclass] struct MyIterator { - iter: Box + Send>, + iter: Mutex> + Send>>, } #[pymethods] @@ -160,18 +179,17 @@ impl MyIterator { fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } - fn __next__(mut slf: PyRefMut<'_, Self>) -> Option { - slf.iter.next() + fn __next__(slf: PyRefMut<'_, Self>) -> Option> { + slf.iter.lock().unwrap().next() } } ``` -In many cases you'll have a distinction between the type being iterated over -(i.e. the *iterable*) and the iterator it provides. In this case, the iterable -only needs to implement `__iter__()` while the iterator must implement both -`__iter__()` and `__next__()`. For example: +In many cases you'll have a distinction between the type being iterated over (i.e. the *iterable*) and the iterator it provides. +In this case, the iterable only needs to implement `__iter__()` while the iterator must implement both `__iter__()` and `__next__()`. +For example: -```rust +```rust,no_run # use pyo3::prelude::*; #[pyclass] @@ -205,9 +223,9 @@ impl Container { } } -# Python::with_gil(|py| { +# Python::attach(|py| { # let container = Container { iter: vec![1, 2, 3, 4] }; -# let inst = pyo3::PyCell::new(py, container).unwrap(); +# let inst = pyo3::Py::new(py, container).unwrap(); # pyo3::py_run!(py, inst, "assert list(inst) == [1, 2, 3, 4]"); # pyo3::py_run!(py, inst, "assert list(iter(iter(inst))) == [1, 2, 3, 4]"); # }); @@ -218,38 +236,46 @@ documentation](https://docs.python.org/library/stdtypes.html#iterator-types). #### Returning a value from iteration -This guide has so far shown how to use `Option` to implement yielding values -during iteration. In Python a generator can also return a value. To express -this in Rust, PyO3 provides the [`IterNextOutput`] enum to both `Yield` values -and `Return` a final value - see its docs for further details and an example. +This guide has so far shown how to use `Option` to implement yielding values during iteration. In Python a generator can also return a value. +This is done by raising a `StopIteration` exception. +To express this in Rust, return `PyResult::Err` with a `PyStopIteration` as the error. ### Awaitable objects - - `__await__() -> object` - - `__aiter__() -> object` - - `__anext__() -> Option or IterANextOutput` +- `__await__() -> object` +- `__aiter__() -> object` +- `__anext__() -> Option` ### Mapping & Sequence types -The magic methods in this section can be used to implement Python container types. They are two main categories of container in Python: "mappings" such as `dict`, with arbitrary keys, and "sequences" such as `list` and `tuple`, with integer keys. +The magic methods in this section can be used to implement Python container types. +They are two main categories of container in Python: "mappings" such as `dict`, with arbitrary keys, and "sequences" such as `list` and `tuple`, with integer keys. + +The Python C-API which PyO3 is built upon has separate "slots" for sequences and mappings. +When writing a `class` in pure Python, there is no such distinction in the implementation - a `__getitem__` implementation will fill the slots for both the mapping and sequence forms, for example. -The Python C-API which PyO3 is built upon has separate "slots" for sequences and mappings. When writing a `class` in pure Python, there is no such distinction in the implementation - a `__getitem__` implementation will fill the slots for both the mapping and sequence forms, for example. +By default PyO3 reproduces the Python behaviour of filling both mapping and sequence slots. +This makes sense for the "simple" case which matches Python, and also for sequences, where the mapping slot is used anyway to implement slice indexing. -By default PyO3 reproduces the Python behaviour of filling both mapping and sequence slots. This makes sense for the "simple" case which matches Python, and also for sequences, where the mapping slot is used anyway to implement slice indexing. +Mapping types usually will not want the sequence slots filled. +Having them filled will lead to outcomes which may be unwanted, such as: -Mapping types usually will not want the sequence slots filled. Having them filled will lead to outcomes which may be unwanted, such as: -- The mapping type will successfully cast to [`PySequence`]. This may lead to consumers of the type handling it incorrectly. -- Python provides a default implementation of `__iter__` for sequences, which calls `__getitem__` with consecutive positive integers starting from 0 until an `IndexError` is returned. Unless the mapping only contains consecutive positive integer keys, this `__iter__` implementation will likely not be the intended behavior. +- The mapping type will successfully cast to [`PySequence`]. + This may lead to consumers of the type handling it incorrectly. +- Python provides a default implementation of `__iter__` for sequences, which calls `__getitem__` with consecutive positive integers starting from 0 until an `IndexError` is returned. + Unless the mapping only contains consecutive positive integer keys, this `__iter__` implementation will likely not be the intended behavior. -Use the `#[pyclass(mapping)]` annotation to instruct PyO3 to only fill the mapping slots, leaving the sequence ones empty. This will apply to `__getitem__`, `__setitem__`, and `__delitem__`. +Use the `#[pyclass(mapping)]` annotation to instruct PyO3 to only fill the mapping slots, leaving the sequence ones empty. +This will apply to `__getitem__`, `__setitem__`, and `__delitem__`. -Use the `#[pyclass(sequence)]` annotation to instruct PyO3 to fill the `sq_length` slot instead of the `mp_length` slot for `__len__`. This will help libraries such as `numpy` recognise the class as a sequence, however will also cause CPython to automatically add the sequence length to any negative indices before passing them to `__getitem__`. (`__getitem__`, `__setitem__` and `__delitem__` mapping slots are still used for sequences, for slice operations.) +Use the `#[pyclass(sequence)]` annotation to instruct PyO3 to fill the `sq_length` slot instead of the `mp_length` slot for `__len__`. +This will help libraries such as `numpy` recognise the class as a sequence, however will also cause CPython to automatically add the sequence length to any negative indices before passing them to `__getitem__`. (`__getitem__`, `__setitem__` and `__delitem__` mapping slots are still used for sequences, for slice operations.) - - `__len__() -> usize` +- `__len__() -> usize` Implements the built-in function `len()`. - - `__contains__(, object) -> bool` +- `__contains__(, object) -> bool` Implements membership test operators. Should return true if `item` is in `self`, false otherwise. @@ -264,7 +290,7 @@ Use the `#[pyclass(sequence)]` annotation to instruct PyO3 to fill the `sq_lengt can override this by setting `__contains__` to `None`. This is the same mechanism as for a pure-Python class. This is done like so: - ```rust + ```rust,no_run # use pyo3::prelude::*; # #[pyclass] @@ -273,12 +299,13 @@ Use the `#[pyclass(sequence)]` annotation to instruct PyO3 to fill the `sq_lengt #[pymethods] impl NoContains { #[classattr] - const __contains__: Option = None; + const __contains__: Option> = None; } ``` + - - `__getitem__(, object) -> object` +- `__getitem__(, object) -> object` Implements retrieval of the `self[a]` element. @@ -287,39 +314,39 @@ Use the `#[pyclass(sequence)]` annotation to instruct PyO3 to fill the `sq_lengt accessed via `PySequence::get_item`, the underlying C API already adjusts the index to be positive. - - `__setitem__(, object, object) -> ()` +- `__setitem__(, object, object) -> ()` Implements assignment to the `self[a]` element. Should only be implemented if elements can be replaced. Same behavior regarding negative indices as for `__getitem__`. - - `__delitem__(, object) -> ()` +- `__delitem__(, object) -> ()` Implements deletion of the `self[a]` element. Should only be implemented if elements can be deleted. Same behavior regarding negative indices as for `__getitem__`. - * `fn __concat__(&self, other: impl FromPyObject) -> PyResult` +- `fn __concat__(&self, other: impl FromPyObject) -> PyResult` Concatenates two sequences. Used by the `+` operator, after trying the numeric addition via the `__add__` and `__radd__` methods. - * `fn __repeat__(&self, count: isize) -> PyResult` +- `fn __repeat__(&self, count: isize) -> PyResult` Repeats the sequence `count` times. Used by the `*` operator, after trying the numeric multiplication via the `__mul__` and `__rmul__` methods. - * `fn __inplace_concat__(&self, other: impl FromPyObject) -> PyResult` +- `fn __inplace_concat__(&self, other: impl FromPyObject) -> PyResult` Concatenates two sequences. Used by the `+=` operator, after trying the numeric addition via the `__iadd__` method. - * `fn __inplace_repeat__(&self, count: isize) -> PyResult` +- `fn __inplace_repeat__(&self, count: isize) -> PyResult` Concatenates two sequences. Used by the `*=` operator, after trying the numeric multiplication via @@ -327,9 +354,9 @@ Use the `#[pyclass(sequence)]` annotation to instruct PyO3 to fill the `sq_lengt ### Descriptors - - `__get__(, object, object) -> object` - - `__set__(, object, object) -> ()` - - `__delete__(, object) -> ()` +- `__get__(, object, object) -> object` +- `__set__(, object, object) -> ()` +- `__delete__(, object) -> ()` ### Numeric types @@ -339,103 +366,97 @@ Binary arithmetic operations (`+`, `-`, `*`, `@`, `/`, `//`, `%`, `divmod()`, (If the `object` is not of the type specified in the signature, the generated code will automatically `return NotImplemented`.) - - `__add__(, object) -> object` - - `__radd__(, object) -> object` - - `__sub__(, object) -> object` - - `__rsub__(, object) -> object` - - `__mul__(, object) -> object` - - `__rmul__(, object) -> object` - - `__matmul__(, object) -> object` - - `__rmatmul__(, object) -> object` - - `__floordiv__(, object) -> object` - - `__rfloordiv__(, object) -> object` - - `__truediv__(, object) -> object` - - `__rtruediv__(, object) -> object` - - `__divmod__(, object) -> object` - - `__rdivmod__(, object) -> object` - - `__mod__(, object) -> object` - - `__rmod__(, object) -> object` - - `__lshift__(, object) -> object` - - `__rlshift__(, object) -> object` - - `__rshift__(, object) -> object` - - `__rrshift__(, object) -> object` - - `__and__(, object) -> object` - - `__rand__(, object) -> object` - - `__xor__(, object) -> object` - - `__rxor__(, object) -> object` - - `__or__(, object) -> object` - - `__ror__(, object) -> object` - - `__pow__(, object, object) -> object` - - `__rpow__(, object, object) -> object` +- `__add__(, object) -> object` +- `__radd__(, object) -> object` +- `__sub__(, object) -> object` +- `__rsub__(, object) -> object` +- `__mul__(, object) -> object` +- `__rmul__(, object) -> object` +- `__matmul__(, object) -> object` +- `__rmatmul__(, object) -> object` +- `__floordiv__(, object) -> object` +- `__rfloordiv__(, object) -> object` +- `__truediv__(, object) -> object` +- `__rtruediv__(, object) -> object` +- `__divmod__(, object) -> object` +- `__rdivmod__(, object) -> object` +- `__mod__(, object) -> object` +- `__rmod__(, object) -> object` +- `__lshift__(, object) -> object` +- `__rlshift__(, object) -> object` +- `__rshift__(, object) -> object` +- `__rrshift__(, object) -> object` +- `__and__(, object) -> object` +- `__rand__(, object) -> object` +- `__xor__(, object) -> object` +- `__rxor__(, object) -> object` +- `__or__(, object) -> object` +- `__ror__(, object) -> object` +- `__pow__(, object, object) -> object` +- `__rpow__(, object, object) -> object` In-place assignment operations (`+=`, `-=`, `*=`, `@=`, `/=`, `//=`, `%=`, `**=`, `<<=`, `>>=`, `&=`, `^=`, `|=`): - - `__iadd__(, object) -> ()` - - `__isub__(, object) -> ()` - - `__imul__(, object) -> ()` - - `__imatmul__(, object) -> ()` - - `__itruediv__(, object) -> ()` - - `__ifloordiv__(, object) -> ()` - - `__imod__(, object) -> ()` - - `__ipow__(, object, object) -> ()` - - `__ilshift__(, object) -> ()` - - `__irshift__(, object) -> ()` - - `__iand__(, object) -> ()` - - `__ixor__(, object) -> ()` - - `__ior__(, object) -> ()` +- `__iadd__(, object) -> ()` +- `__isub__(, object) -> ()` +- `__imul__(, object) -> ()` +- `__imatmul__(, object) -> ()` +- `__itruediv__(, object) -> ()` +- `__ifloordiv__(, object) -> ()` +- `__imod__(, object) -> ()` +- `__ipow__(, object, object) -> ()` +- `__ilshift__(, object) -> ()` +- `__irshift__(, object) -> ()` +- `__iand__(, object) -> ()` +- `__ixor__(, object) -> ()` +- `__ior__(, object) -> ()` Unary operations (`-`, `+`, `abs()` and `~`): - - `__pos__() -> object` - - `__neg__() -> object` - - `__abs__() -> object` - - `__invert__() -> object` +- `__pos__() -> object` +- `__neg__() -> object` +- `__abs__() -> object` +- `__invert__() -> object` Coercions: - - `__index__() -> object (int)` - - `__int__() -> object (int)` - - `__float__() -> object (float)` +- `__index__() -> object (int)` +- `__int__() -> object (int)` +- `__float__() -> object (float)` ### Buffer objects - - `__getbuffer__(, *mut ffi::Py_buffer, flags) -> ()` - - `__releasebuffer__(, *mut ffi::Py_buffer) -> ()` - Errors returned from `__releasebuffer__` will be sent to `sys.unraiseablehook`. It is strongly advised to never return an error from `__releasebuffer__`, and if it really is necessary, to make best effort to perform any required freeing operations before returning. `__releasebuffer__` will not be called a second time; anything not freed will be leaked. +- `__getbuffer__(, *mut ffi::Py_buffer, flags) -> ()` +- `__releasebuffer__(, *mut ffi::Py_buffer) -> ()` Errors returned from `__releasebuffer__` will be sent to `sys.unraiseablehook`. + It is strongly advised to never return an error from `__releasebuffer__`, and if it really is necessary, to make best effort to perform any required freeing operations before returning. `__releasebuffer__` will not be called a second time; anything not freed will be leaked. ### Garbage Collector Integration -If your type owns references to other Python objects, you will need to integrate -with Python's garbage collector so that the GC is aware of those references. To -do this, implement the two methods `__traverse__` and `__clear__`. These -correspond to the slots `tp_traverse` and `tp_clear` in the Python C API. -`__traverse__` must call `visit.call()` for each reference to another Python -object. `__clear__` must clear out any mutable references to other Python -objects (thus breaking reference cycles). Immutable references do not have to be -cleared, as every cycle must contain at least one mutable reference. +If your type owns references to other Python objects, you will need to integrate with Python's garbage collector so that the GC is aware of those references. To do this, implement the two methods `__traverse__` and `__clear__`. These correspond to the slots `tp_traverse` and `tp_clear` in the Python C API. `__traverse__` must call `visit.call()` for each reference to another Python object. `__clear__` must clear out any mutable references to other Python objects (thus breaking reference cycles). +Immutable references do not have to be cleared, as every cycle must contain at least one mutable reference. + +- `__traverse__(, pyo3::class::gc::PyVisit<'_>) -> Result<(), pyo3::class::gc::PyTraverseError>` +- `__clear__() -> ()` - - `__traverse__(, pyo3::class::gc::PyVisit<'_>) -> Result<(), pyo3::class::gc::PyTraverseError>` - - `__clear__() -> ()` +> Note: `__traverse__` does not work with [`#[pyo3(warn(...))]`](../function.md#warn). Example: -```rust +```rust,no_run use pyo3::prelude::*; use pyo3::PyTraverseError; use pyo3::gc::PyVisit; #[pyclass] struct ClassWithGCSupport { - obj: Option, + obj: Option>, } #[pymethods] impl ClassWithGCSupport { fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { - if let Some(obj) = &self.obj { - visit.call(obj)? - } + visit.call(&self.obj)?; Ok(()) } @@ -447,11 +468,11 @@ impl ClassWithGCSupport { ``` Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. -Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, -i.e. `Python::with_gil` will panic. +Most importantly, safe access to the interpreter is prohibited inside implementations of `__traverse__`, +i.e. `Python::attach` will panic. -> Note: these methods are part of the C API, PyPy does not necessarily honor them. If you are building for PyPy you should measure memory consumption to make sure you do not have runaway memory growth. See [this issue on the PyPy bug tracker](https://foss.heptapod.net/pypy/pypy/-/issues/3899). +> Note: these methods are part of the C API, PyPy does not necessarily honor them. If you are building for PyPy you should measure memory consumption to make sure you do not have runaway memory growth. See [this issue on the PyPy bug tracker](https://github.com/pypy/pypy/issues/3848). -[`IterNextOutput`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass/enum.IterNextOutput.html [`PySequence`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PySequence.html + [`CompareOp::matches`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass/enum.CompareOp.html#method.matches diff --git a/guide/src/class/thread-safety.md b/guide/src/class/thread-safety.md new file mode 100644 index 00000000000..eb2b029617f --- /dev/null +++ b/guide/src/class/thread-safety.md @@ -0,0 +1,121 @@ +# `#[pyclass]` thread safety + +Python objects are freely shared between threads by the Python interpreter. +This means that: + +- there is no control which thread might eventually drop the `#[pyclass]` object, meaning `Send` is required. +- multiple threads can potentially be reading the `#[pyclass]` data simultaneously, meaning `Sync` is required. + +This section of the guide discusses various data structures which can be used to make types satisfy these requirements. + +In special cases where it is known that your Python application is never going to use threads (this is rare!), these thread-safety requirements can be opted-out with [`#[pyclass(unsendable)]`](../class.md#customizing-the-class), at the cost of making concurrent access to the Rust data be runtime errors. +This is only for very specific use cases; it is almost always better to make proper thread-safe types. + +## Making `#[pyclass]` types thread-safe + +The general challenge with thread-safety is to make sure that two threads cannot produce a data race, i.e. unsynchronized writes to the same data at the same time. +A data race produces an unpredictable result and is forbidden by Rust. + +By default, `#[pyclass]` employs an ["interior mutability" pattern](../class.md#bound-and-interior-mutability) to allow for either multiple `&T` references or a single exclusive `&mut T` reference to access the data. +This allows for simple `#[pyclass]` types to be thread-safe automatically, at the cost of runtime checking for concurrent access. +Errors will be raised if the usage overlaps. + +For example, the below simple class is thread-safe: + +```rust,no_run +# use pyo3::prelude::*; + +#[pyclass] +struct MyClass { + x: i32, + y: i32, +} + +#[pymethods] +impl MyClass { + fn get_x(&self) -> i32 { + self.x + } + + fn set_y(&mut self, value: i32) { + self.y = value; + } +} +``` + +In the above example, if calls to `get_x` and `set_y` overlap (from two different threads) then at least one of those threads will experience a runtime error indicating that the data was "already borrowed". + +To avoid these errors, you can take control of the interior mutability yourself in one of the following ways. + +### Using atomic data structures + +To remove the possibility of having overlapping `&self` and `&mut self` references produce runtime errors, consider using `#[pyclass(frozen)]` and use [atomic data structures](https://doc.rust-lang.org/std/sync/atomic/) to control modifications directly. + +For example, a thread-safe version of the above `MyClass` using atomic integers would be as follows: + +```rust,no_run +# use pyo3::prelude::*; +use std::sync::atomic::{AtomicI32, Ordering}; + +#[pyclass(frozen)] +struct MyClass { + x: AtomicI32, + y: AtomicI32, +} + +#[pymethods] +impl MyClass { + fn get_x(&self) -> i32 { + self.x.load(Ordering::Relaxed) + } + + fn set_y(&self, value: i32) { + self.y.store(value, Ordering::Relaxed) + } +} +``` + +### Using locks + +An alternative to atomic data structures is to use [locks](https://doc.rust-lang.org/std/sync/struct.Mutex.html) to make threads wait for access to shared data. + +For example, a thread-safe version of the above `MyClass` using locks would be as follows: + +```rust,no_run +# use pyo3::prelude::*; +use std::sync::Mutex; + +struct MyClassInner { + x: i32, + y: i32, +} + +#[pyclass(frozen)] +struct MyClass { + inner: Mutex +} + +#[pymethods] +impl MyClass { + fn get_x(&self) -> i32 { + self.inner.lock().expect("lock not poisoned").x + } + + fn set_y(&self, value: i32) { + self.inner.lock().expect("lock not poisoned").y = value; + } +} +``` + +If you need to lock around state stored in the Python interpreter or otherwise call into the Python C API while a lock is held, you might find the `MutexExt` trait useful. +It provides a `lock_py_attached` method for `std::sync::Mutex` that avoids deadlocks with the GIL or other global synchronization events in the interpreter. +Additionally, support for the `parking_lot` and `lock_api` synchronization libraries is gated behind the `parking_lot` and `lock_api` features. +You can also enable the `arc_lock` feature if you need the `arc_lock` features of either library. + +### Wrapping unsynchronized data + +In some cases, the data structures stored within a `#[pyclass]` may themselves not be thread-safe. +Rust will therefore not implement `Send` and `Sync` on the `#[pyclass]` type. + +To achieve thread-safety, a manual `Send` and `Sync` implementation is required which is `unsafe` and should only be done following careful review of the soundness of the implementation. +Doing this for PyO3 types is no different than for any other Rust code, [the Rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html) has a great discussion on this. diff --git a/guide/src/conversions.md b/guide/src/conversions.md index 991c2061042..ee8dfddebb0 100644 --- a/guide/src/conversions.md +++ b/guide/src/conversions.md @@ -1,3 +1,5 @@ # Type conversions In this portion of the guide we'll talk about the mapping of Python types to Rust types offered by PyO3, as well as the traits available to perform conversions between them. + +See also the conversion [tables](conversions/tables.md) and [traits](conversions/traits.md). diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index b0582431156..1cec2767447 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -1,10 +1,10 @@ -## Mapping of Rust types to Python types +# Mapping of Rust types to Python types -When writing functions callable from Python (such as a `#[pyfunction]` or in a `#[pymethods]` block), the trait `FromPyObject` is required for function arguments, and `IntoPy` is required for function return values. +When writing functions callable from Python (such as a `#[pyfunction]` or in a `#[pymethods]` block), the trait `FromPyObject` is required for function arguments, and `IntoPyObject` is required for function return values. Consult the tables in the following section to find the Rust types provided by PyO3 which implement these traits. -### Argument Types +## Argument Types When accepting a function argument, it is possible to either use Rust library types or PyO3's Python-native types. (See the next section for discussion on when to use each.) @@ -12,71 +12,79 @@ The table below contains the Python type and the corresponding function argument | Python | Rust | Rust (Python-native) | | ------------- |:-------------------------------:|:--------------------:| -| `object` | - | `&PyAny` | -| `str` | `String`, `Cow`, `&str`, `char`, `OsString`, `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | -| `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `&PyBytes` | -| `bool` | `bool` | `&PyBool` | -| `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `&PyLong` | -| `float` | `f32`, `f64` | `&PyFloat` | -| `complex` | `num_complex::Complex`[^2] | `&PyComplex` | -| `list[T]` | `Vec` | `&PyList` | -| `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyDict` | -| `tuple[T, U]` | `(T, U)`, `Vec` | `&PyTuple` | -| `set[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `&PySet` | -| `frozenset[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `&PyFrozenSet` | -| `bytearray` | `Vec`, `Cow<[u8]>` | `&PyByteArray` | -| `slice` | - | `&PySlice` | -| `type` | - | `&PyType` | -| `module` | - | `&PyModule` | +| `object` | - | `PyAny` | +| `str` | `String`, `Cow`, `&str`, `char`, `OsString`, `PathBuf`, `Path` | `PyString` | +| `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `PyBytes` | +| `bool` | `bool` | `PyBool` | +| `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `PyInt` | +| `float` | `f32`, `f64`, `ordered_float::NotNan`[^10], `ordered_float::OrderedFloat`[^10] | `PyFloat` | +| `complex` | `num_complex::Complex`[^2] | `PyComplex` | +| `fractions.Fraction`| `num_rational::Ratio`[^8] | - | +| `list[T]` | `Vec` | `PyList` | +| `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `PyDict` | +| `tuple[T, U]` | `(T, U)`, `Vec` | `PyTuple` | +| `set[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `PySet` | +| `frozenset[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `PyFrozenSet` | +| `bytearray` | `Vec`, `Cow<[u8]>` | `PyByteArray` | +| `slice` | - | `PySlice` | +| `type` | - | `PyType` | +| `module` | - | `PyModule` | | `collections.abc.Buffer` | - | `PyBuffer` | -| `datetime.datetime` | `SystemTime`, `chrono::DateTime`[^5], `chrono::NaiveDateTime`[^5] | `&PyDateTime` | -| `datetime.date` | `chrono::NaiveDate`[^5] | `&PyDate` | -| `datetime.time` | `chrono::NaiveTime`[^5] | `&PyTime` | -| `datetime.tzinfo` | `chrono::FixedOffset`[^5], `chrono::Utc`[^5], `chrono_tz::TimeZone`[^6] | `&PyTzInfo` | -| `datetime.timedelta` | `Duration`, `chrono::Duration`[^5] | `&PyDelta` | +| `datetime.datetime` | `SystemTime`, `chrono::DateTime`[^5], `chrono::NaiveDateTime`[^5] | `PyDateTime` | +| `datetime.date` | `chrono::NaiveDate`[^5] | `PyDate` | +| `datetime.time` | `chrono::NaiveTime`[^5] | `PyTime` | +| `datetime.tzinfo` | `chrono::FixedOffset`[^5], `chrono::Utc`[^5], `chrono_tz::TimeZone`[^6] | `PyTzInfo` | +| `datetime.timedelta` | `Duration`, `chrono::Duration`[^5] | `PyDelta` | | `decimal.Decimal` | `rust_decimal::Decimal`[^7] | - | -| `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::IpV4Addr` | - | -| `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::IpV6Addr` | - | -| `os.PathLike ` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | -| `pathlib.Path` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | +| `decimal.Decimal` | `bigdecimal::BigDecimal`[^9] | - | +| `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::Ipv4Addr` | - | +| `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::Ipv6Addr` | - | +| `os.PathLike` | `PathBuf`, `Path` | `PyString` | +| `pathlib.Path` | `PathBuf`, `Path` | `PyString` | | `typing.Optional[T]` | `Option` | - | -| `typing.Sequence[T]` | `Vec` | `&PySequence` | +| `typing.Sequence[T]` | `Vec` | `PySequence` | | `typing.Mapping[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyMapping` | -| `typing.Iterator[Any]` | - | `&PyIterator` | -| `typing.Union[...]` | See [`#[derive(FromPyObject)]`](traits.html#deriving-a-hrefhttpsdocsrspyo3latestpyo3conversiontraitfrompyobjecthtmlfrompyobjecta-for-enums) | - | +| `typing.Iterator[Any]` | - | `PyIterator` | +| `typing.Union[...]` | See [`#[derive(FromPyObject)]`](traits.md#deriving-frompyobject-for-enums) | - | -There are also a few special types related to the GIL and Rust-defined `#[pyclass]`es which may come in useful: +It is also worth remembering the following special types: -| What | Description | -| ------------- | ------------------------------------- | -| `Python` | A GIL token, used to pass to PyO3 constructors to prove ownership of the GIL | -| `Py` | A Python object isolated from the GIL lifetime. This can be sent to other threads. | -| `PyObject` | An alias for `Py` | -| `&PyCell` | A `#[pyclass]` value owned by Python. | -| `PyRef` | A `#[pyclass]` borrowed immutably. | -| `PyRefMut` | A `#[pyclass]` borrowed mutably. | +| What | Description | +| ---------------- | ------------------------------------- | +| `Python<'py>` | A token used to prove attachment to the Python interpreter. | +| `Bound<'py, T>` | A Python object with a lifetime which binds it to the attachment to the Python interpreter. This provides access to most of PyO3's APIs. | +| `Py` | A Python object not connected to any lifetime of attachment to the Python interpreter. This can be sent to other threads. | +| `PyRef` | A `#[pyclass]` borrowed immutably. | +| `PyRefMut` | A `#[pyclass]` borrowed mutably. | For more detail on accepting `#[pyclass]` values as function arguments, see [the section of this guide on Python Classes](../class.md). -#### Using Rust library types vs Python-native types +### Using Rust library types vs Python-native types -Using Rust library types as function arguments will incur a conversion cost compared to using the Python-native types. Using the Python-native types is almost zero-cost (they just require a type check similar to the Python builtin function `isinstance()`). +Using Rust library types as function arguments will incur a conversion cost compared to using the Python-native types. +Using the Python-native types is almost zero-cost (they just require a type check similar to the Python builtin function `isinstance()`). However, once that conversion cost has been paid, the Rust standard library types offer a number of benefits: + - You can write functionality in native-speed Rust code (free of Python's runtime costs). - You get better interoperability with the rest of the Rust ecosystem. -- You can use `Python::allow_threads` to release the Python GIL and let other Python threads make progress while your Rust code is executing. -- You also benefit from stricter type checking. For example you can specify `Vec`, which will only accept a Python `list` containing integers. The Python-native equivalent, `&PyList`, would accept a Python `list` containing Python objects of any type. +- You can use `Python::detach` to detach from the interpreter and let other Python threads make progress while your Rust code is executing. +- You also benefit from stricter type checking. + For example you can specify `Vec`, which will only accept a Python `list` containing integers. + The Python-native equivalent, `&PyList`, would accept a Python `list` containing Python objects of any type. -For most PyO3 usage the conversion cost is worth paying to get these benefits. As always, if you're not sure it's worth it in your case, benchmark it! +For most PyO3 usage the conversion cost is worth paying to get these benefits. +As always, if you're not sure it's worth it in your case, benchmark it! -### Returning Rust values to Python +## Returning Rust values to Python -When returning values from functions callable from Python, Python-native types (`&PyAny`, `&PyDict` etc.) can be used with zero cost. +When returning values from functions callable from Python, [PyO3's smart pointers](../types.md#pyo3s-smart-pointers) (`Py`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`) can be used with zero cost. -Because these types are references, in some situations the Rust compiler may ask for lifetime annotations. If this is the case, you should use `Py`, `Py` etc. instead - which are also zero-cost. For all of these Python-native types `T`, `Py` can be created from `T` with an `.into()` conversion. +Because `Bound<'py, T>` and `Borrowed<'a, 'py, T>` have lifetime parameters, the Rust compiler may ask for lifetime annotations to be added to your function. +See the [section of the guide dedicated to this](../types.md#function-argument-lifetimes). -If your function is fallible, it should return `PyResult` or `Result` where `E` implements `From for PyErr`. This will raise a `Python` exception if the `Err` variant is returned. +If your function is fallible, it should return `PyResult` or `Result` where `E` implements `From for PyErr`. +This will raise a `Python` exception if the `Err` variant is returned. Finally, the following Rust types are also able to convert to Python as return values: @@ -95,7 +103,8 @@ Finally, the following Rust types are also able to convert to Python as return v | `BTreeMap` | `Dict[K, V]` | | `HashSet` | `Set[T]` | | `BTreeSet` | `Set[T]` | -| `&PyCell` | `T` | +| `Py` | `T` | +| `Bound` | `T` | | `PyRef` | `T` | | `PyRefMut` | `T` | @@ -107,8 +116,14 @@ Finally, the following Rust types are also able to convert to Python as return v [^4]: Requires the `indexmap` optional feature. -[^5]: Requires the `chrono` optional feature. +[^5]: Requires the `chrono` (and maybe `chrono-local`) optional feature(s). [^6]: Requires the `chrono-tz` optional feature. [^7]: Requires the `rust_decimal` optional feature. + +[^8]: Requires the `num-rational` optional feature. + +[^9]: Requires the `bigdecimal` optional feature. + +[^10]: Requires the `ordered-float` optional feature. diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index b46e5c02f4c..d1cacf73591 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -1,8 +1,8 @@ -## Conversion traits +# Conversion traits PyO3 provides some handy traits to convert between Python types and Rust types. -### `.extract()` and the `FromPyObject` trait +## `.extract()` and the `FromPyObject` trait The easiest way to convert a Python object to a Rust value is using `.extract()`. It returns a `PyResult` with a type error if the conversion @@ -12,8 +12,8 @@ fails, so usually you will use something like # use pyo3::prelude::*; # use pyo3::types::PyList; # fn main() -> PyResult<()> { -# Python::with_gil(|py| { -# let list = PyList::new_bound(py, b"foo"); +# Python::attach(|py| { +# let list = PyList::new(py, b"foo")?; let v: Vec = list.extract()?; # assert_eq!(&v, &[102, 111, 111]); # Ok(()) @@ -32,14 +32,13 @@ mutable references, you have to extract the PyO3 reference wrappers [`PyRef`] and [`PyRefMut`]. They work like the reference wrappers of `std::cell::RefCell` and ensure (at runtime) that Rust borrows are allowed. -#### Deriving [`FromPyObject`] +### Deriving [`FromPyObject`] -[`FromPyObject`] can be automatically derived for many kinds of structs and enums -if the member types themselves implement `FromPyObject`. This even includes members -with a generic type `T: FromPyObject`. Derivation for empty enums, enum variants and -structs is not supported. +[`FromPyObject`] can be automatically derived for many kinds of structs and enums if the member types themselves implement `FromPyObject`. +This even includes members with a generic type `T: FromPyObject`. +Derivation for empty enums, enum variants and structs is not supported. -#### Deriving [`FromPyObject`] for structs +### Deriving [`FromPyObject`] for structs The derivation generates code that will attempt to access the attribute `my_string` on the Python object, i.e. `obj.getattr("my_string")`, and call `extract()` on the attribute. @@ -53,14 +52,14 @@ struct RustyStruct { } # # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { +# Python::attach(|py| -> PyResult<()> { # let module = PyModule::from_code( # py, -# "class Foo: +# c"class Foo: # def __init__(self): # self.my_string = 'test'", -# "", -# "", +# c"", +# c"", # )?; # # let class = module.getattr("Foo")?; @@ -85,8 +84,8 @@ struct RustyStruct { # # use pyo3::types::PyDict; # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { -# let dict = PyDict::new_bound(py); +# Python::attach(|py| -> PyResult<()> { +# let dict = PyDict::new(py); # dict.set_item("my_string", "test")?; # # let rustystruct: RustyStruct = dict.extract()?; @@ -110,15 +109,15 @@ struct RustyStruct { } # # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { +# Python::attach(|py| -> PyResult<()> { # let module = PyModule::from_code( # py, -# "class Foo(dict): +# c"class Foo(dict): # def __init__(self): # self.name = 'test' # self['key'] = 'test2'", -# "", -# "", +# c"", +# c"", # )?; # # let class = module.getattr("Foo")?; @@ -132,10 +131,8 @@ struct RustyStruct { # } ``` -This tries to extract `string_attr` from the attribute `name` and `string_in_mapping` -from a mapping with the key `"key"`. The arguments for `attribute` are restricted to -non-empty string literals while `item` can take any valid literal that implements -`ToBorrowedObject`. +This tries to extract `string_attr` from the attribute `name` and `string_in_mapping` from a mapping with the key `"key"`. +The arguments for `attribute` are restricted to non-empty string literals while `item` can take any valid literal that implements `ToBorrowedObject`. You can use `#[pyo3(from_item_all)]` on a struct to extract every field with `get_item` method. In this case, you can't use `#[pyo3(attribute)]` or barely use `#[pyo3(item)]` on any field. @@ -154,8 +151,8 @@ struct RustyStruct { } # # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { -# let py_dict = py.eval_bound("{'foo': 'foo', 'bar': 'bar', 'foobar': 'foobar'}", None, None)?; +# Python::attach(|py| -> PyResult<()> { +# let py_dict = py.eval(c"{'foo': 'foo', 'bar': 'bar', 'foobar': 'foobar'}", None, None)?; # let rustystruct: RustyStruct = py_dict.extract()?; # assert_eq!(rustystruct.foo, "foo"); # assert_eq!(rustystruct.bar, "bar"); @@ -166,11 +163,10 @@ struct RustyStruct { # } ``` -#### Deriving [`FromPyObject`] for tuple structs +### Deriving [`FromPyObject`] for tuple structs -Tuple structs are also supported but do not allow customizing the extraction. The input is -always assumed to be a Python tuple with the same length as the Rust type, the `n`th field -is extracted from the `n`th item in the Python tuple. +Tuple structs are also supported but do not allow customizing the extraction. +The input is always assumed to be a Python tuple with the same length as the Rust type, the `n`th field is extracted from the `n`th item in the Python tuple. ```rust use pyo3::prelude::*; @@ -180,8 +176,8 @@ struct RustyTuple(String, String); # use pyo3::types::PyTuple; # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { -# let tuple = PyTuple::new_bound(py, vec!["test", "test2"]); +# Python::attach(|py| -> PyResult<()> { +# let tuple = PyTuple::new(py, vec!["test", "test2"])?; # # let rustytuple: RustyTuple = tuple.extract()?; # assert_eq!(rustytuple.0, "test"); @@ -192,9 +188,9 @@ struct RustyTuple(String, String); # } ``` -Tuple structs with a single field are treated as wrapper types which are described in the -following section. To override this behaviour and ensure that the input is in fact a tuple, -specify the struct as +Tuple structs with a single field are treated as wrapper types which are described in the following section. +To override this behaviour and ensure that the input is in fact a tuple, specify the struct as + ```rust use pyo3::prelude::*; @@ -203,8 +199,8 @@ struct RustyTuple((String,)); # use pyo3::types::PyTuple; # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { -# let tuple = PyTuple::new_bound(py, vec!["test"]); +# Python::attach(|py| -> PyResult<()> { +# let tuple = PyTuple::new(py, vec!["test"])?; # # let rustytuple: RustyTuple = tuple.extract()?; # assert_eq!((rustytuple.0).0, "test"); @@ -214,12 +210,11 @@ struct RustyTuple((String,)); # } ``` -#### Deriving [`FromPyObject`] for wrapper types +### Deriving [`FromPyObject`] for wrapper types -The `pyo3(transparent)` attribute can be used on structs with exactly one field. This results -in extracting directly from the input object, i.e. `obj.extract()`, rather than trying to access -an item or attribute. This behaviour is enabled per default for newtype structs and tuple-variants -with a single field. +The `pyo3(transparent)` attribute can be used on structs with exactly one field. +This results in extracting directly from the input object, i.e. `obj.extract()`, rather than trying to access an item or attribute. +This behaviour is enabled per default for newtype structs and tuple-variants with a single field. ```rust use pyo3::prelude::*; @@ -235,8 +230,8 @@ struct RustyTransparentStruct { # use pyo3::types::PyString; # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { -# let s = PyString::new_bound(py, "test"); +# Python::attach(|py| -> PyResult<()> { +# let s = PyString::new(py, "test"); # # let tup: RustyTransparentTupleStruct = s.extract()?; # assert_eq!(tup.0, "test"); @@ -249,23 +244,21 @@ struct RustyTransparentStruct { # } ``` -#### Deriving [`FromPyObject`] for enums +### Deriving [`FromPyObject`] for enums -The `FromPyObject` derivation for enums generates code that tries to extract the variants in the -order of the fields. As soon as a variant can be extracted successfully, that variant is returned. +The `FromPyObject` derivation for enums generates code that tries to extract the variants in the order of the fields. +As soon as a variant can be extracted successfully, that variant is returned. This makes it possible to extract Python union types like `str | int`. -The same customizations and restrictions described for struct derivations apply to enum variants, -i.e. a tuple variant assumes that the input is a Python tuple, and a struct variant defaults to -extracting fields as attributes but can be configured in the same manner. The `transparent` -attribute can be applied to single-field-variants. +The same customizations and restrictions described for struct derivations apply to enum variants, i.e. a tuple variant assumes that the input is a Python tuple, and a struct variant defaults to extracting fields as attributes but can be configured in the same manner. +The `transparent` attribute can be applied to single-field-variants. ```rust use pyo3::prelude::*; #[derive(FromPyObject)] # #[derive(Debug)] -enum RustyEnum<'a> { +enum RustyEnum<'py> { Int(usize), // input is a positive int String(String), // input is a string IntTuple(usize, usize), // input is a 2-tuple with positive ints @@ -284,15 +277,15 @@ enum RustyEnum<'a> { b: usize, }, #[pyo3(transparent)] - CatchAll(&'a PyAny), // This extraction never fails + CatchAll(Bound<'py, PyAny>), // This extraction never fails } # # use pyo3::types::{PyBytes, PyString}; # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { +# Python::attach(|py| -> PyResult<()> { # { -# let thing = 42_u8.to_object(py); -# let rust_thing: RustyEnum<'_> = thing.extract(py)?; +# let thing = 42_u8.into_pyobject(py)?; +# let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( # 42, @@ -303,7 +296,7 @@ enum RustyEnum<'a> { # ); # } # { -# let thing = PyString::new_bound(py, "text"); +# let thing = PyString::new(py, "text"); # let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( @@ -315,8 +308,8 @@ enum RustyEnum<'a> { # ); # } # { -# let thing = (32_u8, 73_u8).to_object(py); -# let rust_thing: RustyEnum<'_> = thing.extract(py)?; +# let thing = (32_u8, 73_u8).into_pyobject(py)?; +# let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( # (32, 73), @@ -327,8 +320,8 @@ enum RustyEnum<'a> { # ); # } # { -# let thing = ("foo", 73_u8).to_object(py); -# let rust_thing: RustyEnum<'_> = thing.extract(py)?; +# let thing = ("foo", 73_u8).into_pyobject(py)?; +# let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( # (String::from("foo"), 73), @@ -341,13 +334,13 @@ enum RustyEnum<'a> { # { # let module = PyModule::from_code( # py, -# "class Foo(dict): +# c"class Foo(dict): # def __init__(self): # self.x = 0 # self.y = 1 # self.z = 2", -# "", -# "", +# c"", +# c"", # )?; # # let class = module.getattr("Foo")?; @@ -366,12 +359,12 @@ enum RustyEnum<'a> { # { # let module = PyModule::from_code( # py, -# "class Foo(dict): +# c"class Foo(dict): # def __init__(self): # self.x = 3 # self.y = 4", -# "", -# "", +# c"", +# c"", # )?; # # let class = module.getattr("Foo")?; @@ -388,13 +381,13 @@ enum RustyEnum<'a> { # } # # { -# let thing = PyBytes::new_bound(py, b"text"); +# let thing = PyBytes::new(py, b"text"); # let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( # b"text", # match rust_thing { -# RustyEnum::CatchAll(i) => i.downcast::()?.as_bytes(), +# RustyEnum::CatchAll(ref i) => i.cast::()?.as_bytes(), # other => unreachable!("Error extracting: {:?}", other), # } # ); @@ -404,10 +397,8 @@ enum RustyEnum<'a> { # } ``` -If none of the enum variants match, a `PyTypeError` containing the names of the -tested variants is returned. The names reported in the error message can be customized -through the `#[pyo3(annotation = "name")]` attribute, e.g. to use conventional Python type -names: +If none of the enum variants match, a `PyTypeError` containing the names of the tested variants is returned. +The names reported in the error message can be customized through the `#[pyo3(annotation = "name")]` attribute, e.g. to use conventional Python type names: ```rust use pyo3::prelude::*; @@ -422,10 +413,10 @@ enum RustyEnum { } # # fn main() -> PyResult<()> { -# Python::with_gil(|py| -> PyResult<()> { +# Python::attach(|py| -> PyResult<()> { # { -# let thing = 42_u8.to_object(py); -# let rust_thing: RustyEnum = thing.extract(py)?; +# let thing = 42_u8.into_pyobject(py)?; +# let rust_thing: RustyEnum = thing.extract()?; # # assert_eq!( # 42, @@ -437,8 +428,8 @@ enum RustyEnum { # } # # { -# let thing = "foo".to_object(py); -# let rust_thing: RustyEnum = thing.extract(py)?; +# let thing = "foo".into_pyobject(py)?; +# let rust_thing: RustyEnum = thing.extract()?; # # assert_eq!( # "foo", @@ -450,8 +441,8 @@ enum RustyEnum { # } # # { -# let thing = b"foo".to_object(py); -# let error = thing.extract::(py).unwrap_err(); +# let thing = b"foo".into_pyobject(py)?; +# let error = thing.extract::().unwrap_err(); # assert!(error.is_instance_of::(py)); # } # @@ -461,64 +452,324 @@ enum RustyEnum { ``` If the input is neither a string nor an integer, the error message will be: -`"'' cannot be converted to 'str | int'"`. +`"'' cannot be cast as 'str | int'"`. + +### `#[derive(FromPyObject)]` Container Attributes -#### `#[derive(FromPyObject)]` Container Attributes - `pyo3(transparent)` - - extract the field directly from the object as `obj.extract()` instead of `get_item()` or + - extract the field directly from the object as `obj.extract()` instead of `get_item()` or `getattr()` - - Newtype structs and tuple-variants are treated as transparent per default. - - only supported for single-field structs and enum variants + - Newtype structs and tuple-variants are treated as transparent per default. + - only supported for single-field structs and enum variants - `pyo3(annotation = "name")` - - changes the name of the failed variant in the generated error message in case of failure. - - e.g. `pyo3("int")` reports the variant's type as `int`. - - only supported for enum variants + - changes the name of the failed variant in the generated error message in case of failure. + - e.g. `pyo3("int")` reports the variant's type as `int`. + - only supported for enum variants +- `pyo3(rename_all = "...")` + - renames all attributes/item keys according to the specified renaming rule + - Possible values are: "camelCase", "kebab-case", "lowercase", "PascalCase", "SCREAMING-KEBAB-CASE", "SCREAMING_SNAKE_CASE", "snake_case", "UPPERCASE". + - fields with an explicit renaming via `attribute(...)`/`item(...)` are not affected + +### `#[derive(FromPyObject)]` Field Attributes -#### `#[derive(FromPyObject)]` Field Attributes - `pyo3(attribute)`, `pyo3(attribute("name"))` - - retrieve the field from an attribute, possibly with a custom name specified as an argument - - argument must be a string-literal. + - retrieve the field from an attribute, possibly with a custom name specified as an argument + - argument must be a string-literal. - `pyo3(item)`, `pyo3(item("key"))` - - retrieve the field from a mapping, possibly with the custom key specified as an argument. - - can be any literal that implements `ToBorrowedObject` -- `pyo3(from_py_with = "...")` - - apply a custom function to convert the field from Python the desired Rust type. - - the argument must be the name of the function as a string. - - the function signature must be `fn(&PyAny) -> PyResult` where `T` is the Rust type of the argument. + - retrieve the field from a mapping, possibly with the custom key specified as an argument. + - can be any literal that implements `ToBorrowedObject` +- `pyo3(from_py_with = ...)` + - apply a custom function to convert the field from Python the desired Rust type. + - the argument must be the path to the function. + - the function signature must be `fn(&Bound) -> PyResult` where `T` is the Rust type of the argument. +- `pyo3(default)`, `pyo3(default = ...)` + - if the argument is set, uses the given default value. + - in this case, the argument must be a Rust expression returning a value of the desired Rust type. + - if the argument is not set, [`Default::default`](https://doc.rust-lang.org/std/default/trait.Default.html#tymethod.default) is used. + - note that the default value is only used if the field is not set. + If the field is set and the conversion function from Python to Rust fails, an exception is raised and the default value is not used. + - this attribute is only supported on named fields. + +For example, the code below applies the given conversion function on the `"value"` dict item to compute its length or fall back to the type default value (0): + +```rust +use pyo3::prelude::*; + +#[derive(FromPyObject)] +struct RustyStruct { + #[pyo3(item("value"), default, from_py_with = Bound::<'_, PyAny>::len)] + len: usize, + #[pyo3(item)] + other: usize, +} +# +# use pyo3::types::PyDict; +# fn main() -> PyResult<()> { +# Python::attach(|py| -> PyResult<()> { +# // Filled case +# let dict = PyDict::new(py); +# dict.set_item("value", (1,)).unwrap(); +# dict.set_item("other", 1).unwrap(); +# let result = dict.extract::()?; +# assert_eq!(result.len, 1); +# assert_eq!(result.other, 1); +# +# // Empty case +# let dict = PyDict::new(py); +# dict.set_item("other", 1).unwrap(); +# let result = dict.extract::()?; +# assert_eq!(result.len, 0); +# assert_eq!(result.other, 1); +# Ok(()) +# }) +# } +``` -### `IntoPy` +### ⚠ Phase-Out of `FromPyObject` blanket implementation for cloneable PyClasses ⚠ -This trait defines the to-python conversion for a Rust type. It is usually implemented as -`IntoPy`, which is the trait needed for returning a value from `#[pyfunction]` and -`#[pymethods]`. +Historically PyO3 has provided a blanket implementation for `#[pyclass]` types that also implement `Clone`, to allow extraction of such types by value. +Over time this has turned out problematic for a few reasons, the major one being the prevention of custom conversions by downstream crates if their type is `Clone`. +Over the next few releases the blanket implementation is gradually phased out, and eventually replaced by an opt-in option. +As a first step of this migration a new `skip_from_py_object` option for `#[pyclass]` was introduced, to opt-out of the blanket implementation and allow downstream users to provide their own implementation: +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; + +#[pyclass(skip_from_py_object)] // opt-out of the PyO3 FromPyObject blanket +#[derive(Clone)] +struct Number(i32); + +impl<'py> FromPyObject<'_, 'py> for Number { + type Error = PyErr; + + fn extract(obj: pyo3::Borrowed<'_, 'py, pyo3::PyAny>) -> Result { + if let Ok(obj) = obj.cast::() { // first try extraction via class object + Ok(obj.borrow().clone()) + } else { + obj.extract::().map(Self) // otherwise try integer directly + } + } +} +``` + +As a second step the `from_py_object` option was introduced. +This option also opts-out of the blanket implementation and instead generates a custom `FromPyObject` implementation for the pyclass which is functionally equivalent to the blanket. + +## `IntoPyObject` + +The [`IntoPyObject`] trait defines the to-python conversion for a Rust type. All types in PyO3 implement this trait, as does a `#[pyclass]` which doesn't use `extends`. +This trait defines a single method, `into_pyobject()`, which returns a [`Result`] with `Ok` and `Err` types depending on the input value. +For convenience, there is a companion [`IntoPyObjectExt`] trait which adds methods such as `into_py_any()` which converts the `Ok` and `Err` types to commonly used types (in the case of `into_py_any()`, `Py` and `PyErr` respectively). + Occasionally you may choose to implement this for custom types which are mapped to Python types -_without_ having a unique python type. +*without* having a unique python type. -```rust -use pyo3::prelude::*; +### derive macro + +`IntoPyObject` can be implemented using our derive macro. +Both `struct`s and `enum`s are supported. -struct MyPyObjectWrapper(PyObject); +`struct`s will turn into a `PyDict` using the field names as keys, tuple `struct`s will turn convert +into `PyTuple` with the fields in declaration order. + +```rust,no_run +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use std::collections::HashMap; +# use std::hash::Hash; + +// structs convert into `PyDict` with field names as keys +#[derive(IntoPyObject)] +struct Struct { + count: usize, + obj: Py, +} + +// tuple structs convert into `PyTuple` +// lifetimes and generics are supported, the impl will be bounded by +// `K: IntoPyObject, V: IntoPyObject` +#[derive(IntoPyObject)] +struct Tuple<'a, K: Hash + Eq, V>(&'a str, HashMap); +``` -impl IntoPy for MyPyObjectWrapper { - fn into_py(self, py: Python<'_>) -> PyObject { - self.0 +For structs with a single field (newtype pattern) the `#[pyo3(transparent)]` option can be used to +forward the implementation to the inner type. + +```rust,no_run +# #![allow(dead_code)] +# use pyo3::prelude::*; + +// newtype tuple structs are implicitly `transparent` +#[derive(IntoPyObject)] +struct TransparentTuple(Py); + +#[derive(IntoPyObject)] +#[pyo3(transparent)] +struct TransparentStruct<'py> { + inner: Bound<'py, PyAny>, // `'py` lifetime will be used as the Python lifetime +} +``` + +For `enum`s each variant is converted according to the rules for `struct`s above. + +```rust,no_run +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use std::collections::HashMap; +# use std::hash::Hash; + +#[derive(IntoPyObject)] +enum Enum<'a, 'py, K: Hash + Eq, V> { // enums are supported and convert using the same + TransparentTuple(Py), // rules on the variants as the structs above + #[pyo3(transparent)] + TransparentStruct { inner: Bound<'py, PyAny> }, + Tuple(&'a str, HashMap), + Struct { count: usize, obj: Py } +} +``` + +Additionally `IntoPyObject` can be derived for a reference to a struct or enum using the `IntoPyObjectRef` derive macro. +All the same rules from above apply as well. + +#### `#[derive(IntoPyObject)]`/`#[derive(IntoPyObjectRef)]` Field Attributes + +- `pyo3(into_py_with = ...)` + - apply a custom function to convert the field from Rust into Python. + - the argument must be the function identifier + - the function signature must be `fn(Cow<'_, T>, Python<'py>) -> PyResult>` where `T` is the Rust type of the argument. + - `#[derive(IntoPyObject)]` will invoke the function with `Cow::Owned` + - `#[derive(IntoPyObjectRef)]` will invoke the function with `Cow::Borrowed` + + ```rust,no_run + # use pyo3::prelude::*; + # use pyo3::IntoPyObjectExt; + # use std::borrow::Cow; + #[derive(Clone)] + struct NotIntoPy(usize); + + #[derive(IntoPyObject, IntoPyObjectRef)] + struct MyStruct { + #[pyo3(into_py_with = convert)] + not_into_py: NotIntoPy, + } + + /// Convert `NotIntoPy` into Python + fn convert<'py>(not_into_py: Cow<'_, NotIntoPy>, py: Python<'py>) -> PyResult> { + not_into_py.0.into_bound_py_any(py) + } + ``` + +### manual implementation + +If the derive macro is not suitable for your use case, `IntoPyObject` can be implemented manually as +demonstrated below. + +```rust,no_run +# use pyo3::prelude::*; +# #[allow(dead_code)] +struct MyPyObjectWrapper(Py); + +impl<'py> IntoPyObject<'py> for MyPyObjectWrapper { + type Target = PyAny; // the Python type + type Output = Bound<'py, Self::Target>; // in most cases this will be `Bound` + type Error = std::convert::Infallible; // the conversion error type, has to be convertible to `PyErr` + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.0.into_bound(py)) + } +} + +// equivalent to former `ToPyObject` implementations +impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper { + type Target = PyAny; + type Output = Borrowed<'a, 'py, Self::Target>; // `Borrowed` can be used to optimized reference counting + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.0.bind_borrowed(py)) } } ``` -### The `ToPyObject` trait +### `BoundObject` for conversions that may be `Bound` or `Borrowed` + +`IntoPyObject::into_py_object` returns either `Bound` or `Borrowed` depending on the implementation for a concrete type. +For example, the `IntoPyObject` implementation for `u32` produces a `Bound<'py, PyInt>` and the `bool` implementation produces a `Borrowed<'py, 'py, PyBool>`: + +```rust,no_run +use pyo3::prelude::*; +use pyo3::IntoPyObject; +use pyo3::types::{PyBool, PyInt}; + +let ints: Vec = vec![1, 2, 3, 4]; +let bools = vec![true, false, false, true]; + +Python::attach(|py| { + let ints_as_pyint: Vec> = ints + .iter() + .map(|x| Ok(x.into_pyobject(py)?)) + .collect::>() + .unwrap(); + + let bools_as_pybool: Vec> = bools + .iter() + .map(|x| Ok(x.into_pyobject(py)?)) + .collect::>() + .unwrap(); +}); +``` + +In this example if we wanted to combine `ints_as_pyints` and `bools_as_pybool` into a single `Vec>` to return from the `Python::attach` closure, we would have to manually convert the concrete types for the smart pointers and the python types. + +Instead, we can write a function that generically converts vectors of either integers or bools into a vector of `Py` using the [`BoundObject`] trait: + +```rust,no_run +# use pyo3::prelude::*; +# use pyo3::BoundObject; +# use pyo3::IntoPyObject; + +# let bools = vec![true, false, false, true]; +# let ints = vec![1, 2, 3, 4]; + +fn convert_to_vec_of_pyobj<'py, T>(py: Python<'py>, the_vec: Vec) -> PyResult>> +where + T: IntoPyObject<'py> + Copy +{ + the_vec.iter() + .map(|x| { + Ok( + // Note: the below is equivalent to `x.into_py_any()` + // from the `IntoPyObjectExt` trait + x.into_pyobject(py) + .map_err(Into::into)? + .into_any() + .unbind() + ) + }) + .collect() +} -[`ToPyObject`] is a conversion trait that allows various objects to be -converted into [`PyObject`]. `IntoPy` serves the -same purpose, except that it consumes `self`. +let vec_of_pyobjs: Vec> = Python::attach(|py| { + let mut bools_as_pyany = convert_to_vec_of_pyobj(py, bools).unwrap(); + let mut ints_as_pyany = convert_to_vec_of_pyobj(py, ints).unwrap(); + let mut result: Vec> = vec![]; + result.append(&mut bools_as_pyany); + result.append(&mut ints_as_pyany); + result +}); +``` + +In the example above we used `BoundObject::into_any` and `BoundObject::unbind` to manipulate the python types and smart pointers into the result type we wanted to produce from the function. -[`IntoPy`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPy.html [`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html -[`ToPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.ToPyObject.html -[`PyObject`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyObject.html +[`IntoPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPyObject.html +[`IntoPyObjectExt`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPyObjectExt.html [`PyRef`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html [`PyRefMut`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRefMut.html +[`BoundObject`]: {{#PYO3_DOCS_URL}}/pyo3/instance/trait.BoundObject.html + +[`Result`]: https://doc.rust-lang.org/stable/std/result/enum.Result.html diff --git a/guide/src/debugging.md b/guide/src/debugging.md index 00c22631c3b..2b6b8d062f9 100644 --- a/guide/src/debugging.md +++ b/guide/src/debugging.md @@ -2,7 +2,8 @@ ## Macros -PyO3's attributes (`#[pyclass]`, `#[pymodule]`, etc.) are [procedural macros](https://doc.rust-lang.org/reference/procedural-macros.html), which means that they rewrite the source of the annotated item. You can view the generated source with the following command, which also expands a few other things: +PyO3's attributes (`#[pyclass]`, `#[pymodule]`, etc.) are [procedural macros](https://doc.rust-lang.org/reference/procedural-macros.html), which means that they rewrite the source of the annotated item. +You can view the generated source with the following command, which also expands a few other things: ```bash cargo rustc --profile=check -- -Z unstable-options --pretty=expanded > expanded.rs; rustfmt expanded.rs @@ -16,15 +17,18 @@ You can also debug classic `!`-macros by adding `-Z trace-macros`: cargo rustc --profile=check -- -Z unstable-options --pretty=expanded -Z trace-macros > expanded.rs; rustfmt expanded.rs ``` -Note that those commands require using the nightly build of rust and may occasionally have bugs. See [cargo expand](https://github.com/dtolnay/cargo-expand) for a more elaborate and stable version of those commands. +Note that those commands require using the nightly build of rust and may occasionally have bugs. +See [cargo expand](https://github.com/dtolnay/cargo-expand) for a more elaborate and stable version of those commands. ## Running with Valgrind Valgrind is a tool to detect memory management bugs such as memory leaks. -You first need to install a debug build of Python, otherwise Valgrind won't produce usable results. In Ubuntu there's e.g. a `python3-dbg` package. +You first need to install a debug build of Python, otherwise Valgrind won't produce usable results. +In Ubuntu there's e.g. a `python3-dbg` package. -Activate an environment with the debug interpreter and recompile. If you're on Linux, use `ldd` with the name of your binary and check that you're linking e.g. `libpython3.7d.so.1.0` instead of `libpython3.7.so.1.0`. +Activate an environment with the debug interpreter and recompile. +If you're on Linux, use `ldd` with the name of your binary and check that you're linking e.g. `libpython3.7d.so.1.0` instead of `libpython3.7.so.1.0`. [Download the suppressions file for CPython](https://raw.githubusercontent.com/python/cpython/master/Misc/valgrind-python.supp). @@ -32,16 +36,360 @@ Run Valgrind with `valgrind --suppressions=valgrind-python.supp ./my-command --w ## Getting a stacktrace -The best start to investigate a crash such as an segmentation fault is a backtrace. You can set `RUST_BACKTRACE=1` as an environment variable to get the stack trace on a `panic!`. Alternatively you can use a debugger such as `gdb` to explore the issue. Rust provides a wrapper, `rust-gdb`, which has pretty-printers for inspecting Rust variables. Since PyO3 uses `cdylib` for Python shared objects, it does not receive the pretty-print debug hooks in `rust-gdb` ([rust-lang/rust#96365](https://github.com/rust-lang/rust/issues/96365)). The mentioned issue contains a workaround for enabling pretty-printers in this case. +The best start to investigate a crash such as an segmentation fault is a backtrace. +You can set `RUST_BACKTRACE=1` as an environment variable to get the stack trace on a `panic!`. +Alternatively you can use a debugger such as `gdb` to explore the issue. +Rust provides a wrapper, `rust-gdb`, which has pretty-printers for inspecting Rust variables. +Since PyO3 uses `cdylib` for Python shared objects, it does not receive the pretty-print debug hooks in `rust-gdb` ([rust-lang/rust#96365](https://github.com/rust-lang/rust/issues/96365)). +The mentioned issue contains a workaround for enabling pretty-printers in this case. - * Link against a debug build of python as described in the previous chapter - * Run `rust-gdb ` - * Set a breakpoint (`b`) on `rust_panic` if you are investigating a `panic!` - * Enter `r` to run - * After the crash occurred, enter `bt` or `bt full` to print the stacktrace +- Link against a debug build of python as described in the previous chapter +- Run `rust-gdb ` +- Set a breakpoint (`b`) on `rust_panic` if you are investigating a `panic!` +- Enter `r` to run +- After the crash occurred, enter `bt` or `bt full` to print the stacktrace Often it is helpful to run a small piece of Python code to exercise a section of Rust. ```console rust-gdb --args python -c "import my_package; my_package.sum_to_string(1, 2)" ``` + +## Setting breakpoints in your Rust code + +One of the preferred ways by developers to debug their code is by setting breakpoints. +This can be achieved in PyO3 by using a debugger like `rust-gdb` or `rust-lldb` with your Python interpreter. + +For more information about how to use both `lldb` and `gdb` you can read the [gdb to lldb command map](https://lldb.llvm.org/use/map.html) from the lldb documentation. + +### Common setup + +1. Compile your extension with debug symbols: + + ```bash + # Debug is the default for maturin, but you can explicitly ensure debug symbols with: + RUSTFLAGS="-g" maturin develop + + # For setuptools-rust users: + pip install -e . + ``` + + > **Note**: When using debuggers, make sure that `python` resolves to an actual Python binary or symlink and not a shim script. + Some tools like pyenv use shim scripts which can interfere with debugging. + +### Debugger specific setup + +Depending on your OS and your preferences you can use two different debuggers, `rust-gdb` or `rust-lldb`. + +{{#tabs }} +{{#tab name="Using rust-gdb" }} + +1. Launch rust-gdb with the Python interpreter: + + ```bash + rust-gdb --args python + ``` + +2. Once in gdb, set a breakpoint in your Rust code: + + ```bash + (gdb) break your_module.rs:42 + ``` + +3. Run your Python script that imports and uses your Rust extension: + + ```bash + # Option 1: Run an inline Python command + (gdb) run -c "import your_module; your_module.your_function()" + + # Option 2: Run a Python script + (gdb) run your_script.py + + # Option 3: Run pytest tests + (gdb) run -m pytest tests/test_something.py::TestName + ``` + +{{#endtab }} +{{#tab name="Using rust-lldb (for macOS users)" }} + +1. Start rust-lldb with Python: + + ```bash + rust-lldb -- python + ``` + +2. Set breakpoints in your Rust code: + + ```bash + (lldb) breakpoint set --file your_module.rs --line 42 + ``` + +3. Run your Python script: + + ```bash + # Option 1: Run an inline Python command + (lldb) run -c "import your_module; your_module.your_function()" + + # Option 2: Run a Python script + (lldb) run your_script.py + + # Option 3: Run pytest tests + (lldb) run -m pytest tests/test_something.py::TestName + ``` + +{{#endtab }} +{{#endtabs }} + +### Using VS Code + +VS Code with the Rust and Python extensions provides an integrated debugging experience: + +1. First, install the necessary VS Code extensions: + + - [Rust Analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) + - [CodeLLDB](https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb) + - [Python](https://marketplace.visualstudio.com/items?itemName=ms-python.python) + +2. Create a `.vscode/launch.json` file with a configuration that uses the LLDB Debug Launcher: + + ```json + { + "version": "0.2.0", + "configurations": [ + { + "name": "Debug PyO3", + "type": "lldb", + "request": "attach", + "program": "${workspaceFolder}/.venv/bin/python", + "pid": "${command:pickProcess}", + "sourceLanguages": [ + "rust" + ] + }, + { + "name": "Launch Python with PyO3", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/.venv/bin/python", + "args": ["${file}"], + "cwd": "${workspaceFolder}", + "sourceLanguages": ["rust"] + }, + { + "name": "Debug PyO3 with Args", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/.venv/bin/python", + "args": ["path/to/your/script.py", "arg1", "arg2"], + "cwd": "${workspaceFolder}", + "sourceLanguages": ["rust"] + }, + { + "name": "Debug PyO3 Tests", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/.venv/bin/python", + "args": ["-m", "pytest", "tests/your_test.py::test_function", "-v"], + "cwd": "${workspaceFolder}", + "sourceLanguages": ["rust"] + } + ] + } + ``` + + This configuration supports multiple debugging scenarios: + + - Attaching to a running Python process + - Launching the currently open Python file + - Running a specific script with command-line arguments + - Running pytest tests + + + +3. Set breakpoints in your Rust code by clicking in the gutter next to line numbers. + +4. Start debugging: + - For attaching to a running Python process: First start the process, then select the "Debug PyO3" configuration and click Start Debugging (F5). + You'll be prompted to select the Python process to attach to. + - For launching a Python script: Open your Python script, select the "Launch Python with PyO3" configuration and click Start Debugging (F5). + - For running with arguments: Select "Debug PyO3 with Args" (remember to edit the configuration with your actual script path and arguments). + - For running tests: Select "Debug PyO3 Tests" (edit the test path as needed). + +5. When debugging PyO3 code: + - You can inspect Rust variables and data structures + - Use the debug console to evaluate expressions + - Step through Rust code line by line using the step controls + - Set conditional breakpoints for more complex debugging scenarios + + + +### Advanced Debugging Configurations + +For advanced debugging scenarios, you might want to add environment variables or enable specific Rust debug flags: + +```json +{ + "name": "Debug PyO3 with Environment", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/.venv/bin/python", + "args": ["${file}"], + "env": { + "RUST_BACKTRACE": "1", + "PYTHONPATH": "${workspaceFolder}" + }, + "sourceLanguages": ["rust"] +} +``` + +### Debugging from Jupyter Notebooks + +For Jupyter Notebooks run from VS Code, you can use the following helper functions to automate the launch configuration: + +```python +from pathlib import Path +import os +import json +import sys + + +def update_launch_json(vscode_config_file_path=None): + """Update VSCode launch.json with the correct Jupyter kernel PID. + + Args: + vscode_config_file_path (str, optional): Path to the .vscode/launch.json file. + If not provided, will use the current working directory. + """ + pid = get_jupyter_kernel_pid() + if not pid: + print("Could not determine Jupyter kernel PID.") + return + + # Determine launch.json path + if vscode_config_file_path: + launch_json_path = vscode_config_file_path + else: + launch_json_path = os.path.join(Path(os.getcwd()), ".vscode", "launch.json") + + # Get Python interpreter path + python_path = sys.executable + + # Default debugger config + debug_config = { + "version": "0.2.0", + "configurations": [ + { + "name": "Debug PyO3 (Jupyter)", + "type": "lldb", + "request": "attach", + "program": python_path, + "pid": pid, + "sourceLanguages": ["rust"], + }, + { + "name": "Launch Python with PyO3", + "type": "lldb", + "request": "launch", + "program": python_path, + "args": ["${file}"], + "cwd": "${workspaceFolder}", + "sourceLanguages": ["rust"] + } + ], + } + + # Create .vscode directory if it doesn't exist + try: + os.makedirs(os.path.dirname(launch_json_path), exist_ok=True) + + # If launch.json already exists, try to update it instead of overwriting + if os.path.exists(launch_json_path): + try: + with open(launch_json_path, "r") as f: + existing_config = json.load(f) + + # Check if our configuration already exists + config_exists = False + for config in existing_config.get("configurations", []): + if config.get("name") == "Debug PyO3 (Jupyter)": + config["pid"] = pid + config["program"] = python_path + config_exists = True + + if not config_exists: + existing_config.setdefault("configurations", []).append(debug_config["configurations"][0]) + + debug_config = existing_config + except Exception: + # If reading fails, we'll just overwrite with our new configuration + pass + + with open(launch_json_path, "w") as f: + json.dump(debug_config, f, indent=4) + print(f"Updated launch.json with PID: {pid} at {launch_json_path}") + except Exception as e: + print(f"Error updating launch.json: {e}") + + +def get_jupyter_kernel_pid(): + """Find the process ID (PID) of the running Jupyter kernel. + + Returns: + int: The process ID of the Jupyter kernel, or None if not found. + """ + # Check if we're running in a Jupyter environment + if 'ipykernel' in sys.modules: + pid = os.getpid() + print(f"Jupyter kernel PID: {pid}") + return pid + else: + print("Not running in a Jupyter environment.") + return None +``` + +To use these functions: + +1. Run the cell containing these functions in your Jupyter notebook +2. Run `update_launch_json()` in a cell +3. In VS Code, select the "Debug PyO3 (Jupyter)" configuration and start debugging + +## Thread Safety and Compiler Sanitizers + +PyO3 attempts to match the Rust language-level guarantees for thread safety, but that does not preclude other code outside of the control of PyO3 or buggy code managed by a PyO3 extension from creating a thread safety issue. +Analyzing whether or not a piece of Rust code that uses the CPython C API is thread safe can be quite complicated, since many Python operations can lead to arbitrary Python code execution. +Automated ways to discover thread safety issues can often be more fruitful than code analysis. + +[ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html) is a thread safety checking runtime that can be used to detect data races triggered by thread safety bugs or incorrect use of thread-unsafe data structures. +While it can only detect data races triggered by code at runtime, if it does detect something the reports often point to exactly where the problem is happening. + +To use `ThreadSanitizer` with a library that depends on PyO3, you will need to +install a nightly Rust toolchain, along with the `rust-src` component, since you +will need to compile the Rust standard library: + +```bash +rustup install nightly +rustup override set nightly +rustup component add rust-src +``` + +You will also need a version of CPython compiled using LLVM/Clang with the same major version of LLVM as is currently used to compile nightly Rust. +As of March 2025, Rust nightly uses LLVM 20. + +The [cpython_sanity docker images](https://github.com/nascheme/cpython_sanity) +contain a development environment with a pre-compiled version of CPython 3.13 or +3.14 as well as optionally NumPy and SciPy, all compiled using LLVM 20 and +ThreadSanitizer. + +After activating a nightly Rust toolchain, you can build your project using +`ThreadSanitizer` with the following command: + +```bash +RUSTFLAGS="-Zsanitizer=thread" maturin develop -Zbuild-std --target x86_64-unknown-linux-gnu +``` + +If you are not running on an x86_64 Linux machine, you should replace `x86_64-unknown-linux-gnu` with the [target triple](https://doc.rust-lang.org/rustc/platform-support.html#tier-1-with-host-tools) that is appropriate for your system. +You can also replace `maturin develop` with `cargo test` to run `cargo` tests. +Note that `cargo` runs tests in a thread pool, so `cargo` tests can be a good way to find thread safety issues. + +You can also replace `-Zsanitizer=thread` with `-Zsanitizer=address` or any of the other sanitizers that are [supported by Rust](https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html). +Note that you'll need to build CPython from source with the appropriate [configure script flags](https://docs.python.org/3/using/configure.html#cmdoption-with-address-sanitizer) to use the same sanitizer environment as you want to use for your Rust code. diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index f537ab90df1..2e421428889 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -2,557 +2,13 @@ *`async`/`await` support is currently being integrated in PyO3. See the [dedicated documentation](../async-await.md)* -If you are working with a Python library that makes use of async functions or wish to provide -Python bindings for an async Rust library, [`pyo3-asyncio`](https://github.com/awestlake87/pyo3-asyncio) -likely has the tools you need. It provides conversions between async functions in both Python and -Rust and was designed with first-class support for popular Rust runtimes such as -[`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). In addition, all async Python -code runs on the default `asyncio` event loop, so `pyo3-asyncio` should work just fine with existing -Python libraries. - -In the following sections, we'll give a general overview of `pyo3-asyncio` explaining how to call -async Python functions with PyO3, how to call async Rust functions from Python, and how to configure -your codebase to manage the runtimes of both. - -## Quickstart - -Here are some examples to get you started right away! A more detailed breakdown -of the concepts in these examples can be found in the following sections. - -### Rust Applications -Here we initialize the runtime, import Python's `asyncio` library and run the given future to completion using Python's default `EventLoop` and `async-std`. Inside the future, we convert `asyncio` sleep into a Rust future and await it. - - -```toml -# Cargo.toml dependencies -[dependencies] -pyo3 = { version = "0.14" } -pyo3-asyncio = { version = "0.14", features = ["attributes", "async-std-runtime"] } -async-std = "1.9" -``` - -```rust -//! main.rs - -use pyo3::prelude::*; - -#[pyo3_asyncio::async_std::main] -async fn main() -> PyResult<()> { - let fut = Python::with_gil(|py| { - let asyncio = py.import("asyncio")?; - // convert asyncio.sleep into a Rust Future - pyo3_asyncio::async_std::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) - })?; - - fut.await?; - - Ok(()) -} -``` - -The same application can be written to use `tokio` instead using the `#[pyo3_asyncio::tokio::main]` -attribute. - -```toml -# Cargo.toml dependencies -[dependencies] -pyo3 = { version = "0.14" } -pyo3-asyncio = { version = "0.14", features = ["attributes", "tokio-runtime"] } -tokio = "1.4" -``` - -```rust -//! main.rs - -use pyo3::prelude::*; - -#[pyo3_asyncio::tokio::main] -async fn main() -> PyResult<()> { - let fut = Python::with_gil(|py| { - let asyncio = py.import("asyncio")?; - // convert asyncio.sleep into a Rust Future - pyo3_asyncio::tokio::into_future(asyncio.call_method1("sleep", (1.into_py(py),))?) - })?; - - fut.await?; - - Ok(()) -} -``` - -More details on the usage of this library can be found in the [API docs](https://awestlake87.github.io/pyo3-asyncio/master/doc) and the primer below. - -### PyO3 Native Rust Modules - -PyO3 Asyncio can also be used to write native modules with async functions. - -Add the `[lib]` section to `Cargo.toml` to make your library a `cdylib` that Python can import. -```toml -[lib] -name = "my_async_module" -crate-type = ["cdylib"] -``` - -Make your project depend on `pyo3` with the `extension-module` feature enabled and select your -`pyo3-asyncio` runtime: - -For `async-std`: -```toml -[dependencies] -pyo3 = { version = "0.14", features = ["extension-module"] } -pyo3-asyncio = { version = "0.14", features = ["async-std-runtime"] } -async-std = "1.9" -``` - -For `tokio`: -```toml -[dependencies] -pyo3 = { version = "0.14", features = ["extension-module"] } -pyo3-asyncio = { version = "0.14", features = ["tokio-runtime"] } -tokio = "1.4" -``` - -Export an async function that makes use of `async-std`: - -```rust -//! lib.rs - -use pyo3::{prelude::*, wrap_pyfunction}; - -#[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { - pyo3_asyncio::async_std::future_into_py(py, async { - async_std::task::sleep(std::time::Duration::from_secs(1)).await; - Ok(Python::with_gil(|py| py.None())) - }) -} - -#[pymodule] -fn my_async_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; - - Ok(()) -} -``` - -If you want to use `tokio` instead, here's what your module should look like: - -```rust -//! lib.rs - -use pyo3::{prelude::*, wrap_pyfunction}; - -#[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { - pyo3_asyncio::tokio::future_into_py(py, async { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - Ok(Python::with_gil(|py| py.None())) - }) -} - -#[pymodule] -fn my_async_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; - Ok(()) -} -``` - -You can build your module with maturin (see the [Using Rust in Python](https://pyo3.rs/main/#using-rust-from-python) section in the PyO3 guide for setup instructions). After that you should be able to run the Python REPL to try it out. - -```bash -maturin develop && python3 -🔗 Found pyo3 bindings -🐍 Found CPython 3.8 at python3 - Finished dev [unoptimized + debuginfo] target(s) in 0.04s -Python 3.8.5 (default, Jan 27 2021, 15:41:15) -[GCC 9.3.0] on linux -Type "help", "copyright", "credits" or "license" for more information. ->>> import asyncio ->>> ->>> from my_async_module import rust_sleep ->>> ->>> async def main(): ->>> await rust_sleep() ->>> ->>> # should sleep for 1s ->>> asyncio.run(main()) ->>> -``` - -## Awaiting an Async Python Function in Rust - -Let's take a look at a dead simple async Python function: - -```python -# Sleep for 1 second -async def py_sleep(): - await asyncio.sleep(1) -``` - -**Async functions in Python are simply functions that return a `coroutine` object**. For our purposes, -we really don't need to know much about these `coroutine` objects. The key factor here is that calling -an `async` function is _just like calling a regular function_, the only difference is that we have -to do something special with the object that it returns. - -Normally in Python, that something special is the `await` keyword, but in order to await this -coroutine in Rust, we first need to convert it into Rust's version of a `coroutine`: a `Future`. -That's where `pyo3-asyncio` comes in. -[`pyo3_asyncio::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.into_future.html) -performs this conversion for us. - -The following example uses `into_future` to call the `py_sleep` function shown above and then await the -coroutine object returned from the call: - -```rust -use pyo3::prelude::*; - -#[pyo3_asyncio::tokio::main] -async fn main() -> PyResult<()> { - let future = Python::with_gil(|py| -> PyResult<_> { - // import the module containing the py_sleep function - let example = py.import("example")?; - - // calling the py_sleep method like a normal function - // returns a coroutine - let coroutine = example.call_method0("py_sleep")?; - - // convert the coroutine into a Rust future using the - // tokio runtime - pyo3_asyncio::tokio::into_future(coroutine) - })?; - - // await the future - future.await?; - - Ok(()) -} -``` - -Alternatively, the below example shows how to write a `#[pyfunction]` which uses `into_future` to receive and await -a coroutine argument: - -```rust -#[pyfunction] -fn await_coro(coro: &PyAny) -> PyResult<()> { - // convert the coroutine into a Rust future using the - // async_std runtime - let f = pyo3_asyncio::async_std::into_future(coro)?; - - pyo3_asyncio::async_std::run_until_complete(coro.py(), async move { - // await the future - f.await?; - Ok(()) - }) -} -``` - -This could be called from Python as: - -```python -import asyncio - -async def py_sleep(): - asyncio.sleep(1) - -await_coro(py_sleep()) -``` - -If for you wanted to pass a callable function to the `#[pyfunction]` instead, (i.e. the last line becomes `await_coro(py_sleep))`, then the above example needs to be tweaked to first call the callable to get the coroutine: - -```rust -#[pyfunction] -fn await_coro(callable: &PyAny) -> PyResult<()> { - // get the coroutine by calling the callable - let coro = callable.call0()?; - - // convert the coroutine into a Rust future using the - // async_std runtime - let f = pyo3_asyncio::async_std::into_future(coro)?; - - pyo3_asyncio::async_std::run_until_complete(coro.py(), async move { - // await the future - f.await?; - Ok(()) - }) -} -``` - -This can be particularly helpful where you need to repeatedly create and await a coroutine. Trying to await the same coroutine multiple times will raise an error: - -```python -RuntimeError: cannot reuse already awaited coroutine -``` - -> If you're interested in learning more about `coroutines` and `awaitables` in general, check out the -> [Python 3 `asyncio` docs](https://docs.python.org/3/library/asyncio-task.html) for more information. - -## Awaiting a Rust Future in Python - -Here we have the same async function as before written in Rust using the -[`async-std`](https://async.rs/) runtime: - -```rust -/// Sleep for 1 second -async fn rust_sleep() { - async_std::task::sleep(std::time::Duration::from_secs(1)).await; -} -``` - -Similar to Python, Rust's async functions also return a special object called a -`Future`: - -```rust -let future = rust_sleep(); -``` - -We can convert this `Future` object into Python to make it `awaitable`. This tells Python that you -can use the `await` keyword with it. In order to do this, we'll call -[`pyo3_asyncio::async_std::future_into_py`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.future_into_py.html): - -```rust -use pyo3::prelude::*; - -async fn rust_sleep() { - async_std::task::sleep(std::time::Duration::from_secs(1)).await; -} - -#[pyfunction] -fn call_rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { - pyo3_asyncio::async_std::future_into_py(py, async move { - rust_sleep().await; - Ok(Python::with_gil(|py| py.None())) - }) -} -``` - -In Python, we can call this pyo3 function just like any other async function: - -```python -from example import call_rust_sleep - -async def rust_sleep(): - await call_rust_sleep() -``` - -## Managing Event Loops - -Python's event loop requires some special treatment, especially regarding the main thread. Some of -Python's `asyncio` features, like proper signal handling, require control over the main thread, which -doesn't always play well with Rust. - -Luckily, Rust's event loops are pretty flexible and don't _need_ control over the main thread, so in -`pyo3-asyncio`, we decided the best way to handle Rust/Python interop was to just surrender the main -thread to Python and run Rust's event loops in the background. Unfortunately, since most event loop -implementations _prefer_ control over the main thread, this can still make some things awkward. - -### PyO3 Asyncio Initialization - -Because Python needs to control the main thread, we can't use the convenient proc macros from Rust -runtimes to handle the `main` function or `#[test]` functions. Instead, the initialization for PyO3 has to be done from the `main` function and the main -thread must block on [`pyo3_asyncio::run_forever`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.run_forever.html) or [`pyo3_asyncio::async_std::run_until_complete`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.run_until_complete.html). - -Because we have to block on one of those functions, we can't use [`#[async_std::main]`](https://docs.rs/async-std/latest/async_std/attr.main.html) or [`#[tokio::main]`](https://docs.rs/tokio/1.1.0/tokio/attr.main.html) -since it's not a good idea to make long blocking calls during an async function. - -> Internally, these `#[main]` proc macros are expanded to something like this: -> ```rust -> fn main() { -> // your async main fn -> async fn _main_impl() { /* ... */ } -> Runtime::new().block_on(_main_impl()); -> } -> ``` -> Making a long blocking call inside the `Future` that's being driven by `block_on` prevents that -> thread from doing anything else and can spell trouble for some runtimes (also this will actually -> deadlock a single-threaded runtime!). Many runtimes have some sort of `spawn_blocking` mechanism -> that can avoid this problem, but again that's not something we can use here since we need it to -> block on the _main_ thread. - -For this reason, `pyo3-asyncio` provides its own set of proc macros to provide you with this -initialization. These macros are intended to mirror the initialization of `async-std` and `tokio` -while also satisfying the Python runtime's needs. - -Here's a full example of PyO3 initialization with the `async-std` runtime: -```rust -use pyo3::prelude::*; - -#[pyo3_asyncio::async_std::main] -async fn main() -> PyResult<()> { - // PyO3 is initialized - Ready to go - - let fut = Python::with_gil(|py| -> PyResult<_> { - let asyncio = py.import("asyncio")?; - - // convert asyncio.sleep into a Rust Future - pyo3_asyncio::async_std::into_future( - asyncio.call_method1("sleep", (1.into_py(py),))? - ) - })?; - - fut.await?; - - Ok(()) -} -``` - -### A Note About `asyncio.run` - -In Python 3.7+, the recommended way to run a top-level coroutine with `asyncio` -is with `asyncio.run`. In `v0.13` we recommended against using this function due to initialization issues, but in `v0.14` it's perfectly valid to use this function... with a caveat. - -Since our Rust <--> Python conversions require a reference to the Python event loop, this poses a problem. Imagine we have a PyO3 Asyncio module that defines -a `rust_sleep` function like in previous examples. You might rightfully assume that you can call pass this directly into `asyncio.run` like this: - -```python -import asyncio - -from my_async_module import rust_sleep - -asyncio.run(rust_sleep()) -``` - -You might be surprised to find out that this throws an error: -```bash -Traceback (most recent call last): - File "example.py", line 5, in - asyncio.run(rust_sleep()) -RuntimeError: no running event loop -``` - -What's happening here is that we are calling `rust_sleep` _before_ the future is -actually running on the event loop created by `asyncio.run`. This is counter-intuitive, but expected behaviour, and unfortunately there doesn't seem to be a good way of solving this problem within PyO3 Asyncio itself. - -However, we can make this example work with a simple workaround: - -```python -import asyncio - -from my_async_module import rust_sleep - -# Calling main will just construct the coroutine that later calls rust_sleep. -# - This ensures that rust_sleep will be called when the event loop is running, -# not before. -async def main(): - await rust_sleep() - -# Run the main() coroutine at the top-level instead -asyncio.run(main()) -``` - -### Non-standard Python Event Loops - -Python allows you to use alternatives to the default `asyncio` event loop. One -popular alternative is `uvloop`. In `v0.13` using non-standard event loops was -a bit of an ordeal, but in `v0.14` it's trivial. - -#### Using `uvloop` in a PyO3 Asyncio Native Extensions - -```toml -# Cargo.toml - -[lib] -name = "my_async_module" -crate-type = ["cdylib"] - -[dependencies] -pyo3 = { version = "0.14", features = ["extension-module"] } -pyo3-asyncio = { version = "0.14", features = ["tokio-runtime"] } -async-std = "1.9" -tokio = "1.4" -``` - -```rust -//! lib.rs - -use pyo3::{prelude::*, wrap_pyfunction}; - -#[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { - pyo3_asyncio::tokio::future_into_py(py, async { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - Ok(Python::with_gil(|py| py.None())) - }) -} - -#[pymodule] -fn my_async_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; - - Ok(()) -} -``` - -```bash -$ maturin develop && python3 -🔗 Found pyo3 bindings -🐍 Found CPython 3.8 at python3 - Finished dev [unoptimized + debuginfo] target(s) in 0.04s -Python 3.8.8 (default, Apr 13 2021, 19:58:26) -[GCC 7.3.0] :: Anaconda, Inc. on linux -Type "help", "copyright", "credits" or "license" for more information. ->>> import asyncio ->>> import uvloop ->>> ->>> import my_async_module ->>> ->>> uvloop.install() ->>> ->>> async def main(): -... await my_async_module.rust_sleep() -... ->>> asyncio.run(main()) ->>> -``` - -#### Using `uvloop` in Rust Applications - -Using `uvloop` in Rust applications is a bit trickier, but it's still possible -with relatively few modifications. - -Unfortunately, we can't make use of the `#[pyo3_asyncio::::main]` attribute with non-standard event loops. This is because the `#[pyo3_asyncio::::main]` proc macro has to interact with the Python -event loop before we can install the `uvloop` policy. - -```toml -[dependencies] -async-std = "1.9" -pyo3 = "0.14" -pyo3-asyncio = { version = "0.14", features = ["async-std-runtime"] } -``` - -```rust -//! main.rs - -use pyo3::{prelude::*, types::PyType}; - -fn main() -> PyResult<()> { - pyo3::prepare_freethreaded_python(); - - Python::with_gil(|py| { - let uvloop = py.import("uvloop")?; - uvloop.call_method0("install")?; - - // store a reference for the assertion - let uvloop = PyObject::from(uvloop); - - pyo3_asyncio::async_std::run(py, async move { - // verify that we are on a uvloop.Loop - Python::with_gil(|py| -> PyResult<()> { - assert!(pyo3_asyncio::async_std::get_current_loop(py)?.is_instance( - uvloop - .as_ref(py) - .getattr("Loop")? - )?); - Ok(()) - })?; - - async_std::task::sleep(std::time::Duration::from_secs(1)).await; - - Ok(()) - }) - }) -} -``` +If you are working with a Python library that makes use of async functions or wish to provide Python bindings for an async Rust library, [`pyo3-async-runtimes`](https://github.com/PyO3/pyo3-async-runtimes) likely has the tools you need. +It provides conversions between async functions in both Python and Rust and was designed with first-class support for popular Rust runtimes such as [`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). +In addition, all async Python code runs on the default `asyncio` event loop, so `pyo3-async-runtimes` should work just fine with existing Python libraries. ## Additional Information -- Managing event loop references can be tricky with pyo3-asyncio. See [Event Loop References](https://docs.rs/pyo3-asyncio/#event-loop-references) in the API docs to get a better intuition for how event loop references are managed in this library. -- Testing pyo3-asyncio libraries and applications requires a custom test harness since Python requires control over the main thread. You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/testing) + +- Managing event loop references can be tricky with `pyo3-async-runtimes`. + See [Event Loop References](https://docs.rs/pyo3-async-runtimes/#event-loop-references-and-contextvars) in the API docs to get a better intuition for how event loop references are managed in this library. +- Testing `pyo3-async-runtimes` libraries and applications requires a custom test harness since Python requires control over the main thread. + You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-async-runtimes/latest/pyo3_async_runtimes/testing) diff --git a/guide/src/ecosystem/logging.md b/guide/src/ecosystem/logging.md index 2e7d4a087c6..fa97c9bc7dd 100644 --- a/guide/src/ecosystem/logging.md +++ b/guide/src/ecosystem/logging.md @@ -3,39 +3,36 @@ It is desirable if both the Python and Rust parts of the application end up logging using the same configuration into the same place. -This section of the guide briefly discusses how to connect the two languages' -logging ecosystems together. The recommended way for Python extension modules is -to configure Rust's logger to send log messages to Python using the `pyo3-log` -crate. For users who want to do the opposite and send Python log messages to -Rust, see the note at the end of this guide. +This section of the guide briefly discusses how to connect the two languages' logging ecosystems together. +The recommended way for Python extension modules is to configure Rust's logger to send log messages to Python using the `pyo3-log` crate. +For users who want to do the opposite and send Python log messages to Rust, see the note at the end of this guide. ## Using `pyo3-log` to send Rust log messages to Python -The [pyo3-log] crate allows sending the messages from the Rust side to Python's -[logging] system. This is mostly suitable for writing native extensions for -Python programs. +The [pyo3-log] crate allows sending the messages from the Rust side to Python's [logging] system. +This is mostly suitable for writing native extensions for Python programs. Use [`pyo3_log::init`][init] to install the logger in its default configuration. It's also possible to tweak its configuration (mostly to tune its performance). -```rust -use log::info; -use pyo3::prelude::*; - -#[pyfunction] -fn log_something() { - // This will use the logger installed in `my_module` to send the `info` - // message to the Python logging facilities. - info!("Something!"); -} - -#[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - // A good place to install the Rust -> Python logger. - pyo3_log::init(); - - m.add_function(wrap_pyfunction!(log_something))?; - Ok(()) +```rust,no_run +#[pyo3::pymodule] +mod my_module { + use log::info; + use pyo3::prelude::*; + + #[pyfunction] + fn log_something() { + // This will use the logger installed in `my_module` to send the `info` + // message to the Python logging facilities. + info!("Something!"); + } + + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + // A good place to install the Rust -> Python logger. + pyo3_log::init(); + } } ``` @@ -51,9 +48,8 @@ logging.getLogger().setLevel(logging.INFO) my_module.log_something() ``` -It is important to initialize the Python loggers first, before calling any Rust -functions that may log. This limitation can be worked around if it is not -possible to satisfy, read the documentation about [caching]. +It is important to initialize the Python loggers first, before calling any Rust functions that may log. +This limitation can be worked around if it is not possible to satisfy, read the documentation about [caching]. ## The Python to Rust direction @@ -61,7 +57,7 @@ To have python logs be handled by Rust, one need only register a rust function t This has been implemented within the [pyo3-pylogger] crate. -```rust +```rust,no_run use log::{info, warn}; use pyo3::prelude::*; @@ -78,7 +74,7 @@ fn main() -> PyResult<()> { warn!("Something spooky happened!"); // Log some messages from Python - Python::with_gil(|py| { + Python::attach(|py| { py.run( " import logging diff --git a/guide/src/ecosystem/tracing.md b/guide/src/ecosystem/tracing.md new file mode 100644 index 00000000000..34beb03ed05 --- /dev/null +++ b/guide/src/ecosystem/tracing.md @@ -0,0 +1,106 @@ +# Tracing + +Python projects that write extension modules for performance reasons may want to +tap into [Rust's `tracing` ecosystem] to gain insight into the performance of +their extension module. + +This section of the guide describes a few crates that provide ways to do that. +They build on [`tracing_subscriber`][tracing-subscriber] and require code changes in both Python and Rust to integrate. +Note that each extension module must configure its own `tracing` integration; one extension module will not see `tracing` data from a different module. + +## `pyo3-tracing-subscriber` ([documentation][pyo3-tracing-subscriber-docs]) + +[`pyo3-tracing-subscriber`][pyo3-tracing-subscriber] provides a way for Python projects to configure `tracing_subscriber`. +It exposes a few `tracing_subscriber` layers: + +- `tracing_subscriber::fmt` for writing human-readable output to file or stdout +- `opentelemetry-stdout` for writing OTLP output to file or stdout +- `opentelemetry-otlp` for writing OTLP output to an OTLP endpoint + +The extension module must call [`pyo3_tracing_subscriber::add_submodule`][add-submodule] +to export the Python classes needed to configure and initialize `tracing`. + +On the Python side, use the `Tracing` context manager to initialize tracing and +run Rust code inside the context manager's block. `Tracing` takes a +`GlobalTracingConfig` instance describing the layers to be used. + +See [the README on crates.io][pyo3-tracing-subscriber] +for example code. + +## `pyo3-python-tracing-subscriber` ([documentation][pyo3-python-tracing-subscriber-docs]) + +The similarly-named [`pyo3-python-tracing-subscriber`][pyo3-python-tracing-subscriber] +implements a shim in Rust that forwards `tracing` data to a `Layer` +implementation defined in and passed in from Python. + +There are many ways an extension module could integrate `pyo3-python-tracing-subscriber` +but a simple one may look something like this: + +```rust,no_run +#[tracing::instrument] +#[pyfunction] +fn fibonacci(index: usize, use_memoized: bool) -> PyResult { + // ... +} + +#[pyfunction] +pub fn initialize_tracing(py_impl: Bound<'_, PyAny>) { + tracing_subscriber::registry() + .with(pyo3_python_tracing_subscriber::PythonCallbackLayerBridge::new(py_impl)) + .init(); +} +``` + +The extension module must provide some way for Python to pass in one or more Python objects that implement [the `Layer` interface]. +Then it should construct [`pyo3_python_tracing_subscriber::PythonCallbackLayerBridge`][PythonCallbackLayerBridge] instances with each of those Python objects and initialize `tracing_subscriber` as shown above. + +The Python objects implement a modified version of the `Layer` interface: + +- `on_new_span()` may return some state that will stored inside the Rust span +- other callbacks will be given that state as an additional positional argument + +A dummy `Layer` implementation may look like this: + +```python +import rust_extension + +class MyPythonLayer: + def __init__(self): + pass + + # `on_new_span` can return some state + def on_new_span(self, span_attrs: str, span_id: str) -> int: + print(f"[on_new_span]: {span_attrs} | {span_id}") + return random.randint(1, 1000) + + # The state from `on_new_span` is passed back into other trait methods + def on_event(self, event: str, state: int): + print(f"[on_event]: {event} | {state}") + + def on_close(self, span_id: str, state: int): + print(f"[on_close]: {span_id} | {state}") + + def on_record(self, span_id: str, values: str, state: int): + print(f"[on_record]: {span_id} | {values} | {state}") + +def main(): + rust_extension.initialize_tracing(MyPythonLayer()) + + print("10th fibonacci number: ", rust_extension.fibonacci(10, True)) +``` + +`pyo3-python-tracing-subscriber` has [working examples] +showing both the Rust side and the Python side of an integration. + +[pyo3-tracing-subscriber]: https://crates.io/crates/pyo3-tracing-subscriber +[pyo3-tracing-subscriber-docs]: https://docs.rs/pyo3-tracing-subscriber +[add-submodule]: https://docs.rs/pyo3-tracing-subscriber/*/pyo3_tracing_subscriber/fn.add_submodule.html + +[pyo3-python-tracing-subscriber]: https://crates.io/crates/pyo3-python-tracing-subscriber +[pyo3-python-tracing-subscriber-docs]: https://docs.rs/pyo3-python-tracing-subscriber +[PythonCallbackLayerBridge]: https://docs.rs/pyo3-python-tracing-subscriber/*/pyo3_python_tracing_subscriber/struct.PythonCallbackLayerBridge.html +[working examples]: https://github.com/getsentry/pyo3-python-tracing-subscriber/tree/main/demo + +[Rust's `tracing` ecosystem]: https://crates.io/crates/tracing +[tracing-subscriber]: https://docs.rs/tracing-subscriber/*/tracing_subscriber/ +[the `Layer` interface]: https://docs.rs/tracing-subscriber/*/tracing_subscriber/layer/trait.Layer.html diff --git a/guide/src/exception.md b/guide/src/exception.md index 225118576f7..bc7be1f65c7 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -10,8 +10,8 @@ use pyo3::create_exception; create_exception!(module, MyError, pyo3::exceptions::PyException); ``` -* `module` is the name of the containing module. -* `MyError` is the name of the new exception type. +- `module` is the name of the containing module. +- `MyError` is the name of the new exception type. For example: @@ -23,38 +23,43 @@ use pyo3::exceptions::PyException; create_exception!(mymodule, CustomError, PyException); -Python::with_gil(|py| { - let ctx = [("CustomError", py.get_type::())].into_py_dict_bound(py); +# fn main() -> PyResult<()> { +Python::attach(|py| { + let ctx = [("CustomError", py.get_type::())].into_py_dict(py)?; pyo3::py_run!( py, *ctx, "assert str(CustomError) == \"\"" ); pyo3::py_run!(py, *ctx, "assert CustomError('oops').args == ('oops',)"); -}); +# Ok(()) +}) +# } ``` When using PyO3 to create an extension module, you can add the new exception to the module like this, so that it is importable from Python: -```rust +```rust,no_run +# fn main() {} use pyo3::prelude::*; use pyo3::exceptions::PyException; pyo3::create_exception!(mymodule, CustomError, PyException); #[pymodule] -fn mymodule(py: Python<'_>, m: &PyModule) -> PyResult<()> { - // ... other elements added to module ... - m.add("CustomError", py.get_type::())?; +mod mymodule { + #[pymodule_export] + use super::CustomError; - Ok(()) + // ... other elements added to module ... } ``` ## Raising an exception -As described in the [function error handling](./function/error_handling.md) chapter, to raise an exception from a `#[pyfunction]` or `#[pymethods]`, return an `Err(PyErr)`. PyO3 will automatically raise this exception for you when returning the result to Python. +As described in the [function error handling](./function/error-handling.md) chapter, to raise an exception from a `#[pyfunction]` or `#[pymethods]`, return an `Err(PyErr)`. +PyO3 will automatically raise this exception for you when returning the result to Python. You can also manually write and fetch errors in the Python interpreter's global state: @@ -62,7 +67,7 @@ You can also manually write and fetch errors in the Python interpreter's global use pyo3::{Python, PyErr}; use pyo3::exceptions::PyTypeError; -Python::with_gil(|py| { +Python::attach(|py| { PyTypeError::new_err("Error").restore(py); assert!(PyErr::occurred(py)); drop(PyErr::fetch(py)); @@ -74,24 +79,27 @@ Python::with_gil(|py| { Python has an [`isinstance`](https://docs.python.org/3/library/functions.html#isinstance) method to check an object's type. In PyO3 every object has the [`PyAny::is_instance`] and [`PyAny::is_instance_of`] methods which do the same thing. -```rust +```rust,no_run use pyo3::prelude::*; use pyo3::types::{PyBool, PyList}; -Python::with_gil(|py| { - assert!(PyBool::new_bound(py, true).is_instance_of::()); - let list = PyList::new_bound(py, &[1, 2, 3, 4]); +# fn main() -> PyResult<()> { +Python::attach(|py| { + assert!(PyBool::new(py, true).is_instance_of::()); + let list = PyList::new(py, &[1, 2, 3, 4])?; assert!(!list.is_instance_of::()); assert!(list.is_instance_of::()); -}); +# Ok(()) +}) +# } ``` To check the type of an exception, you can similarly do: -```rust +```rust,no_run # use pyo3::exceptions::PyTypeError; # use pyo3::prelude::*; -# Python::with_gil(|py| { +# Python::attach(|py| { # let err = PyTypeError::new_err(()); err.is_instance_of::(py); # }); @@ -100,10 +108,10 @@ err.is_instance_of::(py); ## Using exceptions defined in Python code It is possible to use an exception defined in Python code as a native Rust type. -The `import_exception!` macro allows importing a specific exception class and defines a Rust type +The [`import_exception!`] macro allows importing a specific exception class and defines a Rust type for that exception. -```rust +```rust,no_run #![allow(dead_code)] use pyo3::prelude::*; @@ -111,7 +119,7 @@ mod io { pyo3::import_exception!(io, UnsupportedOperation); } -fn tell(file: &PyAny) -> PyResult { +fn tell(file: &Bound<'_, PyAny>) -> PyResult { match file.call_method0("tell") { Err(_) => Err(io::UnsupportedOperation::new_err("not supported: tell")), Ok(x) => x.extract::(), @@ -122,11 +130,54 @@ fn tell(file: &PyAny) -> PyResult { [`pyo3::exceptions`]({{#PYO3_DOCS_URL}}/pyo3/exceptions/index.html) defines exceptions for several standard library modules. +## Creating more complex exceptions + +If you need to create an exception with more complex behavior, you can also manually create a subclass of `PyException`: + +```rust +#![allow(dead_code)] +# #[cfg(any(not(feature = "abi3")))] { +use pyo3::prelude::*; +use pyo3::types::IntoPyDict; +use pyo3::exceptions::PyException; + +#[pyclass(extends=PyException)] +struct CustomError { + #[pyo3(get)] + url: String, + + #[pyo3(get)] + message: String, +} + +#[pymethods] +impl CustomError { + #[new] + fn new(url: String, message: String) -> Self { + Self { url, message } + } +} + +# fn main() -> PyResult<()> { +Python::attach(|py| { + let ctx = [("CustomError", py.get_type::())].into_py_dict(py)?; + pyo3::py_run!( + py, + *ctx, + "assert str(CustomError) == \"\", repr(CustomError)" + ); + pyo3::py_run!(py, *ctx, "assert CustomError('https://example.com/', 'something went bad').args == ('https://example.com/', 'something went bad')"); + pyo3::py_run!(py, *ctx, "assert CustomError('https://example.com/', 'something went bad').url == 'https://example.com/'"); +# Ok(()) +}) +# } +# } + +``` + +Note that this is not possible when the ``abi3`` feature is enabled, as that prevents subclassing ``PyException``. + [`create_exception!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.create_exception.html [`import_exception!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.import_exception.html - -[`PyErr`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html -[`PyResult`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyResult.html -[`PyErr::from_value`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html#method.from_value -[`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#method.is_instance -[`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#method.is_instance_of +[`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.is_instance +[`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.is_instance_of diff --git a/guide/src/faq.md b/guide/src/faq.md index 8308acc64de..4fdb7f2b78c 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -1,55 +1,68 @@ # Frequently Asked Questions and troubleshooting -## I'm experiencing deadlocks using PyO3 with lazy_static or once_cell! +Sorry that you're having trouble using PyO3. +If you can't find the answer to your problem in the list below, you can also reach out for help on [GitHub Discussions](https://github.com/PyO3/pyo3/discussions) and on [Discord](https://discord.gg/33kcChzH7f). -`lazy_static` and `once_cell::sync` both use locks to ensure that initialization is performed only by a single thread. Because the Python GIL is an additional lock this can lead to deadlocks in the following way: +## I'm experiencing deadlocks using PyO3 with `std::sync::OnceLock`, `std::sync::LazyLock`, `lazy_static`, and `once_cell` -1. A thread (thread A) which has acquired the Python GIL starts initialization of a `lazy_static` value. -2. The initialization code calls some Python API which temporarily releases the GIL e.g. `Python::import`. -3. Another thread (thread B) acquires the Python GIL and attempts to access the same `lazy_static` value. -4. Thread B is blocked, because it waits for `lazy_static`'s initialization to lock to release. -5. Thread A is blocked, because it waits to re-acquire the GIL which thread B still holds. +`OnceLock`, `LazyLock`, and their thirdparty predecessors use blocking to ensure only one thread ever initializes them. +Because the Python interpreter can introduce additional locks (the Python GIL and GC can both require all other threads to pause) this can lead to deadlocks in the following way: + +1. A thread (thread A) which is attached to the Python interpreter starts initialization of a `OnceLock` value. +2. The initialization code calls some Python API which temporarily detaches from the interpreter e.g. `Python::import`. +3. Another thread (thread B) attaches to the Python interpreter and attempts to access the same `OnceLock` value. +4. Thread B is blocked, because it waits for `OnceLock`'s initialization to lock to release. +5. On non-free-threaded Python, thread A is now also blocked, because it waits to re-attach to the interpreter (by taking the GIL which thread B still holds). 6. Deadlock. -PyO3 provides a struct [`GILOnceCell`] which works equivalently to `OnceCell` but relies solely on the Python GIL for thread safety. This means it can be used in place of `lazy_static` or `once_cell` where you are experiencing the deadlock described above. See the documentation for [`GILOnceCell`] for an example how to use it. +PyO3 provides a struct [`PyOnceLock`] which implements a single-initialization API based on these types that avoids deadlocks. +You can also make use of the [`OnceExt`] and [`OnceLockExt`] extension traits that enable using the standard library types for this purpose by providing new methods for these types that avoid the risk of deadlocking with the Python interpreter. +This means they can be used in place of other choices when you are experiencing the deadlock described above. +See the documentation for [`PyOnceLock`] and [`OnceExt`] for further details and an example how to use them. -[`GILOnceCell`]: {{#PYO3_DOCS_URL}}/pyo3/sync/struct.GILOnceCell.html +[`PyOnceLock`]: {{#PYO3_DOCS_URL}}/pyo3/sync/struct.PyOnceLock.html +[`OnceExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html +[`OnceLockExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceLockExt.html -## I can't run `cargo test`; or I can't build in a Cargo workspace: I'm having linker issues like "Symbol not found" or "Undefined reference to _PyExc_SystemError"! +## I can't run `cargo test`; or I can't build in a Cargo workspace: I'm having linker issues like "Symbol not found" or "Undefined reference to _PyExc_SystemError" -Currently, [#340](https://github.com/PyO3/pyo3/issues/340) causes `cargo test` to fail with linking errors when the `extension-module` feature is activated. Linking errors can also happen when building in a cargo workspace where a different crate also uses PyO3 (see [#2521](https://github.com/PyO3/pyo3/issues/2521)). For now, there are three ways we can work around these issues. +Currently, [#340](https://github.com/PyO3/pyo3/issues/340) causes `cargo test` to fail with linking errors when the `extension-module` feature is activated. +Linking errors can also happen when building in a cargo workspace where a different crate also uses PyO3 (see [#2521](https://github.com/PyO3/pyo3/issues/2521)). +For now, there are three ways we can work around these issues. -1. Make the `extension-module` feature optional. Build with `maturin develop --features "extension-module"` +1. Make the `extension-module` feature optional. + Build with `maturin develop --features "extension-module"` -```toml -[dependencies.pyo3] -{{#PYO3_CRATE_VERSION}} + ```toml + [dependencies.pyo3] + {{#PYO3_CRATE_VERSION}} -[features] -extension-module = ["pyo3/extension-module"] -``` + [features] + extension-module = ["pyo3/extension-module"] + ``` -2. Make the `extension-module` feature optional and default. Run tests with `cargo test --no-default-features`: +2. Make the `extension-module` feature optional and default. + Run tests with `cargo test --no-default-features`: -```toml -[dependencies.pyo3] -{{#PYO3_CRATE_VERSION}} + ```toml + [dependencies.pyo3] + {{#PYO3_CRATE_VERSION}} -[features] -extension-module = ["pyo3/extension-module"] -default = ["extension-module"] -``` + [features] + extension-module = ["pyo3/extension-module"] + default = ["extension-module"] + ``` 3. If you are using a [`pyproject.toml`](https://maturin.rs/metadata.html) file to control maturin settings, add the following section: -```toml -[tool.maturin] -features = ["pyo3/extension-module"] -# Or for maturin 0.12: -# cargo-extra-args = ["--features", "pyo3/extension-module"] -``` + ```toml + [tool.maturin] + features = ["pyo3/extension-module"] + # Or for maturin 0.12: + # cargo-extra-args = ["--features", "pyo3/extension-module"] + ``` -## I can't run `cargo test`: my crate cannot be found for tests in `tests/` directory! +## I can't run `cargo test`: my crate cannot be found for tests in `tests/` directory The Rust book suggests to [put integration tests inside a `tests/` directory](https://doc.rust-lang.org/book/ch11-03-test-organization.html#integration-tests). @@ -72,17 +85,19 @@ The best solution is to make your crate types include both `rlib` and `cdylib`: crate-type = ["cdylib", "rlib"] ``` -## Ctrl-C doesn't do anything while my Rust code is executing! +## Ctrl-C doesn't do anything while my Rust code is executing -This is because Ctrl-C raises a SIGINT signal, which is handled by the calling Python process by simply setting a flag to action upon later. This flag isn't checked while Rust code called from Python is executing, only once control returns to the Python interpreter. +This is because Ctrl-C raises a SIGINT signal, which is handled by the calling Python process by simply setting a flag to action upon later. +This flag isn't checked while Rust code called from Python is executing, only once control returns to the Python interpreter. -You can give the Python interpreter a chance to process the signal properly by calling `Python::check_signals`. It's good practice to call this function regularly if you have a long-running Rust function so that your users can cancel it. +You can give the Python interpreter a chance to process the signal properly by calling `Python::check_signals`. +It's good practice to call this function regularly if you have a long-running Rust function so that your users can cancel it. -## `#[pyo3(get)]` clones my field! +## `#[pyo3(get)]` clones my field You may have a nested struct similar to this: -```rust +```rust,no_run # use pyo3::prelude::*; #[pyclass] #[derive(Clone)] @@ -119,18 +134,19 @@ AssertionError: a: b: ``` -This can be especially confusing if the field is mutable, as getting the field and then mutating it won't persist - you'll just get a fresh clone of the original on the next access. Unfortunately Python and Rust don't agree about ownership - if PyO3 gave out references to (possibly) temporary Rust objects to Python code, Python code could then keep that reference alive indefinitely. Therefore returning Rust objects requires cloning. +This can be especially confusing if the field is mutable, as getting the field and then mutating it won't persist - you'll just get a fresh clone of the original on the next access. +Unfortunately Python and Rust don't agree about ownership - if PyO3 gave out references to (possibly) temporary Rust objects to Python code, Python code could then keep that reference alive indefinitely. +Therefore returning Rust objects requires cloning. If you don't want that cloning to happen, a workaround is to allocate the field on the Python heap and store a reference to that, by using [`Py<...>`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html): -```rust + +```rust,no_run # use pyo3::prelude::*; #[pyclass] -#[derive(Clone)] struct Inner {/* fields omitted */} #[pyclass] struct Outer { - #[pyo3(get)] inner: Py, } @@ -142,9 +158,16 @@ impl Outer { inner: Py::new(py, Inner {})?, }) } + + #[getter] + fn inner(&self, py: Python<'_>) -> Py { + self.inner.clone_ref(py) + } } ``` + This time `a` and `b` *are* the same object: + ```python outer = Outer() @@ -159,9 +182,11 @@ print(f"a: {a}\nb: {b}") a: b: ``` -The downside to this approach is that any Rust code working on the `Outer` struct now has to acquire the GIL to do anything with its field. -## I want to use the `pyo3` crate re-exported from from dependency but the proc-macros fail! +The downside to this approach is that any Rust code working on the `Outer` struct potentially has to attach to the Python interpreter to do anything with the `inner` field. (If `Inner` is `#[pyclass(frozen)]` and implements `Sync`, then `Py::get` +may be used to access the `Inner` contents from `Py` without needing to attach to the interpreter.) + +## I want to use the `pyo3` crate re-exported from dependency but the proc-macros fail All PyO3 proc-macros (`#[pyclass]`, `#[pyfunction]`, `#[derive(FromPyObject)]` and so on) expect the `pyo3` crate to be available under that name in your crate @@ -172,27 +197,33 @@ However, when the dependency is renamed, or your crate only indirectly depends on `pyo3`, you need to let the macro code know where to find the crate. This is done with the `crate` attribute: -```rust +```rust,no_run # use pyo3::prelude::*; # pub extern crate pyo3; # mod reexported { pub use ::pyo3; } +# #[allow(dead_code)] #[pyclass] #[pyo3(crate = "reexported::pyo3")] struct MyClass; ``` -## I'm trying to call Python from Rust but I get `STATUS_DLL_NOT_FOUND` or `STATUS_ENTRYPOINT_NOT_FOUND`! +## I'm trying to call Python from Rust but I get `STATUS_DLL_NOT_FOUND` or `STATUS_ENTRYPOINT_NOT_FOUND` + +This happens on Windows when linking to the python DLL fails or the wrong one is linked. +The Python DLL on Windows will usually be called something like: -This happens on Windows when linking to the python DLL fails or the wrong one is linked. The Python DLL on Windows will usually be called something like: - `python3X.dll` for Python 3.X, e.g. `python310.dll` for Python 3.10 - `python3.dll` when using PyO3's `abi3` feature -The DLL needs to be locatable using the [Windows DLL search order](https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order#standard-search-order-for-unpackaged-apps). Some ways to achieve this are: +The DLL needs to be locatable using the [Windows DLL search order](https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order#standard-search-order-for-unpackaged-apps). +Some ways to achieve this are: + - Put the Python DLL in the same folder as your build artifacts - Add the directory containing the Python DLL to your `PATH` environment variable, for example `C:\Users\\AppData\Local\Programs\Python\Python310` - If this happens when you are *distributing* your program, consider using [PyOxidizer](https://github.com/indygreg/PyOxidizer) to package it with your binary. -If the wrong DLL is linked it is possible that this happened because another program added itself and its own Python DLLs to `PATH`. Rearrange your `PATH` variables to give the correct DLL priority. +If the wrong DLL is linked it is possible that this happened because another program added itself and its own Python DLLs to `PATH`. +Rearrange your `PATH` variables to give the correct DLL priority. > **Note**: Changes to `PATH` (or any other environment variable) are not visible to existing shells. Restart it for changes to take effect. diff --git a/guide/src/features.md b/guide/src/features.md index 43124e0076e..44172b258aa 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -1,6 +1,7 @@ # Features reference -PyO3 provides a number of Cargo features to customize functionality. This chapter of the guide provides detail on each of them. +PyO3 provides a number of Cargo features to customize functionality. +This chapter of the guide provides detail on each of them. By default, only the `macros` feature is enabled. @@ -12,7 +13,7 @@ This feature is required when building a Python extension module using PyO3. It tells PyO3's build script to skip linking against `libpython.so` on Unix platforms, where this must not be done. -See the [building and distribution](building_and_distribution.md#linking) section for further detail. +See the [building and distribution](building-and-distribution.md#the-extension-module-feature) section for further detail. ### `abi3` @@ -20,7 +21,7 @@ This feature is used when building Python extension modules to create wheels whi It restricts PyO3's API to a subset of the full Python API which is guaranteed by [PEP 384](https://www.python.org/dev/peps/pep-0384/) to be forwards-compatible with future Python versions. -See the [building and distribution](building_and_distribution.md#py_limited_apiabi3) section for further detail. +See the [building and distribution](building-and-distribution.md#py_limited_apiabi3) section for further detail. ### The `abi3-pyXY` features @@ -28,7 +29,7 @@ See the [building and distribution](building_and_distribution.md#py_limited_apia These features are extensions of the `abi3` feature to specify the exact minimum Python version which the multiple-version-wheel will support. -See the [building and distribution](building_and_distribution.md#minimum-python-version-for-abi3) section for further detail. +See the [building and distribution](building-and-distribution.md#minimum-python-version-for-abi3) section for further detail. ### `generate-import-lib` @@ -38,30 +39,45 @@ for MinGW-w64 and MSVC (cross-)compile targets. Enabling it allows to (cross-)compile extension modules to any Windows targets without having to install the Windows Python distribution files for the target. -See the [building and distribution](building_and_distribution.md#building-abi3-extensions-without-a-python-interpreter) +See the [building and distribution](building-and-distribution.md#building-abi3-extensions-without-a-python-interpreter) section for further detail. ## Features for embedding Python in Rust ### `auto-initialize` -This feature changes [`Python::with_gil`]({{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.with_gil) to automatically initialize a Python interpreter (by calling [`prepare_freethreaded_python`]({{#PYO3_DOCS_URL}}/pyo3/fn.prepare_freethreaded_python.html)) if needed. +This feature changes [`Python::attach`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.attach) to automatically initialize a Python interpreter (by calling [`Python::initialize`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.initialize)) if needed. -If you do not enable this feature, you should call `pyo3::prepare_freethreaded_python()` before attempting to call any other Python APIs. +If you do not enable this feature, you should call `Python::initialize()` before attempting to call any other Python APIs. ## Advanced Features +### `experimental-async` + +This feature adds support for `async fn` in `#[pyfunction]` and `#[pymethods]`. + +The feature has some unfinished refinements and performance improvements. +To help finish this off, see [issue #1632](https://github.com/PyO3/pyo3/issues/1632) and its associated draft PRs. + ### `experimental-inspect` -This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types. +This feature adds to the built binaries introspection data that can be then retrieved using the `pyo3-introspection` crate to generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html). + +Also, this feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types. + +This is a first step towards adding first-class support for generating type annotations automatically in PyO3, however work is needed to finish this off. +All feedback and offers of help welcome on [issue #2454](https://github.com/PyO3/pyo3/issues/2454). -This is a first step towards adding first-class support for generating type annotations automatically in PyO3, however work is needed to finish this off. All feedback and offers of help welcome on [issue #2454](https://github.com/PyO3/pyo3/issues/2454). +### `py-clone` -### `gil-refs` +This feature was introduced to ease migration. +It was found that delayed reference counting (which PyO3 used historically) could not be made sound and hence `Clone`-ing an instance of `Py` is impossible when not attached to Python interpreter (it will panic). +To avoid migrations introducing new panics without warning, the `Clone` implementation itself is now gated behind this feature. -This feature is a backwards-compatibility feature to allow continued use of the "GIL Refs" APIs deprecated in PyO3 0.21. These APIs have performance drawbacks and soundness edge cases which the newer `Bound` smart pointer and accompanying APIs resolve. +### `pyo3_disable_reference_pool` -This feature and the APIs it enables is expected to be removed in a future PyO3 version. +This is a performance-oriented conditional compilation flag, e.g. [set via `$RUSTFLAGS`][set-configuration-options], which disabled the global reference pool and the associated overhead for the crossing the Python-Rust boundary. +However, if enabled, `Drop`ping an instance of `Py` when not attached to the Python interpreter will abort the process. ### `macros` @@ -75,27 +91,31 @@ This feature enables a dependency on the `pyo3-macros` crate, which provides the It also provides the `py_run!` macro. -These macros require a number of dependencies which may not be needed by users who just need PyO3 for Python FFI. Disabling this feature enables faster builds for those users, as these dependencies will not be built if this feature is disabled. +These macros require a number of dependencies which may not be needed by users who just need PyO3 for Python FFI. +Disabling this feature enables faster builds for those users, as these dependencies will not be built if this feature is disabled. > This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml. ### `multiple-pymethods` -This feature enables a dependency on `inventory`, which enables each `#[pyclass]` to have more than one `#[pymethods]` block. This feature also requires a minimum Rust version of 1.62 due to limitations in the `inventory` crate. +This feature enables each `#[pyclass]` to have more than one `#[pymethods]` block. -Most users should only need a single `#[pymethods]` per `#[pyclass]`. In addition, not all platforms (e.g. Wasm) are supported by `inventory`. For this reason this feature is not enabled by default, meaning fewer dependencies and faster compilation for the majority of users. +Most users should only need a single `#[pymethods]` per `#[pyclass]`. +In addition, not all platforms (e.g. Wasm) are supported by `inventory`, which is used in the implementation of the feature. +For this reason this feature is not enabled by default, meaning fewer dependencies and faster compilation for the majority of users. See [the `#[pyclass]` implementation details](class.md#implementation-details) for more information. ### `nightly` -The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use the `auto_traits` and `negative_impls` features to fix the `Python::allow_threads` function. +The `nightly` feature needs the nightly Rust compiler. +This allows PyO3 to use the `auto_traits` and `negative_impls` features to fix the `Python::detach` function. ### `resolve-config` -The `resolve-config` feature of the `pyo3-build-config` crate controls whether that crate's -build script automatically resolves a Python interpreter / build configuration. This feature is primarily useful when building PyO3 -itself. By default this feature is not enabled, meaning you can freely use `pyo3-build-config` as a standalone library to read or write PyO3 build configuration files or resolve metadata about a Python interpreter. +The `resolve-config` feature of the `pyo3-build-config` crate controls whether that crate's build script automatically resolves a Python interpreter / build configuration. +This feature is primarily useful when building PyO3 itself. +By default this feature is not enabled, meaning you can freely use `pyo3-build-config` as a standalone library to read or write PyO3 build configuration files or resolve metadata about a Python interpreter. ## Optional Dependencies @@ -103,31 +123,60 @@ These features enable conversions between Python types and types from other Rust ### `anyhow` -Adds a dependency on [anyhow](https://docs.rs/anyhow). Enables a conversion from [anyhow](https://docs.rs/anyhow)’s [`Error`](https://docs.rs/anyhow/latest/anyhow/struct.Error.html) type to [`PyErr`]({{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html), for easy error handling. +Adds a dependency on [anyhow](https://docs.rs/anyhow). +Enables a conversion from [anyhow](https://docs.rs/anyhow)’s [`Error`](https://docs.rs/anyhow/latest/anyhow/struct.Error.html) type to [`PyErr`]({{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html), for easy error handling. + +### `arc_lock` + +Enables Pyo3's `MutexExt` trait for all Mutexes that extend on [`lock_api::Mutex`](https://docs.rs/lock_api/latest/lock_api/struct.Mutex.html) or [`parking_lot::ReentrantMutex`](https://docs.rs/lock_api/latest/lock_api/struct.ReentrantMutex.html) and are wrapped in an [`Arc`](https://doc.rust-lang.org/std/sync/struct.Arc.html) type. +Like [`Arc`](https://docs.rs/parking_lot/latest/parking_lot/type.Mutex.html#method.lock_arc) + +### `bigdecimal` + +Adds a dependency on [bigdecimal](https://docs.rs/bigdecimal) and enables conversions into its [`BigDecimal`](https://docs.rs/bigdecimal/latest/bigdecimal/struct.BigDecimal.html) type. + +### `bytes` + +Adds a dependency on [bytes](https://docs.rs/bytes/latest/bytes) and enables conversions into its [`Bytes`](https://docs.rs/bytes/latest/bytes/struct.Bytes.html) type. ### `chrono` -Adds a dependency on [chrono](https://docs.rs/chrono). Enables a conversion from [chrono](https://docs.rs/chrono)'s types to python: -- [Duration](https://docs.rs/chrono/latest/chrono/struct.Duration.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) +Adds a dependency on [chrono](https://docs.rs/chrono). +Enables a conversion from [chrono](https://docs.rs/chrono)'s types to python: + +- [TimeDelta](https://docs.rs/chrono/latest/chrono/struct.TimeDelta.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) - [FixedOffset](https://docs.rs/chrono/latest/chrono/offset/struct.FixedOffset.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) - [Utc](https://docs.rs/chrono/latest/chrono/offset/struct.Utc.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html) - [NaiveDate](https://docs.rs/chrono/latest/chrono/naive/struct.NaiveDate.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html) - [NaiveTime](https://docs.rs/chrono/latest/chrono/naive/struct.NaiveTime.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html) - [DateTime](https://docs.rs/chrono/latest/chrono/struct.DateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +### `chrono-local` + +Enables conversion from and to [Local](https://docs.rs/chrono/latest/chrono/struct.Local.html) timezones. +The current system timezone as determined by [`iana_time_zone::get_timezone()`](https://docs.rs/iana-time-zone/latest/iana_time_zone/fn.get_timezone.html) will be used for conversions. + +`chrono::DateTime` will convert from either of: + +- `datetime` objects with `tzinfo` equivalent to the current system timezone. +- "naive" `datetime` objects (those without a `tzinfo`), as it is a convention that naive datetime objects should be treated as using the system timezone. + +When converting to Python, `Local` tzinfo is converted to a `zoneinfo.ZoneInfo` matching the current system timezone. + ### `chrono-tz` Adds a dependency on [chrono-tz](https://docs.rs/chrono-tz). Enables conversion from and to [`Tz`](https://docs.rs/chrono-tz/latest/chrono_tz/enum.Tz.html). -It requires at least Python 3.9. ### `either` -Adds a dependency on [either](https://docs.rs/either). Enables a conversions into [either](https://docs.rs/either)’s [`Either`](https://docs.rs/either/latest/either/struct.Report.html) type. +Adds a dependency on [either](https://docs.rs/either). +Enables a conversions into [either](https://docs.rs/either)’s [`Either`](https://docs.rs/either/latest/either/enum.Either.html) type. ### `eyre` -Adds a dependency on [eyre](https://docs.rs/eyre). Enables a conversion from [eyre](https://docs.rs/eyre)’s [`Report`](https://docs.rs/eyre/latest/eyre/struct.Report.html) type to [`PyErr`]({{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html), for easy error handling. +Adds a dependency on [eyre](https://docs.rs/eyre). +Enables a conversion from [eyre](https://docs.rs/eyre)’s [`Report`](https://docs.rs/eyre/latest/eyre/struct.Report.html) type to [`PyErr`]({{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html), for easy error handling. ### `hashbrown` @@ -137,24 +186,71 @@ Adds a dependency on [hashbrown](https://docs.rs/hashbrown) and enables conversi Adds a dependency on [indexmap](https://docs.rs/indexmap) and enables conversions into its [`IndexMap`](https://docs.rs/indexmap/latest/indexmap/map/struct.IndexMap.html) type. +### `jiff-02` + +Adds a dependency on [jiff@0.2](https://docs.rs/jiff/0.2). +Enables a conversion from [jiff](https://docs.rs/jiff)'s types to python: + +- [SignedDuration](https://docs.rs/jiff/0.2/jiff/struct.SignedDuration.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) +- [TimeZone](https://docs.rs/jiff/0.2/jiff/tz/struct.TimeZone.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html) +- [Offset](https://docs.rs/jiff/0.2/jiff/tz/struct.Offset.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html) +- [Date](https://docs.rs/jiff/0.2/jiff/civil/struct.Date.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html) +- [Time](https://docs.rs/jiff/0.2/jiff/civil/struct.Time.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html) +- [DateTime](https://docs.rs/jiff/0.2/jiff/civil/struct.DateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [Zoned](https://docs.rs/jiff/0.2/jiff/struct.Zoned.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [Timestamp](https://docs.rs/jiff/0.2/jiff/struct.Timestamp.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [ISOWeekDate](https://docs.rs/jiff/0.2/jiff/civil/struct.ISOWeekDate.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html) + +### `lock_api` + +Adds a dependency on [lock_api](https://docs.rs/lock_api) and enables Pyo3's `MutexExt` trait for all mutexes that extend on [`lock_api::Mutex`](https://docs.rs/lock_api/latest/lock_api/struct.Mutex.html) and [`parking_lot::ReentrantMutex`](https://docs.rs/lock_api/latest/lock_api/struct.ReentrantMutex.html) (like `parking_lot` or `spin`). + ### `num-bigint` -Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conversions into its [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUInt.html) types. +Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conversions into its [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUint.html) types. ### `num-complex` Adds a dependency on [num-complex](https://docs.rs/num-complex) and enables conversions into its [`Complex`](https://docs.rs/num-complex/latest/num_complex/struct.Complex.html) type. +### `num-rational` + +Adds a dependency on [num-rational](https://docs.rs/num-rational) and enables conversions into its [`Ratio`](https://docs.rs/num-rational/latest/num_rational/struct.Ratio.html) type. + +### `ordered-float` + +Adds a dependency on [ordered-float](https://docs.rs/ordered-float) and enables conversions between [ordered-float](https://docs.rs/ordered-float)'s types and Python: + +- [NotNan](https://docs.rs/ordered-float/latest/ordered_float/struct.NotNan.html) -> [`PyFloat`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFloat.html) +- [OrderedFloat](https://docs.rs/ordered-float/latest/ordered_float/struct.OrderedFloat.html) -> [`PyFloat`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFloat.html) + +### `parking-lot` + +Adds a dependency on [parking_lot](https://docs.rs/parking_lot) and enables Pyo3's `OnceExt` & `MutexExt` traits for [`parking_lot::Once`](https://docs.rs/parking_lot/latest/parking_lot/struct.Once.html) [`parking_lot::Mutex`](https://docs.rs/parking_lot/latest/parking_lot/type.Mutex.html) and [`parking_lot::ReentrantMutex`](https://docs.rs/parking_lot/latest/parking_lot/type.ReentrantMutex.html) types. + ### `rust_decimal` Adds a dependency on [rust_decimal](https://docs.rs/rust_decimal) and enables conversions into its [`Decimal`](https://docs.rs/rust_decimal/latest/rust_decimal/struct.Decimal.html) type. +### `time` + +Adds a dependency on [time](https://docs.rs/time). +Enables conversions between [time](https://docs.rs/time)'s types and Python: + +- [Date](https://docs.rs/time/0.3.38/time/struct.Date.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html) +- [Time](https://docs.rs/time/0.3.38/time/struct.Time.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html) +- [OffsetDateTime](https://docs.rs/time/0.3.38/time/struct.OffsetDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [PrimitiveDateTime](https://docs.rs/time/0.3.38/time/struct.PrimitiveDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [Duration](https://docs.rs/time/0.3.38/time/struct.Duration.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) +- [UtcOffset](https://docs.rs/time/0.3.38/time/struct.UtcOffset.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html) +- [UtcDateTime](https://docs.rs/time/0.3.38/time/struct.UtcDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) + ### `serde` Enables (de)serialization of `Py` objects via [serde](https://serde.rs/). This allows to use [`#[derive(Serialize, Deserialize)`](https://serde.rs/derive.html) on structs that hold references to `#[pyclass]` instances -```rust +```rust,no_run # #[cfg(feature = "serde")] # #[allow(dead_code)] # mod serde_only { @@ -179,3 +275,9 @@ struct User { ### `smallvec` Adds a dependency on [smallvec](https://docs.rs/smallvec) and enables conversions into its [`SmallVec`](https://docs.rs/smallvec/latest/smallvec/struct.SmallVec.html) type. + +[set-configuration-options]: https://doc.rust-lang.org/reference/conditional-compilation.html#set-configuration-options + +### `uuid` + +Adds a dependency on [uuid](https://docs.rs/uuid) and enables conversions into its [`Uuid`](https://docs.rs/uuid/latest/uuid/struct.Uuid.html) type. diff --git a/guide/src/free-threading.md b/guide/src/free-threading.md new file mode 100644 index 00000000000..5222ff2202b --- /dev/null +++ b/guide/src/free-threading.md @@ -0,0 +1,316 @@ +# Supporting Free-Threaded CPython + +CPython 3.14 declared support for the "free-threaded" build of CPython that does not rely on the [global interpreter lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) (often referred to as the GIL) for thread safety. +Since version 0.23, PyO3 supports building Rust extensions for the free-threaded Python build and calling into free-threaded Python from Rust. + +If you want more background on free-threaded Python in general, see the +[what's new](https://docs.python.org/3/whatsnew/3.13.html#whatsnew313-free-threaded-cpython) +entry in the 3.13 release notes (when the "free-threaded" build was first added as an experimental +mode), the +[free-threading HOWTO guide](https://docs.python.org/3/howto/free-threading-extensions.html#freethreading-extensions-howto) +in the CPython docs, the +[extension porting guide](https://py-free-threading.github.io/porting-extensions/) +in the community-maintained Python free-threading guide, and +[PEP 703](https://peps.python.org/pep-0703/), which provides the technical background +for the free-threading implementation in CPython. + +In the GIL-enabled build (the only choice before the "free-threaded" build was introduced), the global interpreter lock serializes access to the Python runtime. +The GIL is therefore a fundamental limitation to parallel scaling of multithreaded Python workflows, due to [Amdahl's law](https://en.wikipedia.org/wiki/Amdahl%27s_law), because any time spent executing a parallel processing task on only one execution context fundamentally cannot be sped up using parallelism. + +The free-threaded build removes this limit on multithreaded Python scaling. +This means it's much more straightforward to achieve parallelism using the Python [`threading`] module. +If you have ever needed to use [`multiprocessing`](https://docs.python.org/3/library/multiprocessing.html) to achieve a parallel speedup for some Python code, free-threading will likely allow the use of Python threads instead for the same workflow. + +PyO3's support for free-threaded Python will enable authoring native Python extensions that are thread-safe by construction, with much stronger safety guarantees than C extensions. +Our goal is to enable ["fearless concurrency"](https://doc.rust-lang.org/book/ch16-00-concurrency.html) in the native Python runtime by building on the Rust [`Send` and `Sync`](https://doc.rust-lang.org/nomicon/send-and-sync.html) traits. + +This document provides advice for porting Rust code using PyO3 to run under +free-threaded Python. + +## Supporting free-threaded Python with PyO3 + +Many simple uses of PyO3, like exposing bindings for a "pure" Rust function with no side-effects or defining an immutable Python class, will likely work "out of the box" on the free-threaded build. +All that will be necessary is to annotate Python modules declared by rust code in your project to declare that they support free-threaded Python, for example by declaring the module with `#[pymodule(gil_used = false)]`. + +More complicated `#[pyclass]` types may need to deal with thread-safety directly; there is [a dedicated section of the guide](./class/thread-safety.md) to discuss this. + +At a low-level, annotating a module sets the `Py_MOD_GIL` slot on modules defined by an extension to `Py_MOD_GIL_NOT_USED`, which allows the interpreter to see at runtime that the author of the extension thinks the extension is thread-safe. +You should only do this if you know that your extension is thread-safe. +Because of Rust's guarantees, this is already true for many extensions, however see below for more discussion about how to evaluate the thread safety of existing Rust extensions and how to think about the PyO3 API using a Python runtime with no GIL. + +If you do not explicitly mark that modules are thread-safe, the Python interpreter will re-enable the GIL at runtime while importing your module and print a `RuntimeWarning` with a message containing the name of the module causing it to re-enable the GIL. +You can force the GIL to remain disabled by setting the `PYTHON_GIL=0` as an environment variable or passing `-Xgil=0` when starting Python (`0` means the GIL is turned off). + +If you are sure that all data structures exposed in a `PyModule` are +thread-safe, then pass `gil_used = false` as a parameter to the +`pymodule` procedural macro declaring the module or call +`PyModule::gil_used` on a `PyModule` instance. For example: + +```rust,no_run +use pyo3::prelude::*; + +/// This module supports free-threaded Python +#[pymodule(gil_used = false)] +fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { + // add members to the module that you know are thread-safe + Ok(()) +} +``` + +Or for a module that is set up without using the `pymodule` macro: + +```rust,no_run +use pyo3::prelude::*; + +# #[allow(dead_code)] +fn register_child_module(parent_module: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(parent_module.py(), "child_module")?; + child_module.gil_used(false)?; + parent_module.add_submodule(&child_module) +} + +``` + +For now you must explicitly opt in to free-threading support by annotating modules defined in your extension. +In a future version of `PyO3`, we plan to make `gil_used = false` the default. + +See the +[`string-sum`](https://github.com/PyO3/pyo3/tree/main/pyo3-ffi/examples/string-sum) +example for how to declare free-threaded support using raw FFI calls for modules +using single-phase initialization and the +[`sequential`](https://github.com/PyO3/pyo3/tree/main/pyo3-ffi/examples/sequential) +example for modules using multi-phase initialization. + +If you would like to use conditional compilation to trigger different code paths under the free-threaded build, you can use the `Py_GIL_DISABLED` attribute once you have configured your crate to generate the necessary build configuration data. +See [the guide section](./building-and-distribution/multiple-python-versions.md) for more details about supporting multiple different Python versions, including the free-threaded build. + +## Special considerations for the free-threaded build + +The free-threaded interpreter does not have a GIL. +Many existing extensions providing mutable data structures relied on the GIL to lock Python objects and make interior mutability thread-safe. + +Calling into the CPython C API is only legal when an OS thread is explicitly attached to the interpreter runtime. +In the GIL-enabled build, this happens when the GIL is acquired. +In the free-threaded build there is no GIL, but the same C macros that release or acquire the GIL in the GIL-enabled build instead ask the interpreter to attach the thread to the Python runtime, and there can be many threads simultaneously attached. +See [PEP 703](https://peps.python.org/pep-0703/#thread-states) for more background about how threads can be attached and detached from the interpreter runtime, in a manner analogous to releasing and acquiring the GIL in the GIL-enabled build. + +In the GIL-enabled build, PyO3 uses the [`Python<'py>`] type and the `'py` lifetime to signify that the global interpreter lock is held. +In the freethreaded build, holding a `'py` lifetime means only that the thread is currently attached to the Python interpreter -- other threads can be simultaneously interacting with the interpreter. + +### Attaching to the runtime + +You still need to obtain a `'py` lifetime to interact with Python objects or call into the CPython C API. +If you are not yet attached to the Python runtime, you can register a thread using the [`Python::attach`] function. +Threads created via the Python [`threading`] module do not need to do this, and pyo3 will handle setting up the [`Python<'py>`] token when CPython calls into your extension. + +### Detaching to avoid hangs and deadlocks + +The free-threaded build triggers global synchronization events in the following +situations: + +- During garbage collection in order to get a globally consistent view of + reference counts and references between objects +- In Python 3.13, when the first background thread is started in + order to mark certain objects as immortal +- When either `sys.settrace` or `sys.setprofile` are called in order to + instrument running code objects and threads +- During a call to `os.fork()`, to ensure a process-wide consistent state. + +This is a non-exhaustive list and there may be other situations in future Python +versions that can trigger global synchronization events. + +This means that you should detach from the interpreter runtime using [`Python::detach`] in exactly the same situations as you should detach from the runtime in the GIL-enabled build: when doing long-running tasks that do not require the CPython runtime or when doing any task that needs to re-attach to the runtime (see the [guide section](parallelism.md#sharing-python-objects-between-rust-threads) that covers this). +In the former case, you would observe a hang on threads that are waiting on the long-running task to complete, and in the latter case you would see a deadlock while a thread tries to attach after the runtime triggers a global synchronization event, but the spawning thread prevents the synchronization event from completing. + +### Exceptions and panics for multithreaded access of mutable `pyclass` instances + +Data attached to `pyclass` instances is protected from concurrent access by a `RefCell`-like pattern of runtime borrow checking. +Like a `RefCell`, PyO3 will raise exceptions (or in some cases panic) to enforce exclusive access for mutable borrows. +It was always possible to generate panics like this in PyO3 in code that releases the GIL with [`Python::detach`] or calling a python method accepting `&self` from a `&mut self` (see [the docs on interior mutability](./class.md#bound-and-interior-mutability),) but now in free-threaded Python there are more opportunities to trigger these panics from Python because there is no GIL to lock concurrent access to mutably borrowed data from Python. + +The most straightforward way to trigger this problem is to use the Python [`threading`] module to simultaneously call a rust function that mutably borrows a [`pyclass`]({{#PYO3_DOCS_URL}}/pyo3/attr.pyclass.html) in multiple threads. +For example, consider the following implementation: + +```rust,no_run +# use pyo3::prelude::*; +#[pyclass] +#[derive(Default)] +struct ThreadIter { + count: usize, +} + +#[pymethods] +impl ThreadIter { + #[new] + pub fn new() -> Self { + Default::default() + } + + fn __next__(&mut self, py: Python<'_>) -> usize { + self.count += 1; + self.count + } +} +``` + +And then if we do something like this in Python: + +```python +import concurrent.futures +from my_module import ThreadIter + +i = ThreadIter() + +def increment(): + next(i) + +with concurrent.futures.ThreadPoolExecutor(max_workers=16) as tpe: + futures = [tpe.submit(increment) for _ in range(100)] + [f.result() for f in futures] +``` + +We will see an exception: + +```text +Traceback (most recent call last) + File "example.py", line 5, in + next(i) +RuntimeError: Already borrowed +``` + +We may allow user-selectable semantics for mutable pyclass definitions in a future version of PyO3, allowing some form of opt-in locking to emulate the GIL if that is needed. +For now you should explicitly add locking, possibly using conditional compilation or using the critical section API, to avoid creating deadlocks with the GIL. + +### Cannot build extensions using the limited API + +The free-threaded build uses a completely new ABI and there is not yet an equivalent to the limited API for the free-threaded ABI. +That means if your crate depends on PyO3 using the `abi3` feature or an an `abi3-pyxx` feature, PyO3 will print a warning and ignore that setting when building extensions using the free-threaded interpreter. + +This means that if your package makes use of the ABI forward compatibility +provided by the limited API to upload only one wheel for each release of your +package, you will need to update your release procedure to also upload a +version-specific free-threaded wheel. + +See [the guide section](./building-and-distribution/multiple-python-versions.md) +for more details about supporting multiple different Python versions, including +the free-threaded build. + +### Thread-safe single initialization + +To initialize data exactly once, use the [`PyOnceLock`] type, which is a close equivalent to [`std::sync::OnceLock`][`OnceLock`] that also helps avoid deadlocks by detaching from the Python interpreter when threads are blocking waiting for another thread to complete initialization. +If already using [`OnceLock`] and it is impractical to replace with a [`PyOnceLock`], there is the [`OnceLockExt`] extension trait which adds [`OnceLockExt::get_or_init_py_attached`] to detach from the interpreter when blocking in the same fashion as [`PyOnceLock`]. +Here is an example using [`PyOnceLock`] to single-initialize a runtime cache holding a `Py`: + +```rust +# use pyo3::prelude::*; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyDict; + +let cache: PyOnceLock> = PyOnceLock::new(); + +Python::attach(|py| { + // guaranteed to be called once and only once + cache.get_or_init(py, || PyDict::new(py).unbind()) +}); +``` + +In cases where a function must run exactly once, you can bring the [`OnceExt`] trait into scope. +The [`OnceExt`] trait adds [`OnceExt::call_once_py_attached`] and [`OnceExt::call_once_force_py_attached`] functions to the api of `std::sync::Once`, enabling use of [`Once`] in contexts where the thread is attached to the Python interpreter. +These functions are analogous to [`Once::call_once`], [`Once::call_once_force`] except they accept a [`Python<'py>`] token in addition to an `FnOnce`. +All of these functions detach from the interpreter before blocking and re-attach before executing the function, avoiding deadlocks that are possible without using the PyO3 extension traits. +Here the same example as above built using a [`Once`] instead of a +[`PyOnceLock`]: + +```rust +# use pyo3::prelude::*; +use std::sync::Once; +use pyo3::sync::OnceExt; +use pyo3::types::PyDict; + +struct RuntimeCache { + once: Once, + cache: Option> +} + +let mut cache = RuntimeCache { + once: Once::new(), + cache: None +}; + +Python::attach(|py| { + // guaranteed to be called once and only once + cache.once.call_once_py_attached(py, || { + cache.cache = Some(PyDict::new(py).unbind()); + }); +}); +``` + +### `GILProtected` is not exposed + +[`GILProtected`] is a (deprecated) PyO3 type that allows mutable access to static data by leveraging the GIL to lock concurrent access from other threads. +In free-threaded Python there is no GIL, so you will need to replace this type with some other form of locking. +In many cases, a type from [`std::sync::atomic`](https://doc.rust-lang.org/std/sync/atomic/) or a [`std::sync::Mutex`](https://doc.rust-lang.org/std/sync/struct.Mutex.html) will be sufficient. + +Before: + +```rust +# #![allow(deprecated)] +# fn main() { +# #[cfg(not(Py_GIL_DISABLED))] { +# use pyo3::prelude::*; +use pyo3::sync::GILProtected; +use pyo3::types::{PyDict, PyNone}; +use std::cell::RefCell; + +static OBJECTS: GILProtected>>> = + GILProtected::new(RefCell::new(Vec::new())); + +Python::attach(|py| { + // stand-in for something that executes arbitrary Python code + let d = PyDict::new(py); + d.set_item(PyNone::get(py), PyNone::get(py)).unwrap(); + OBJECTS.get(py).borrow_mut().push(d.unbind()); +}); +# }} +``` + +After: + +```rust +# use pyo3::prelude::*; +# fn main() { +use pyo3::types::{PyDict, PyNone}; +use std::sync::Mutex; + +static OBJECTS: Mutex>> = Mutex::new(Vec::new()); + +Python::attach(|py| { + // stand-in for something that executes arbitrary Python code + let d = PyDict::new(py); + d.set_item(PyNone::get(py), PyNone::get(py)).unwrap(); + // as with any `Mutex` usage, lock the mutex for as little time as possible + // in this case, we do it just while pushing into the `Vec` + OBJECTS.lock().unwrap().push(d.unbind()); +}); +# } +``` + +If you are executing arbitrary Python code while holding the lock, then you should import the [`MutexExt`] trait and use the `lock_py_attached` method instead of `lock`. +This ensures that global synchronization events started by the Python runtime can proceed, avoiding possible deadlocks with the interpreter. + +[`GILProtected`]: https://docs.rs/pyo3/0.22/pyo3/sync/struct.GILProtected.html +[`MutexExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.MutexExt.html +[`Once`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html +[`Once::call_once`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html#method.call_once +[`Once::call_once_force`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html#method.call_once_force +[`OnceExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html +[`OnceExt::call_once_py_attached`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html#tymethod.call_once_py_attached +[`OnceExt::call_once_force_py_attached`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html#tymethod.call_once_force_py_attached +[`OnceLockExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceLockExt.html +[`OnceLockExt::get_or_init_py_attached`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceLockExt.html#tymethod.get_or_init_py_attached +[`OnceLock`]: https://doc.rust-lang.org/stable/std/sync/struct.OnceLock.html +[`Python::detach`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.detach +[`Python::attach`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.attach +[`Python<'py>`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html +[`PyOnceLock`]: {{#PYO3_DOCS_URL}}/pyo3/sync/struct.PyOnceLock.html +[`threading`]: https://docs.python.org/3/library/threading.html diff --git a/guide/src/function-calls.md b/guide/src/function-calls.md new file mode 100644 index 00000000000..e5c9c1ed9b3 --- /dev/null +++ b/guide/src/function-calls.md @@ -0,0 +1 @@ +# Calling Python functions diff --git a/guide/src/function.md b/guide/src/function.md index f3955ba554b..3eb13058c57 100644 --- a/guide/src/function.md +++ b/guide/src/function.md @@ -1,44 +1,45 @@ # Python functions -The `#[pyfunction]` attribute is used to define a Python function from a Rust function. Once defined, the function needs to be added to a [module](./module.md) using the `wrap_pyfunction!` macro. +The `#[pyfunction]` attribute is used to define a Python function from a Rust function. +Once defined, the function needs to be added to a [module](./module.md). The following example defines a function called `double` in a Python module called `my_extension`: -```rust -use pyo3::prelude::*; - -#[pyfunction] -fn double(x: usize) -> usize { - x * 2 -} +```rust,no_run +#[pyo3::pymodule] +mod my_extension { + use pyo3::prelude::*; -#[pymodule] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(double, m)?)?; - Ok(()) + #[pyfunction] + fn double(x: usize) -> usize { + x * 2 + } } ``` -This chapter of the guide explains full usage of the `#[pyfunction]` attribute. In this first section, the following topics are covered: +This chapter of the guide explains full usage of the `#[pyfunction]` attribute. +In this first section, the following topics are covered: - [Function options](#function-options) - [`#[pyo3(name = "...")]`](#name) - [`#[pyo3(signature = (...))]`](#signature) - [`#[pyo3(text_signature = "...")]`](#text_signature) - [`#[pyo3(pass_module)]`](#pass_module) + - [`#[pyo3(warn(message = "...", category = ...))]`](#warn) - [Per-argument options](#per-argument-options) - [Advanced function patterns](#advanced-function-patterns) -- [`#[pyfn]` shorthand](#pyfn-shorthand) There are also additional sections on the following topics: - [Function Signatures](./function/signature.md) +- [Error Handling](./function/error-handling.md) ## Function options -The `#[pyo3]` attribute can be used to modify properties of the generated Python function. It can take any combination of the following options: +The `#[pyo3]` attribute can be used to modify properties of the generated Python function. +It can take any combination of the following options: - - `#[pyo3(name = "...")]` +- `#[pyo3(name = "...")]` Overrides the name exposed to Python. @@ -46,81 +47,184 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python `module_with_functions` as the Python function `no_args`: ```rust - use pyo3::prelude::*; - - #[pyfunction] - #[pyo3(name = "no_args")] - fn no_args_py() -> usize { - 42 + # use pyo3::prelude::*; + #[pyo3::pymodule] + mod module_with_functions { + use pyo3::prelude::*; + + #[pyfunction] + #[pyo3(name = "no_args")] + fn no_args_py() -> usize { + 42 + } } - #[pymodule] - fn module_with_functions(py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(no_args_py, m)?)?; - Ok(()) - } - - # Python::with_gil(|py| { + # Python::attach(|py| { # let m = pyo3::wrap_pymodule!(module_with_functions)(py); # assert!(m.getattr(py, "no_args").is_ok()); # assert!(m.getattr(py, "no_args_py").is_err()); # }); ``` - - `#[pyo3(signature = (...))]` +- `#[pyo3(signature = (...))]` Defines the function signature in Python. See [Function Signatures](./function/signature.md). - - `#[pyo3(text_signature = "...")]` +- `#[pyo3(text_signature = "...")]` Overrides the PyO3-generated function signature visible in Python tooling (such as via [`inspect.signature`]). See the [corresponding topic in the Function Signatures subchapter](./function/signature.md#making-the-function-signature-available-to-python). - - `#[pyo3(pass_module)]` +- `#[pyo3(pass_module)]` - Set this option to make PyO3 pass the containing module as the first argument to the function. It is then possible to use the module in the function body. The first argument **must** be of type `&PyModule`. + Set this option to make PyO3 pass the containing module as the first argument to the function. It is then possible to use the module in the function body. The first argument **must** be of type `&Bound<'_, PyModule>`, `Bound<'_, PyModule>`, or `Py`. The following example creates a function `pyfunction_with_module` which returns the containing module's name (i.e. `module_with_fn`): + ```rust,no_run + #[pyo3::pymodule] + mod module_with_fn { + use pyo3::prelude::*; + use pyo3::types::PyString; + + #[pyfunction] + #[pyo3(pass_module)] + fn pyfunction_with_module<'py>( + module: &Bound<'py, PyModule>, + ) -> PyResult> { + module.name() + } + } + ``` + +- `#[pyo3(warn(message = "...", category = ...))]` + + This option is used to display a warning when the function is used in Python. It is equivalent to [`warnings.warn(message, category)`](https://docs.python.org/3/library/warnings.html#warnings.warn). + The `message` parameter is a string that will be displayed when the function is called, and the `category` parameter is optional and has to be a subclass of [`Warning`](https://docs.python.org/3/library/exceptions.html#Warning). + When the `category` parameter is not provided, the warning will be defaulted to [`UserWarning`](https://docs.python.org/3/library/exceptions.html#UserWarning). + + > Note: when used with `#[pymethods]`, this attribute does not work with `#[classattr]` nor `__traverse__` magic method. + + The following are examples of using the `#[pyo3(warn)]` attribute: + ```rust use pyo3::prelude::*; - use pyo3::types::PyString; - - #[pyfunction] - #[pyo3(pass_module)] - fn pyfunction_with_module<'py>(module: &Bound<'py, PyModule>) -> PyResult> { - module.name() - } #[pymodule] - fn module_with_fn(py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(pyfunction_with_module, m)?) + mod raising_warning_fn { + use pyo3::prelude::pyfunction; + use pyo3::exceptions::PyFutureWarning; + + #[pyfunction] + #[pyo3(warn(message = "This is a warning message"))] + fn function_with_warning() -> usize { + 42 + } + + #[pyfunction] + #[pyo3(warn(message = "This function is warning with FutureWarning", category = PyFutureWarning))] + fn function_with_warning_and_custom_category() -> usize { + 42 + } } + + # use pyo3::exceptions::{PyFutureWarning, PyUserWarning}; + # use pyo3::types::{IntoPyDict, PyList}; + # use pyo3::PyTypeInfo; + # + # fn catch_warning(py: Python<'_>, f: impl FnOnce(&Bound<'_, PyList>) -> ()) -> PyResult<()> { + # let warnings = py.import("warnings")?; + # let kwargs = [("record", true)].into_py_dict(py)?; + # let catch_warnings = warnings + # .getattr("catch_warnings")? + # .call((), Some(&kwargs))?; + # let list = catch_warnings.call_method0("__enter__")?.cast_into()?; + # warnings.getattr("simplefilter")?.call1(("always",))?; // show all warnings + # f(&list); + # catch_warnings + # .call_method1("__exit__", (py.None(), py.None(), py.None())) + # .unwrap(); + # Ok(()) + # } + # + # macro_rules! assert_warnings { + # ($py:expr, $body:expr, [$(($category:ty, $message:literal)),+] $(,)? ) => { + # catch_warning($py, |list| { + # $body; + # let expected_warnings = [$((<$category as PyTypeInfo>::type_object($py), $message)),+]; + # assert_eq!(list.len(), expected_warnings.len()); + # for (warning, (category, message)) in list.iter().zip(expected_warnings) { + # assert!(warning.getattr("category").unwrap().is(&category)); + # assert_eq!( + # warning.getattr("message").unwrap().str().unwrap().to_string_lossy(), + # message + # ); + # } + # }).unwrap(); + # }; + # } + # + # Python::attach(|py| { + # assert_warnings!( + # py, + # { + # let m = pyo3::wrap_pymodule!(raising_warning_fn)(py); + # let f1 = m.getattr(py, "function_with_warning").unwrap(); + # let f2 = m.getattr(py, "function_with_warning_and_custom_category").unwrap(); + # f1.call0(py).unwrap(); + # f2.call0(py).unwrap(); + # }, + # [ + # (PyUserWarning, "This is a warning message"), + # ( + # PyFutureWarning, + # "This function is warning with FutureWarning" + # ) + # ] + # ); + # }); + ``` + + When the functions are called as the following, warnings will be displayed. + + ```python + import warnings + from raising_warning_fn import function_with_warning, function_with_warning_and_custom_category + + function_with_warning() + function_with_warning_and_custom_category() + ``` + + The warning output will be: + + ```plaintext + UserWarning: This is a warning message + FutureWarning: This function is warning with FutureWarning ``` ## Per-argument options -The `#[pyo3]` attribute can be used on individual arguments to modify properties of them in the generated function. It can take any combination of the following options: +The `#[pyo3]` attribute can be used on individual arguments to modify properties of them in the generated function. +It can take any combination of the following options: - - `#[pyo3(from_py_with = "...")]` +- `#[pyo3(from_py_with = ...)]` - Set this on an option to specify a custom function to convert the function argument from Python to the desired Rust type, instead of using the default `FromPyObject` extraction. The function signature must be `fn(&PyAny) -> PyResult` where `T` is the Rust type of the argument. + Set this on an option to specify a custom function to convert the function argument from Python to the desired Rust type, instead of using the default `FromPyObject` extraction. The function signature must be `fn(&Bound<'_, PyAny>) -> PyResult` where `T` is the Rust type of the argument. The following example uses `from_py_with` to convert the input Python object to its length: ```rust use pyo3::prelude::*; - fn get_length(obj: &PyAny) -> PyResult { - let length = obj.len()?; - Ok(length) + fn get_length(obj: &Bound<'_, PyAny>) -> PyResult { + obj.len() } #[pyfunction] - fn object_length(#[pyo3(from_py_with = "get_length")] argument: usize) -> usize { + fn object_length(#[pyo3(from_py_with = get_length)] argument: usize) -> usize { argument } - # Python::with_gil(|py| { + # Python::attach(|py| { # let f = pyo3::wrap_pyfunction!(object_length)(py).unwrap(); # assert_eq!(f.call1((vec![1, 2, 3],)).unwrap().extract::().unwrap(), 3); # }); @@ -134,11 +238,9 @@ You can pass Python `def`'d functions and built-in functions to Rust functions [ corresponds to regular Python functions while [`PyCFunction`] describes built-ins such as `repr()`. -You can also use [`PyAny::is_callable`] to check if you have a callable object. `is_callable` will -return `true` for functions (including lambdas), methods and objects with a `__call__` method. -You can call the object with [`PyAny::call`] with the args as first parameter and the kwargs -(or `None`) as second parameter. There are also [`PyAny::call0`] with no args and [`PyAny::call1`] -with only positional args. +You can also use [`Bound<'_, PyAny>::is_callable`] to check if you have a callable object. `is_callable` will return `true` for functions (including lambdas), methods and objects with a `__call__` method. +You can call the object with [`Bound<'_, PyAny>::call`] with the args as first parameter and the kwargs (or `None`) as second parameter. +There are also [`Bound<'_, PyAny>::call0`] with no args and [`Bound<'_, PyAny>::call1`] with only positional args. ### Calling Rust functions in Python @@ -149,63 +251,24 @@ The ways to convert a Rust function into a Python object vary depending on the f - use a `#[pyclass]` struct which stores the function as a field and implement `__call__` to call the stored function. - use `PyCFunction::new_closure` to create an object directly from the function. -[`PyAny::is_callable`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#tymethod.is_callable -[`PyAny::call`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#tymethod.call -[`PyAny::call0`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#tymethod.call0 -[`PyAny::call1`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#tymethod.call1 -[`PyObject`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyObject.html -[`wrap_pyfunction!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.wrap_pyfunction.html -[`PyFunction`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFunction.html -[`PyCFunction`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyCFunction.html - ### Accessing the FFI functions -In order to make Rust functions callable from Python, PyO3 generates an `extern "C"` -function whose exact signature depends on the Rust signature. (PyO3 chooses the optimal -Python argument passing convention.) It then embeds the call to the Rust function inside this -FFI-wrapper function. This wrapper handles extraction of the regular arguments and the keyword -arguments from the input `PyObject`s. - -The `wrap_pyfunction` macro can be used to directly get a `PyCFunction` given a -`#[pyfunction]` and a `PyModule`: `wrap_pyfunction!(rust_fun, module)`. - -## `#[pyfn]` shorthand - -There is a shorthand to `#[pyfunction]` and `wrap_pymodule!`: the function can be placed inside the module definition and -annotated with `#[pyfn]`. To simplify PyO3, it is expected that `#[pyfn]` may be removed in a future release (See [#694](https://github.com/PyO3/pyo3/issues/694)). - -An example of `#[pyfn]` is below: - -```rust -use pyo3::prelude::*; - -#[pymodule] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { - #[pyfn(m)] - fn double(x: usize) -> usize { - x * 2 - } - - Ok(()) -} -``` - -`#[pyfn(m)]` is just syntactic sugar for `#[pyfunction]`, and takes all the same options -documented in the rest of this chapter. The code above is expanded to the following: +In order to make Rust functions callable from Python, PyO3 generates an `extern "C"` function whose exact signature depends on the Rust signature. (PyO3 chooses the optimal Python argument passing convention.) It then embeds the call to the Rust function inside this FFI-wrapper function. +This wrapper handles extraction of the regular arguments and the keyword arguments from the input `PyObject`s. -```rust -use pyo3::prelude::*; - -#[pymodule] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { - #[pyfunction] - fn double(x: usize) -> usize { - x * 2 - } - - m.add_function(wrap_pyfunction!(double, m)?)?; - Ok(()) -} -``` +The `wrap_pyfunction` macro can be used to directly get a `Bound` given a +`#[pyfunction]` and a `Bound`: `wrap_pyfunction!(rust_fun, module)`. + +[`Bound<'_, PyAny>::is_callable`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.is_callable + +[`Bound<'_, PyAny>::call`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.call + +[`Bound<'_, PyAny>::call0`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.call0 + +[`Bound<'_, PyAny>::call1`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.call1 +[`wrap_pyfunction!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.wrap_pyfunction.html +[`PyFunction`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFunction.html +[`PyCFunction`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyCFunction.html + [`inspect.signature`]: https://docs.python.org/3/library/inspect.html#inspect.signature diff --git a/guide/src/function/error_handling.md b/guide/src/function/error-handling.md similarity index 78% rename from guide/src/function/error_handling.md rename to guide/src/function/error-handling.md index b0f63885cdf..5db344bb6c1 100644 --- a/guide/src/function/error_handling.md +++ b/guide/src/function/error-handling.md @@ -8,11 +8,14 @@ There is a later section of the guide on [Python exceptions](../exception.md) wh ## Representing Python exceptions -Rust code uses the generic [`Result`] enum to propagate errors. The error type `E` is chosen by the code author to describe the possible errors which can happen. +Rust code uses the generic [`Result`] enum to propagate errors. +The error type `E` is chosen by the code author to describe the possible errors which can happen. -PyO3 has the [`PyErr`] type which represents a Python exception. If a PyO3 API could result in a Python exception being raised, the return type of that `API` will be [`PyResult`], which is an alias for the type `Result`. +PyO3 has the [`PyErr`] type which represents a Python exception. +If a PyO3 API could result in a Python exception being raised, the return type of that `API` will be [`PyResult`], which is an alias for the type `Result`. In summary: + - When Python exceptions are raised and caught by PyO3, the exception will be stored in the `Err` variant of the `PyResult`. - Passing Python exceptions through Rust code then uses all the "normal" techniques such as the `?` operator, with `PyErr` as the error type. - Finally, when a `PyResult` crosses from Rust back to Python via PyO3, if the result is an `Err` variant the contained exception will be raised. @@ -23,7 +26,8 @@ In summary: As indicated in the previous section, when a `PyResult` containing an `Err` crosses from Rust to Python, PyO3 will raise the exception contained within. -Accordingly, to raise an exception from a `#[pyfunction]`, change the return type `T` to `PyResult`. When the function returns an `Err` it will raise a Python exception. (Other `Result` types can be used as long as the error `E` has a `From` conversion for `PyErr`, see [implementing a conversion](#implementing-an-error-conversion) below.) +Accordingly, to raise an exception from a `#[pyfunction]`, change the return type `T` to `PyResult`. +When the function returns an `Err` it will raise a Python exception. (Other `Result` types can be used as long as the error `E` has a `From` conversion for `PyErr`, see [custom Rust error types](#custom-rust-error-types) below.) This also works for functions in `#[pymethods]`. @@ -43,7 +47,7 @@ fn check_positive(x: i32) -> PyResult<()> { } # # fn main(){ -# Python::with_gil(|py|{ +# Python::attach(|py|{ # let fun = pyo3::wrap_pyfunction!(check_positive, py).unwrap(); # fun.call1((-1,)).unwrap_err(); # fun.call1((1,)).unwrap(); @@ -51,11 +55,13 @@ fn check_positive(x: i32) -> PyResult<()> { # } ``` -All built-in Python exception types are defined in the [`pyo3::exceptions`] module. They have a `new_err` constructor to directly build a `PyErr`, as seen in the example above. +All built-in Python exception types are defined in the [`pyo3::exceptions`] module. +They have a `new_err` constructor to directly build a `PyErr`, as seen in the example above. ## Custom Rust error types -PyO3 will automatically convert a `Result` returned by a `#[pyfunction]` into a `PyResult` as long as there is an implementation of `std::from::From for PyErr`. Many error types in the Rust standard library have a [`From`] conversion defined in this way. +PyO3 will automatically convert a `Result` returned by a `#[pyfunction]` into a `PyResult` as long as there is an implementation of `std::from::From for PyErr`. +Many error types in the Rust standard library have a [`From`] conversion defined in this way. If the type `E` you are handling is defined in a third-party crate, see the section on [foreign rust error types](#foreign-rust-error-types) below for ways to work with this error. @@ -71,7 +77,7 @@ fn parse_int(x: &str) -> Result { } # fn main() { -# Python::with_gil(|py| { +# Python::attach(|py| { # let fun = pyo3::wrap_pyfunction!(parse_int, py).unwrap(); # let value: usize = fun.call1(("5",)).unwrap().extract().unwrap(); # assert_eq!(value, 5); @@ -88,7 +94,8 @@ Traceback (most recent call last): ValueError: invalid digit found in string ``` -As a more complete example, the following snippet defines a Rust error named `CustomIOError`. It then defines a `From for PyErr`, which returns a `PyErr` representing Python's `OSError`. +As a more complete example, the following snippet defines a Rust error named `CustomIOError`. +It then defines a `From for PyErr`, which returns a `PyErr` representing Python's `OSError`. Therefore, it can use this error in the result of a `#[pyfunction]` directly, relying on the conversion if it has to be propagated into a Python exception. ```rust @@ -131,7 +138,7 @@ fn connect(s: String) -> Result<(), CustomIOError> { } fn main() { - Python::with_gil(|py| { + Python::attach(|py| { let fun = pyo3::wrap_pyfunction!(connect, py).unwrap(); let err = fun.call1(("0.0.0.0",)).unwrap_err(); assert!(err.is_instance_of::(py)); @@ -139,13 +146,11 @@ fn main() { } ``` -If lazy construction of the Python exception instance is desired, the -[`PyErrArguments`]({{#PYO3_DOCS_URL}}/pyo3/trait.PyErrArguments.html) -trait can be implemented instead of `From`. In that case, actual exception argument creation is delayed -until the `PyErr` is needed. +If lazy construction of the Python exception instance is desired, the [`PyErrArguments`]({{#PYO3_DOCS_URL}}/pyo3/trait.PyErrArguments.html) trait can be implemented instead of `From`. +In that case, actual exception argument creation is delayed until the `PyErr` is needed. -A final note is that any errors `E` which have a `From` conversion can be used with the `?` -("try") operator with them. An alternative implementation of the above `parse_int` which instead returns `PyResult` is below: +A final note is that any errors `E` which have a `From` conversion can be used with the `?` ("try") operator with them. +An alternative implementation of the above `parse_int` which instead returns `PyResult` is below: ```rust use pyo3::prelude::*; @@ -158,7 +163,7 @@ fn parse_int(s: String) -> PyResult { # use pyo3::exceptions::PyValueError; # # fn main() { -# Python::with_gil(|py| { +# Python::attach(|py| { # assert_eq!(parse_int(String::from("1")).unwrap(), 1); # assert_eq!(parse_int(String::from("1337")).unwrap(), 1337); # @@ -181,10 +186,13 @@ The Rust compiler will not permit implementation of traits for types outside of Given a type `OtherError` which is defined in third-party code, there are two main strategies available to integrate it with PyO3: -- Create a newtype wrapper, e.g. `MyOtherError`. Then implement `From for PyErr` (or `PyErrArguments`), as well as `From` for `MyOtherError`. -- Use Rust's Result combinators such as `map_err` to write code freely to convert `OtherError` into whatever is needed. This requires boilerplate at every usage however gives unlimited flexibility. +- Create a newtype wrapper, e.g. `MyOtherError`. + Then implement `From for PyErr` (or `PyErrArguments`), as well as `From` for `MyOtherError`. +- Use Rust's Result combinators such as `map_err` to write code freely to convert `OtherError` into whatever is needed. + This requires boilerplate at every usage however gives unlimited flexibility. -To detail the newtype strategy a little further, the key trick is to return `Result` from the `#[pyfunction]`. This means that PyO3 will make use of `From for PyErr` to create Python exceptions while the `#[pyfunction]` implementation can use `?` to convert `OtherError` to `MyOtherError` automatically. +To detail the newtype strategy a little further, the key trick is to return `Result` from the `#[pyfunction]`. +This means that PyO3 will make use of `From for PyErr` to create Python exceptions while the `#[pyfunction]` implementation can use `?` to convert `OtherError` to `MyOtherError` automatically. The following example demonstrates this for some imaginary third-party crate `some_crate` with a function `get_x` returning `Result`: @@ -223,7 +231,7 @@ fn wrapped_get_x() -> Result { } # fn main() { -# Python::with_gil(|py| { +# Python::attach(|py| { # let fun = pyo3::wrap_pyfunction!(wrapped_get_x, py).unwrap(); # let value: usize = fun.call0().unwrap().extract().unwrap(); # assert_eq!(value, 5); @@ -231,11 +239,13 @@ fn wrapped_get_x() -> Result { # } ``` +## Notes + +In Python 3.11 and up, notes can be added to Python exceptions to provide additional debugging information when printing the exception. +In PyO3, you can use the `add_note` method on `PyErr` to accomplish this functionality. [`From`]: https://doc.rust-lang.org/stable/std/convert/trait.From.html [`Result`]: https://doc.rust-lang.org/stable/std/result/enum.Result.html - -[`PyResult`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/type.PyResult.html [`PyResult`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/type.PyResult.html [`PyErr`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html [`pyo3::exceptions`]: {{#PYO3_DOCS_URL}}/pyo3/exceptions/index.html diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md index 8341f51c38c..a185bb541c0 100644 --- a/guide/src/function/signature.md +++ b/guide/src/function/signature.md @@ -1,8 +1,12 @@ # Function signatures -The `#[pyfunction]` attribute also accepts parameters to control how the generated Python function accepts arguments. Just like in Python, arguments can be positional-only, keyword-only, or accept either. `*args` lists and `**kwargs` dicts can also be accepted. These parameters also work for `#[pymethods]` which will be introduced in the [Python Classes](../class.md) section of the guide. +The `#[pyfunction]` attribute also accepts parameters to control how the generated Python function accepts arguments. +Just like in Python, arguments can be positional-only, keyword-only, or accept either. `*args` lists and `**kwargs` dicts can also be accepted. +These parameters also work for `#[pymethods]` which will be introduced in the [Python Classes](../class.md) section of the guide. -Like Python, by default PyO3 accepts all arguments as either positional or keyword arguments. Most arguments are required by default, except for trailing `Option<_>` arguments, which are [implicitly given a default of `None`](#trailing-optional-arguments). This behaviour can be configured by the `#[pyo3(signature = (...))]` option which allows writing a signature in Python syntax. +Like Python, by default PyO3 accepts all arguments as either positional or keyword arguments. +All arguments are required by default. +This behaviour can be configured by the `#[pyo3(signature = (...))]` option which allows writing a signature in Python syntax. This section of the guide goes into detail about use of the `#[pyo3(signature = (...))]` option and its related option `#[pyo3(text_signature = "...")]` @@ -10,36 +14,35 @@ This section of the guide goes into detail about use of the `#[pyo3(signature = For example, below is a function that accepts arbitrary keyword arguments (`**kwargs` in Python syntax) and returns the number that was passed: -```rust -use pyo3::prelude::*; -use pyo3::types::PyDict; - -#[pyfunction] -#[pyo3(signature = (**kwds))] -fn num_kwds(kwds: Option<&PyDict>) -> usize { - kwds.map_or(0, |dict| dict.len()) -} +```rust,no_run +#[pyo3::pymodule] +mod module_with_functions { + use pyo3::prelude::*; + use pyo3::types::PyDict; -#[pymodule] -fn module_with_functions(py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(num_kwds, m)?).unwrap(); - Ok(()) + #[pyfunction] + #[pyo3(signature = (**kwds))] + fn num_kwds(kwds: Option<&Bound<'_, PyDict>>) -> usize { + kwds.map_or(0, |dict| dict.len()) + } } ``` Just like in Python, the following constructs can be part of the signature:: - * `/`: positional-only arguments separator, each parameter defined before `/` is a positional-only parameter. - * `*`: var arguments separator, each parameter defined after `*` is a keyword-only parameter. - * `*args`: "args" is var args. Type of the `args` parameter has to be `&PyTuple`. - * `**kwargs`: "kwargs" receives keyword arguments. The type of the `kwargs` parameter has to be `Option<&PyDict>`. - * `arg=Value`: arguments with default value. +- `/`: positional-only arguments separator, each parameter defined before `/` is a positional-only parameter. +- `*`: var arguments separator, each parameter defined after `*` is a keyword-only parameter. +- `*args`: "args" is var args. + Type of the `args` parameter has to be `&Bound<'_, PyTuple>`. +- `**kwargs`: "kwargs" receives keyword arguments. + The type of the `kwargs` parameter has to be `Option<&Bound<'_, PyDict>>`. +- `arg=Value`: arguments with default value. If the `arg` argument is defined after var arguments, it is treated as a keyword-only argument. - Note that `Value` has to be valid rust code, PyO3 just inserts it into the generated - code unmodified. + Note that `Value` has to be valid rust code, PyO3 just inserts it into the generated code unmodified. Example: -```rust + +```rust,no_run # use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; # @@ -59,9 +62,9 @@ impl MyClass { fn method( &mut self, num: i32, - py_args: &PyTuple, + py_args: &Bound<'_, PyTuple>, name: &str, - py_kwargs: Option<&PyDict>, + py_kwargs: Option<&Bound<'_, PyDict>>, ) -> String { let num_before = self.num; self.num = num; @@ -80,36 +83,41 @@ impl MyClass { Arguments of type `Python` must not be part of the signature: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; #[pyfunction] #[pyo3(signature = (lambda))] -pub fn simple_python_bound_function(py: Python<'_>, lambda: PyObject) -> PyResult<()> { +pub fn simple_python_bound_function(py: Python<'_>, lambda: Py) -> PyResult<()> { Ok(()) } ``` -N.B. the position of the `/` and `*` arguments (if included) control the system of handling positional and keyword arguments. In Python: +N.B. the position of the `/` and `*` arguments (if included) control the system of handling positional and keyword arguments. +In Python: + ```python import mymodule mc = mymodule.MyClass() print(mc.method(44, False, "World", 666, x=44, y=55)) print(mc.method(num=-1, name="World")) -print(mc.make_change(44, False)) +print(mc.make_change(44)) ``` + Produces output: + ```text -py_args=('World', 666), py_kwargs=Some({'x': 44, 'y': 55}), name=Hello, num=44 -py_args=(), py_kwargs=None, name=World, num=-1 +num=44 (was previously=-1), py_args=(False, 'World', 666), name=Hello, py_kwargs=Some({'x': 44, 'y': 55}) +num=-1 (was previously=44), py_args=(), name=World, py_kwargs=None num=44 -num=-1 ``` + + > Note: to use keywords like `struct` as a function argument, use "raw identifier" syntax `r#struct` in both the signature and the function definition: > -> ```rust +> ```rust,no_run > # #![allow(dead_code)] > # use pyo3::prelude::*; > #[pyfunction(signature = (r#struct = "foo"))] @@ -119,74 +127,15 @@ num=-1 > } > ``` -## Trailing optional arguments - -As a convenience, functions without a `#[pyo3(signature = (...))]` option will treat trailing `Option` arguments as having a default of `None`. In the example below, PyO3 will create `increment` with a signature of `increment(x, amount=None)`. - -```rust -use pyo3::prelude::*; - -/// Returns a copy of `x` increased by `amount`. -/// -/// If `amount` is unspecified or `None`, equivalent to `x + 1`. -#[pyfunction] -fn increment(x: u64, amount: Option) -> u64 { - x + amount.unwrap_or(1) -} -# -# fn main() -> PyResult<()> { -# Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(increment, py)?; -# -# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; -# let sig: String = inspect -# .call1((fun,))? -# .call_method0("__str__")? -# .extract()?; -# -# #[cfg(Py_3_8)] // on 3.7 the signature doesn't render b, upstream bug? -# assert_eq!(sig, "(x, amount=None)"); -# -# Ok(()) -# }) -# } -``` - -To make trailing `Option` arguments required, but still accept `None`, add a `#[pyo3(signature = (...))]` annotation. For the example above, this would be `#[pyo3(signature = (x, amount))]`: - -```rust -# use pyo3::prelude::*; -#[pyfunction] -#[pyo3(signature = (x, amount))] -fn increment(x: u64, amount: Option) -> u64 { - x + amount.unwrap_or(1) -} -# -# fn main() -> PyResult<()> { -# Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(increment, py)?; -# -# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; -# let sig: String = inspect -# .call1((fun,))? -# .call_method0("__str__")? -# .extract()?; -# -# #[cfg(Py_3_8)] // on 3.7 the signature doesn't render b, upstream bug? -# assert_eq!(sig, "(x, amount)"); -# -# Ok(()) -# }) -# } -``` - -To help avoid confusion, PyO3 requires `#[pyo3(signature = (...))]` when an `Option` argument is surrounded by arguments which aren't `Option`. + ## Making the function signature available to Python -The function signature is exposed to Python via the `__text_signature__` attribute. PyO3 automatically generates this for every `#[pyfunction]` and all `#[pymethods]` directly from the Rust function, taking into account any override done with the `#[pyo3(signature = (...))]` option. +The function signature is exposed to Python via the `__text_signature__` attribute. +PyO3 automatically generates this for every `#[pyfunction]` and all `#[pymethods]` directly from the Rust function, taking into account any override done with the `#[pyo3(signature = (...))]` option. -This automatic generation can only display the value of default arguments for strings, integers, boolean types, and `None`. Any other default arguments will be displayed as `...`. (`.pyi` type stub files commonly also use `...` for default arguments in the same way.) +This automatic generation can only display the value of default arguments for strings, integers, boolean types, and `None`. +Any other default arguments will be displayed as `...`. (`.pyi` type stub files commonly also use `...` for default arguments in the same way.) In cases where the automatically-generated signature needs adjusting, it can [be overridden](#overriding-the-generated-signature) using the `#[pyo3(text_signature)]` option.) @@ -203,7 +152,7 @@ fn add(a: u64, b: u64) -> u64 { } # # fn main() -> PyResult<()> { -# Python::with_gil(|py| { +# Python::attach(|py| { # let fun = pyo3::wrap_pyfunction!(add, py)?; # # let doc: String = fun.getattr("__doc__")?.extract()?; @@ -251,7 +200,7 @@ fn add(a: u64, b: u64) -> u64 { } # # fn main() -> PyResult<()> { -# Python::with_gil(|py| { +# Python::attach(|py| { # let fun = pyo3::wrap_pyfunction!(add, py)?; # # let doc: String = fun.getattr("__doc__")?.extract()?; @@ -269,7 +218,8 @@ fn add(a: u64, b: u64) -> u64 { # } ``` -PyO3 will include the contents of the annotation unmodified as the `__text_signature`. Below shows how IPython will now present this (see the default value of 0 for b): +PyO3 will include the contents of the annotation unmodified as the `__text_signature__`. +Below shows how IPython will now present this (see the default value of 0 for b): ```text >>> pyo3_test.add.__text_signature__ @@ -280,7 +230,8 @@ Docstring: This function adds two unsigned 64-bit integers. Type: builtin_function_or_method ``` -If no signature is wanted at all, `#[pyo3(text_signature = None)]` will disable the built-in signature. The snippet below demonstrates use of this: +If no signature is wanted at all, `#[pyo3(text_signature = None)]` will disable the built-in signature. +The snippet below demonstrates use of this: ```rust use pyo3::prelude::*; @@ -293,7 +244,7 @@ fn add(a: u64, b: u64) -> u64 { } # # fn main() -> PyResult<()> { -# Python::with_gil(|py| { +# Python::attach(|py| { # let fun = pyo3::wrap_pyfunction!(add, py)?; # # let doc: String = fun.getattr("__doc__")?.extract()?; @@ -314,3 +265,40 @@ True Docstring: This function adds two unsigned 64-bit integers. Type: builtin_function_or_method ``` + +### Type annotations in the signature + +When the `experimental-inspect` Cargo feature is enabled, the `signature` attribute can also contain type hints: + +```rust +# #[cfg(feature = "experimental-inspect")] { +use pyo3::prelude::*; + +#[pymodule] +pub mod example { + use pyo3::prelude::*; + + #[pyfunction] + #[pyo3(signature = (arg: "list[int]") -> "list[int]")] + fn list_of_int_identity(arg: Bound<'_, PyAny>) -> Bound<'_, PyAny> { + arg + } +} +# } +``` + +It enables the [work-in-progress capacity of PyO3 to autogenerate type stubs](../type-stub.md) to generate a file with the correct type hints: + +```python +def list_of_int_identity(arg: list[int]) -> list[int]: ... +``` + +instead of the generic: + +```python +import typing + +def list_of_int_identity(arg: typing.Any) -> typing.Any: ... +``` + +Note that currently type annotations must be written as Rust strings. diff --git a/guide/src/getting_started.md b/guide/src/getting-started.md similarity index 53% rename from guide/src/getting_started.md rename to guide/src/getting-started.md index 22bce336d80..a621b97ca1a 100644 --- a/guide/src/getting_started.md +++ b/guide/src/getting-started.md @@ -1,64 +1,76 @@ # Installation -To get started using PyO3 you will need three things: a Rust toolchain, a Python environment, and a way to build. We'll cover each of these below. +To get started using PyO3 you will need three things: a Rust toolchain, a Python environment, and a way to build. +We'll cover each of these below. + +> If you'd like to chat to the PyO3 maintainers and other PyO3 users, consider joining the [PyO3 Discord server](https://discord.gg/33kcChzH7f). We're keen to hear about your experience getting started, so we can make PyO3 as accessible as possible for everyone! ## Rust -First, make sure you have Rust installed on your system. If you haven't already done so, try following the instructions [here](https://www.rust-lang.org/tools/install). PyO3 runs on both the `stable` and `nightly` versions so you can choose whichever one fits you best. The minimum required Rust version is 1.56. +First, make sure you have Rust installed on your system. +If you haven't already done so, try following the instructions [here](https://www.rust-lang.org/tools/install). +PyO3 runs on both the `stable` and `nightly` versions so you can choose whichever one fits you best. +The minimum required Rust version is 1.83. If you can run `rustc --version` and the version is new enough you're good to go! ## Python -To use PyO3, you need at least Python 3.7. While you can simply use the default Python interpreter on your system, it is recommended to use a virtual environment. +To use PyO3, you need at least Python 3.7. +While you can simply use the default Python interpreter on your system, it is recommended to use a virtual environment. ## Virtualenvs -While you can use any virtualenv manager you like, we recommend the use of `pyenv` in particular if you want to develop or test for multiple different Python versions, so that is what the examples in this book will use. The installation instructions for `pyenv` can be found [here](https://github.com/pyenv/pyenv#getting-pyenv). (Note: To get the `pyenv activate` and `pyenv virtualenv` commands, you will also need to install the [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) plugin. The [pyenv installer](https://github.com/pyenv/pyenv-installer#installation--update--uninstallation) will install both together.) +While you can use any virtualenv manager you like, we recommend the use of `pyenv` in particular if you want to develop or test for multiple different Python versions, so that is what the examples in this book will use. +The installation instructions for `pyenv` can be found [here](https://github.com/pyenv/pyenv#a-getting-pyenv). (Note: To get the `pyenv activate` and `pyenv virtualenv` commands, you will also need to install the [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) plugin. +The [pyenv installer](https://github.com/pyenv/pyenv-installer#installation--update--uninstallation) will install both together.) -If you intend to run Python from Rust (for example in unit tests) you should set the following environment variable when installing a new Python version using `pyenv`: -```bash -PYTHON_CONFIGURE_OPTS="--enable-shared" -``` +It can be useful to keep the sources used when installing using `pyenv` so that future debugging can see the original source files. +This can be done by passing the `--keep` flag as part of the `pyenv install` command. For example: ```bash -env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.12 +pyenv install 3.12 --keep ``` -You can read more about `pyenv`'s configuration options [here](https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#building-with---enable-shared). - ### Building -There are a number of build and Python package management systems such as [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](https://pyo3.rs/latest/building_and_distribution.html#manual-builds). We recommend the use of `maturin`, which you can install [here](https://maturin.rs/installation.html). It is developed to work with PyO3 and provides the most "batteries included" experience, especially if you are aiming to publish to PyPI. `maturin` is just a Python package, so you can add it in the same you already install Python packages. +There are a number of build and Python package management systems such as [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](./building-and-distribution.md#manual-builds). +We recommend the use of `maturin`, which you can install [here](https://maturin.rs/installation.html). +It is developed to work with PyO3 and provides the most "batteries included" experience, especially if you are aiming to publish to PyPI. `maturin` is just a Python package, so you can add it in the same way you already install Python packages. System Python: + ```bash pip install maturin --user ``` pipx: + ```bash pipx install maturin ``` pyenv: + ```bash pyenv activate pyo3 pip install maturin ``` poetry: + ```bash poetry add -G dev maturin ``` After installation, you can run `maturin --version` to check that you have correctly installed it. -# Starting a new project +## Starting a new project -First you should create the folder and virtual environment that are going to contain your new project. Here we will use the recommended `pyenv`: +First you should create the folder and virtual environment that are going to contain your new project. +Here we will use the recommended `pyenv`: ```bash mkdir pyo3-example @@ -67,7 +79,9 @@ pyenv virtualenv pyo3 pyenv local pyo3 ``` -After this, you should install your build manager. In this example, we will use `maturin`. After you've activated your virtualenv, add `maturin` to it: +After this, you should install your build manager. +In this example, we will use `maturin`. +After you've activated your virtualenv, add `maturin` to it: ```bash pip install maturin @@ -88,7 +102,7 @@ pyenv virtualenv pyo3 pyenv local pyo3 ``` -# Adding to an existing project +## Adding to an existing project Sadly, `maturin` cannot currently be run in existing projects, so if you want to use Python in an existing project you basically have two options: @@ -99,8 +113,9 @@ If you opt for the second option, here are the things you need to pay attention ## Cargo.toml -Make sure that the Rust crate you want to be able to access from Python is compiled into a library. You can have a binary output as well, but the code you want to access from Python has to be in the library part. Also, make sure that the crate type is `cdylib` and add PyO3 as a dependency as so: - +Make sure that the Rust crate you want to be able to access from Python is compiled into a library. +You can have a binary output as well, but the code you want to access from Python has to be in the library part. +Also, make sure that the crate type is `cdylib` and add PyO3 as a dependency as so: ```toml # If you already have [package] information in `Cargo.toml`, you can ignore @@ -148,22 +163,19 @@ classifiers = [ After this you can setup Rust code to be available in Python as below; for example, you can place this code in `src/lib.rs`: -```rust -use pyo3::prelude::*; - -/// Formats the sum of two numbers as string. -#[pyfunction] -fn sum_as_string(a: usize, b: usize) -> PyResult { - Ok((a + b).to_string()) -} - +```rust,no_run /// A Python module implemented in Rust. The name of this function must match /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to /// import the module. -#[pymodule] -fn pyo3_example(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; - Ok(()) +#[pyo3::pymodule] +mod pyo3_example { + use pyo3::prelude::*; + + /// Formats the sum of two numbers as string. + #[pyfunction] + fn sum_as_string(a: usize, b: usize) -> PyResult { + Ok((a + b).to_string()) + } } ``` @@ -178,4 +190,9 @@ $ python '25' ``` -For more instructions on how to use Python code from Rust, see the [Python from Rust](python_from_rust.md) page. +For more instructions on how to use Python code from Rust, see the [Python from Rust](python-from-rust.md) page. + +## Maturin Import Hook + +In development, any changes in the code would require running `maturin develop` before testing. +To streamline the development process, you may want to install [Maturin Import Hook](https://github.com/PyO3/maturin-import-hook) which will run `maturin develop` automatically when the library with code changes is being imported. diff --git a/guide/src/index.md b/guide/src/index.md index 80534caea5f..309809ad9cb 100644 --- a/guide/src/index.md +++ b/guide/src/index.md @@ -1,7 +1,17 @@ # The PyO3 user guide -Welcome to the PyO3 user guide! This book is a companion to [PyO3's API docs](https://docs.rs/pyo3). It contains examples and documentation to explain all of PyO3's use cases in detail. +Welcome to the PyO3 user guide! +This book is a companion to [PyO3's API docs](https://docs.rs/pyo3). +It contains examples and documentation to explain all of PyO3's use cases in detail. + +The rough order of material in this user guide is as follows: + 1. Getting started + 2. Wrapping Rust code for use from Python + 3. How to use Python code from Rust + 4. Remaining topics which go into advanced concepts in detail Please choose from the chapters on the left to jump to individual topics, or continue below to start with PyO3's README. +
+ {{#include ../../README.md}} diff --git a/guide/src/memory.md b/guide/src/memory.md deleted file mode 100644 index 2f5e5d9b0bd..00000000000 --- a/guide/src/memory.md +++ /dev/null @@ -1,233 +0,0 @@ -# Memory management - -Rust and Python have very different notions of memory management. Rust has -a strict memory model with concepts of ownership, borrowing, and lifetimes, -where memory is freed at predictable points in program execution. Python has -a looser memory model in which variables are reference-counted with shared, -mutable state by default. A global interpreter lock (GIL) is needed to prevent -race conditions, and a garbage collector is needed to break reference cycles. -Memory in Python is freed eventually by the garbage collector, but not usually -in a predictable way. - -PyO3 bridges the Rust and Python memory models with two different strategies for -accessing memory allocated on Python's heap from inside Rust. These are -GIL-bound, or "owned" references, and GIL-independent `Py` smart pointers. - -## GIL-bound memory - -PyO3's GIL-bound, "owned references" (`&PyAny` etc.) make PyO3 more ergonomic to -use by ensuring that their lifetime can never be longer than the duration the -Python GIL is held. This means that most of PyO3's API can assume the GIL is -held. (If PyO3 could not assume this, every PyO3 API would need to take a -`Python` GIL token to prove that the GIL is held.) This allows us to write -very simple and easy-to-understand programs like this: - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; - println!("Python says: {}", hello); - Ok(()) -})?; -# Ok(()) -# } -``` - -Internally, calling `Python::with_gil()` creates a `GILPool` which owns the -memory pointed to by the reference. In the example above, the lifetime of the -reference `hello` is bound to the `GILPool`. When the `with_gil()` closure ends -the `GILPool` is also dropped and the Python reference counts of the variables -it owns are decreased, releasing them to the Python garbage collector. Most -of the time we don't have to think about this, but consider the following: - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - for _ in 0..10 { - let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; - println!("Python says: {}", hello); - } - // There are 10 copies of `hello` on Python's heap here. - Ok(()) -})?; -# Ok(()) -# } -``` - -We might assume that the `hello` variable's memory is freed at the end of each -loop iteration, but in fact we create 10 copies of `hello` on Python's heap. -This may seem surprising at first, but it is completely consistent with Rust's -memory model. The `hello` variable is dropped at the end of each loop, but it -is only a reference to the memory owned by the `GILPool`, and its lifetime is -bound to the `GILPool`, not the for loop. The `GILPool` isn't dropped until -the end of the `with_gil()` closure, at which point the 10 copies of `hello` -are finally released to the Python garbage collector. - -In general we don't want unbounded memory growth during loops! One workaround -is to acquire and release the GIL with each iteration of the loop. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -for _ in 0..10 { - Python::with_gil(|py| -> PyResult<()> { - let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; - println!("Python says: {}", hello); - Ok(()) - })?; // only one copy of `hello` at a time -} -# Ok(()) -# } -``` - -It might not be practical or performant to acquire and release the GIL so many -times. Another workaround is to work with the `GILPool` object directly, but -this is unsafe. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - for _ in 0..10 { - let pool = unsafe { py.new_pool() }; - let py = pool.python(); - let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; - println!("Python says: {}", hello); - } - Ok(()) -})?; -# Ok(()) -# } -``` - -The unsafe method `Python::new_pool` allows you to create a nested `GILPool` -from which you can retrieve a new `py: Python` GIL token. Variables created -with this new GIL token are bound to the nested `GILPool` and will be released -when the nested `GILPool` is dropped. Here, the nested `GILPool` is dropped -at the end of each loop iteration, before the `with_gil()` closure ends. - -When doing this, you must be very careful to ensure that once the `GILPool` is -dropped you do not retain access to any owned references created after the -`GILPool` was created. Read the -[documentation for `Python::new_pool()`]({{#PYO3_DOCS_URL}}/pyo3/prelude/struct.Python.html#method.new_pool) -for more information on safety. - -This memory management can also be applicable when writing extension modules. -`#[pyfunction]` and `#[pymethods]` will create a `GILPool` which lasts the entire -function call, releasing objects when the function returns. Most functions only create -a few objects, meaning this doesn't have a significant impact. Occasionally functions -with long complex loops may need to use `Python::new_pool` as shown above. - -This behavior may change in future, see [issue #1056](https://github.com/PyO3/pyo3/issues/1056). - -## GIL-independent memory - -Sometimes we need a reference to memory on Python's heap that can outlive the -GIL. Python's `Py` is analogous to `Arc`, but for variables whose -memory is allocated on Python's heap. Cloning a `Py` increases its -internal reference count just like cloning `Arc`. The smart pointer can -outlive the "GIL is held" period in which it was created. It isn't magic, -though. We need to reacquire the GIL to access the memory pointed to by the -`Py`. - -What happens to the memory when the last `Py` is dropped and its -reference count reaches zero? It depends whether or not we are holding the GIL. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - let hello: Py = py.eval_bound("\"Hello World!\"", None, None)?.extract()?; - println!("Python says: {}", hello.bind(py)); - Ok(()) -})?; -# Ok(()) -# } -``` - -At the end of the `Python::with_gil()` closure `hello` is dropped, and then the -GIL is dropped. Since `hello` is dropped while the GIL is still held by the -current thread, its memory is released to the Python garbage collector -immediately. - -This example wasn't very interesting. We could have just used a GIL-bound -`&PyString` reference. What happens when the last `Py` is dropped while -we are *not* holding the GIL? - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -let hello: Py = Python::with_gil(|py| { - py.eval_bound("\"Hello World!\"", None, None)?.extract() -})?; -// Do some stuff... -// Now sometime later in the program we want to access `hello`. -Python::with_gil(|py| { - println!("Python says: {}", hello.as_ref(py)); -}); -// Now we're done with `hello`. -drop(hello); // Memory *not* released here. -// Sometime later we need the GIL again for something... -Python::with_gil(|py| - // Memory for `hello` is released here. -# () -); -# Ok(()) -# } -``` - -When `hello` is dropped *nothing* happens to the pointed-to memory on Python's -heap because nothing _can_ happen if we're not holding the GIL. Fortunately, -the memory isn't leaked. PyO3 keeps track of the memory internally and will -release it the next time we acquire the GIL. - -We can avoid the delay in releasing memory if we are careful to drop the -`Py` while the GIL is held. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -let hello: Py = - Python::with_gil(|py| py.eval_bound("\"Hello World!\"", None, None)?.extract())?; -// Do some stuff... -// Now sometime later in the program: -Python::with_gil(|py| { - println!("Python says: {}", hello.bind(py)); - drop(hello); // Memory released here. -}); -# Ok(()) -# } -``` - -We could also have used `Py::into_ref()`, which consumes `self`, instead of -`Py::as_ref()`. But note that in addition to being slower than `as_ref()`, -`into_ref()` binds the memory to the lifetime of the `GILPool`, which means -that rather than being released immediately, the memory will not be released -until the GIL is dropped. - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyString; -# fn main() -> PyResult<()> { -let hello: Py = - Python::with_gil(|py| py.eval_bound("\"Hello World!\"", None, None)?.extract())?; -// Do some stuff... -// Now sometime later in the program: -Python::with_gil(|py| { - println!("Python says: {}", hello.into_bound(py)); - // Memory not released yet. - // Do more stuff... - // Memory released here at end of `with_gil()` closure. -}); -# Ok(()) -# } -``` diff --git a/guide/src/migration.md b/guide/src/migration.md index 936784897bd..c92fed5126d 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -3,24 +3,800 @@ This guide can help you upgrade code through breaking changes from one PyO3 version to the next. For a detailed list of all changes, see the [CHANGELOG](changelog.md). +## from 0.26.* to 0.27 + +### `FromPyObject` reworked for flexibility and efficiency + +
+Click to expand + +With the removal of the `gil-ref` API in PyO3 0.23 it is now possible to fully split the Python lifetime `'py` and the input lifetime `'a`. +This allows borrowing from the input data without extending the lifetime of being attached to the interpreter. + +`FromPyObject` now takes an additional lifetime `'a` describing the input lifetime. +The argument type of the `extract` method changed from `&Bound<'py, PyAny>` to `Borrowed<'a, 'py, PyAny>`. +This was done because `&'a Bound<'py, PyAny>` would have an implicit restriction `'py: 'a` due to the reference type. + +This new form was partly implemented already in 0.22 using the internal `FromPyObjectBound` trait and +is now extended to all types. + +Most implementations can just add an elided lifetime to migrate. + +Additionally `FromPyObject` gained an associated type `Error`. +This is the error type that can be used in case of a conversion error. +During migration using `PyErr` is a good default, later a custom error type can be introduced to prevent unneccessary creation of Python exception objects and improved type safety. + +Before: + +```rust,ignore +impl<'py> FromPyObject<'py> for IpAddr { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + ... + } +} +``` + +After + +```rust,ignore +impl<'py> FromPyObject<'_, 'py> for IpAddr { + type Error = PyErr; + + fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result { + ... + // since `Borrowed` derefs to `&Bound`, the body often + // needs no changes, or adding an occasional `&` + } +} +``` + +Occasionally, more steps are necessary. +For generic types, the bounds need to be adjusted. +The correct bound depends on how the type is used. + +For simple wrapper types usually it's possible to just forward the bound. + +Before: + +```rust,ignore +struct MyWrapper(T); + +impl<'py, T> FromPyObject<'py> for MyWrapper +where + T: FromPyObject<'py> +{ + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + ob.extract().map(MyWrapper) + } +} +``` + +After: + +```rust +# use pyo3::prelude::*; +# #[allow(dead_code)] +# pub struct MyWrapper(T); +impl<'a, 'py, T> FromPyObject<'a, 'py> for MyWrapper +where + T: FromPyObject<'a, 'py> +{ + type Error = T::Error; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + obj.extract().map(MyWrapper) + } +} +``` + +Container types that need to create temporary Python references during extraction, for example extracing from a `PyList`, requires a stronger bound. +For these the `FromPyObjectOwned` trait was introduced. +It is automatically implemented for any type that implements `FromPyObject` and does not borrow from the input. +It is intended to be used as a trait bound in these situations. + +Before: + +```rust,ignore +struct MyVec(Vec); +impl<'py, T> FromPyObject<'py> for Vec +where + T: FromPyObject<'py>, +{ + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + let mut v = MyVec(Vec::new()); + for item in obj.try_iter()? { + v.0.push(item?.extract::()?); + } + Ok(v) + } +} +``` + +After: + +```rust +# use pyo3::prelude::*; +# #[allow(dead_code)] +# pub struct MyVec(Vec); +impl<'py, T> FromPyObject<'_, 'py> for MyVec +where + T: FromPyObjectOwned<'py> // 👈 can only extract owned values, because each `item` below + // is a temporary short lived owned reference +{ + type Error = PyErr; + + fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result { + let mut v = MyVec(Vec::new()); + for item in obj.try_iter()? { + v.0.push(item?.extract::().map_err(Into::into)?); // `map_err` is needed because `?` uses `From`, not `Into` 🙁 + } + Ok(v) + } +} +``` + +This is very similar to `serde`s [`Deserialize`] and [`DeserializeOwned`] traits, see [here](https://serde.rs/lifetimes.html). + +[`Deserialize`]: https://docs.rs/serde/latest/serde/trait.Deserialize.html +[`DeserializeOwned`]: https://docs.rs/serde/latest/serde/de/trait.DeserializeOwned.html +
+ +## `.downcast()` and `DowncastError` replaced with `.cast()` and `CastError` + +The `.downcast()` family of functions were only available on `Bound`. +In corner cases (particularly related to `.downcast_into()`) this would require use of `.as_any().downcast()` or `.into_any().downcast_into()` chains. +Additionally, `DowncastError` produced Python exception messages which are not very Pythonic due to use of Rust type names in the error messages. + +The `.cast()` family of functions are available on all `Bound` and `Borrowed` smart pointers, whatever the type, and have error messages derived from the actual type at runtime. +This produces a nicer experience for both PyO3 module authors and consumers. + +To migrate, replace `.downcast()` with `.cast()` and `DowncastError` with `CastError` (and similar with `.downcast_into()` / `DowncastIntoError` etc). + +`CastError` requires a Python `type` object (or other "classinfo" object compatible with `isinstance()`) as the second object, so in the rare case where `DowncastError` was manually constructed, small adjustments to code may apply. + +## `PyTypeCheck` is now an `unsafe trait` + +Because `PyTypeCheck` is the trait used to guard the `.cast()` functions to treat Python objects as specific concrete types, the trait is `unsafe` to implement. + +This should always have been the case, it was an unfortunate omission from its original implementation which is being corrected in this release. + +## from 0.25.* to 0.26 + +### Rename of `Python::with_gil`, `Python::allow_threads`, and `pyo3::prepare_freethreaded_python` + +
+Click to expand + +The names for these APIs were created when the global interpreter lock (GIL) was mandatory. +With the introduction of free-threading in Python 3.13 this is no longer the case, and the naming has no universal meaning anymore. +For this reason, we chose to rename these to more modern terminology introduced in free-threading: + +- `Python::with_gil` is now called `Python::attach`, it attaches a Python thread-state to the current thread. + In GIL enabled builds there can only be 1 thread attached to the interpreter, in free-threading there can be more. +- `Python::allow_threads` is now called `Python::detach`, it detaches a previously attached thread-state. +- `pyo3::prepare_freethreaded_python` is now called `Python::initialize`. +
+ +### Deprecation of `PyObject` type alias + +
+Click to expand + +The type alias `PyObject` (aka `Py`) is often confused with the identically named FFI definition `pyo3::ffi::PyObject`. +For this reason we are deprecating its usage. +To migrate simply replace its usage by the target type `Py`. +
+ +### Replacement of `GILOnceCell` with `PyOnceLock` + +
+Click to expand + +Similar to the above renaming of `Python::with_gil` and related APIs, the `GILOnceCell` type was designed for a Python interpreter which was limited by the GIL. +Aside from its name, it allowed for the "once" initialization to race because the racing was mediated by the GIL and was extremely unlikely to manifest in practice. + +With the introduction of free-threaded Python the racy initialization behavior is more likely to be problematic and so a new type `PyOnceLock` has been introduced which performs true single-initialization correctly while attached to the Python interpreter. +It exposes the same API as `GILOnceCell`, so should be a drop-in replacement with the notable exception that if the racy initialization of `GILOnceCell` was inadvertently relied on (e.g. due to circular references) then the stronger once-ever guarantee of `PyOnceLock` may lead to deadlocking which requires refactoring. + +Before: + +```rust +# #![allow(deprecated)] +# use pyo3::prelude::*; +# use pyo3::sync::GILOnceCell; +# use pyo3::types::PyType; +# fn main() -> PyResult<()> { +# Python::attach(|py| { +static DECIMAL_TYPE: GILOnceCell> = GILOnceCell::new(); +DECIMAL_TYPE.import(py, "decimal", "Decimal")?; +# Ok(()) +# }) +# } +``` + +After: + +```rust +# use pyo3::prelude::*; +# use pyo3::sync::PyOnceLock; +# use pyo3::types::PyType; +# fn main() -> PyResult<()> { +# Python::attach(|py| { +static DECIMAL_TYPE: PyOnceLock> = PyOnceLock::new(); +DECIMAL_TYPE.import(py, "decimal", "Decimal")?; +# Ok(()) +# }) +# } +``` + +
+ +### Deprecation of `GILProtected` + +
+Click to expand + +As another cleanup related to concurrency primitives designed for a Python constrained by the GIL, the `GILProtected` type is now deprecated. +Prefer to use concurrency primitives which are compatible with free-threaded Python, such as [`std::sync::Mutex`](https://doc.rust-lang.org/std/sync/struct.Mutex.html) (in combination with PyO3's [`MutexExt`]({{#PYO3_DOCS_URL}}/pyo3/sync/trait.MutexExt.html) trait). + +Before: + +```rust +# #![allow(deprecated)] +# use pyo3::prelude::*; +# fn main() { +# #[cfg(not(Py_GIL_DISABLED))] { +use pyo3::sync::GILProtected; +use std::cell::RefCell; +# Python::attach(|py| { +static NUMBERS: GILProtected>> = GILProtected::new(RefCell::new(Vec::new())); +Python::attach(|py| { + NUMBERS.get(py).borrow_mut().push(42); +}); +# }) +# } +# } +``` + +After: + +```rust +# use pyo3::prelude::*; +use pyo3::sync::MutexExt; +use std::sync::Mutex; +# fn main() { +# Python::attach(|py| { +static NUMBERS: Mutex> = Mutex::new(Vec::new()); +Python::attach(|py| { + NUMBERS.lock_py_attached(py).expect("no poisoning").push(42); +}); +# }) +# } +``` + +
+ +### `PyMemoryError` now maps to `io::ErrorKind::OutOfMemory` when converted to `io::Error` + +
+Click to expand + +Previously, converting a `PyMemoryError` into a Rust `io::Error` would result in an error with kind `Other`. +Now, it produces an error with kind `OutOfMemory`. +Similarly, converting an `io::Error` with kind `OutOfMemory` back into a Python error would previously yield a generic `PyOSError`. +Now, it yields a `PyMemoryError`. + +This change makes error conversions more precise and matches the semantics of out-of-memory errors between Python and Rust. +
+ +## from 0.24.* to 0.25 + +### `AsPyPointer` removal + +
+Click to expand +The `AsPyPointer` trait is mostly a leftover from the now removed gil-refs API. The last remaining uses were the GC API, namely `PyVisit::call`, and identity comparison (`PyAnyMethods::is` and `Py::is`). + +`PyVisit::call` has been updated to take `T: Into>>`, which allows for arguments of type `&Py`, `&Option>` and `Option<&Py>`. +It is unlikely any changes are needed here to migrate. + +`PyAnyMethods::is`/ `Py::is` has been updated to take `T: AsRef>>`. +Additionally `AsRef>>` implementations were added for `Py`, `Bound` and `Borrowed`. +Because of the existing `AsRef> for Bound` implementation this may cause inference issues in non-generic code. +This can be easily migrated by switching to `as_any` instead of `as_ref` for these calls. +
+ +## from 0.23.* to 0.24 + +
+Click to expand +There were no significant changes from 0.23 to 0.24 which required documenting in this guide. +
+ +## from 0.22.* to 0.23 + +
+Click to expand + +PyO3 0.23 is a significant rework of PyO3's internals for two major improvements: + +- Support of Python 3.13's new freethreaded build (aka "3.13t") +- Rework of to-Python conversions with a new `IntoPyObject` trait. + +These changes are both substantial and reasonable efforts have been made to allow as much code as possible to continue to work as-is despite the changes. +The impacts are likely to be seen in three places when upgrading: + +- PyO3's data structures [are now thread-safe](#free-threaded-python-support) instead of reliant on the GIL for synchronization. + In particular, `#[pyclass]` types are [now required to be `Sync`](./class/thread-safety.md). +- The [`IntoPyObject` trait](#new-intopyobject-trait-unifies-to-python-conversions) may need to be implemented for types in your codebase. + In most cases this can simply be done with [`#[derive(IntoPyObject)]`](#intopyobject-and-intopyobjectref-derive-macros). + There will be many deprecation warnings from the replacement of `IntoPy` and `ToPyObject` traits. +- There will be many deprecation warnings from the [final removal of the `gil-refs` feature](#gil-refs-feature-removed), which opened up API space for a cleanup and simplification to PyO3's "Bound" API. + +The sections below discuss the rationale and details of each change in more depth. +
+ +### Free-threaded Python Support + +
+Click to expand + +PyO3 0.23 introduces initial support for the new free-threaded build of +CPython 3.13, aka "3.13t". + +Because this build allows multiple Python threads to operate simultaneously on underlying Rust data, the `#[pyclass]` macro now requires that types it operates on implement `Sync`. + +Aside from the change to `#[pyclass]`, most features of PyO3 work unchanged, as the changes have been to the internal data structures to make them thread-safe. +An example of this is the `GILOnceCell` type, which used the GIL to synchronize single-initialization. +It now uses internal locks to guarantee that only one write ever succeeds, however it allows for multiple racing runs of the initialization closure. +It may be preferable to instead use `std::sync::OnceLock` in combination with the `pyo3::sync::OnceLockExt` trait which adds `OnceLock::get_or_init_py_attached` for single-initialization where the initialization closure is guaranteed only ever to run once and without deadlocking with the GIL. + +Future PyO3 versions will likely add more traits and data structures to make working with free-threaded Python easier. + +Some features are unaccessible on the free-threaded build: + +- The `GILProtected` type, which relied on the GIL to expose synchronized access to inner contents +- `PyList::get_item_unchecked`, which cannot soundly be used due to races between time-of-check and time-of-use + +If you make use of these features then you will need to account for the unavailability of the API in the free-threaded build. +One way to handle it is via conditional compilation -- extensions can use `pyo3-build-config` to get access to a `#[cfg(Py_GIL_DISABLED)]` guard. + +See [the guide section on free-threaded Python](free-threading.md) for more details about supporting free-threaded Python in your PyO3 extensions. +
+ +### New `IntoPyObject` trait unifies to-Python conversions + +
+Click to expand + +PyO3 0.23 introduces a new `IntoPyObject` trait to convert Rust types into Python objects which replaces both `IntoPy` and `ToPyObject`. +Notable features of this new trait include: + +- conversions can now return an error +- it is designed to work efficiently for both `T` owned types and `&T` references +- compared to `IntoPy` the generic `T` moved into an associated type, so + - there is now only one way to convert a given type + - the output type is stronger typed and may return any Python type instead of just `PyAny` +- byte collections are specialized to convert into `PyBytes` now, see [below](#to-python-conversions-changed-for-byte-collections-vecu8-u8-n-and-smallvecu8-n) +- `()` (unit) is now only specialized in return position of `#[pyfunction]` and `#[pymethods]` to return `None`, in normal usage it converts into an empty `PyTuple` + +All PyO3 provided types as well as `#[pyclass]`es already implement `IntoPyObject`. +Other types will need to adapt an implementation of `IntoPyObject` to stay compatible with the Python APIs. +In many cases the new [`#[derive(IntoPyObject)]`](#intopyobject-and-intopyobjectref-derive-macros) macro can be used instead of [manual implementations](#intopyobject-manual-implementation). + +Since `IntoPyObject::into_pyobject` may return either a `Bound` or `Borrowed`, you may find the [`BoundObject`](conversions/traits.md#boundobject-for-conversions-that-may-be-bound-or-borrowed) trait to be useful to write code that generically handles either type of smart pointer. + +Together with the introduction of `IntoPyObject` the old conversion traits `ToPyObject` and `IntoPy` +are deprecated and will be removed in a future PyO3 version. + +#### `IntoPyObject` and `IntoPyObjectRef` derive macros + +To implement the new trait you may use the new `IntoPyObject` and `IntoPyObjectRef` derive macros as below. + +```rust,no_run +# use pyo3::prelude::*; +#[derive(IntoPyObject, IntoPyObjectRef)] +struct Struct { + count: usize, + obj: Py, +} +``` + +The `IntoPyObjectRef` derive macro derives implementations for references (e.g. for `&Struct` in the example above), which is a replacement for the `ToPyObject` trait. + +#### `IntoPyObject` manual implementation + +Before: + +```rust,ignore +# use pyo3::prelude::*; +# #[allow(dead_code)] +struct MyPyObjectWrapper(PyObject); + +impl IntoPy for MyPyObjectWrapper { + fn into_py(self, py: Python<'_>) -> PyObject { + self.0 + } +} + +impl ToPyObject for MyPyObjectWrapper { + fn to_object(&self, py: Python<'_>) -> PyObject { + self.0.clone_ref(py) + } +} +``` + +After: + +```rust,no_run +# #![allow(deprecated)] +# use pyo3::prelude::*; +# #[allow(dead_code)] +# struct MyPyObjectWrapper(PyObject); + +impl<'py> IntoPyObject<'py> for MyPyObjectWrapper { + type Target = PyAny; // the Python type + type Output = Bound<'py, Self::Target>; // in most cases this will be `Bound` + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.0.into_bound(py)) + } +} + +// `ToPyObject` implementations should be converted to implementations on reference types +impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper { + type Target = PyAny; + type Output = Borrowed<'a, 'py, Self::Target>; // `Borrowed` can be used to optimized reference counting + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.0.bind_borrowed(py)) + } +} +``` + +
+ +### To-Python conversions changed for byte collections (`Vec`, `[u8; N]` and `SmallVec<[u8; N]>`) + +
+Click to expand + +With the introduction of the `IntoPyObject` trait, PyO3's macros now prefer `IntoPyObject` implementations over `IntoPy` when producing Python values. +This applies to `#[pyfunction]` and `#[pymethods]` return values and also fields accessed via `#[pyo3(get)]`. + +This change has an effect on functions and methods returning _byte_ collections like + +- `Vec` +- `[u8; N]` +- `SmallVec<[u8; N]>` + +In their new `IntoPyObject` implementation these will now turn into `PyBytes` rather than a `PyList`. +All other `T`s are unaffected and still convert into a `PyList`. + +```rust,no_run +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyfunction] +fn foo() -> Vec { // would previously turn into a `PyList`, now `PyBytes` + vec![0, 1, 2, 3] +} + +#[pyfunction] +fn bar() -> Vec { // unaffected, returns `PyList` + vec![0, 1, 2, 3] +} +``` + +If this conversion is _not_ desired, consider building a list manually using `PyList::new`. + +The following types were previously _only_ implemented for `u8` and now allow other `T`s turn into `PyList`: + +- `&[T]` +- `Cow<[T]>` + +This is purely additional and should just extend the possible return types. + +
+ +### `gil-refs` feature removed + +
+Click to expand + +PyO3 0.23 completes the removal of the "GIL Refs" API in favour of the new "Bound" API introduced in PyO3 0.21. + +With the removal of the old API, many "Bound" API functions which had been introduced with `_bound` suffixes no longer need the suffixes as these names have been freed up. +For example, `PyTuple::new_bound` is now just `PyTuple::new` (the existing name remains but is deprecated). + +Before: + +```rust,ignore +# #![allow(deprecated)] +# use pyo3::prelude::*; +# use pyo3::types::PyTuple; +# fn main() { +# Python::attach(|py| { +// For example, for PyTuple. Many such APIs have been changed. +let tup = PyTuple::new_bound(py, [1, 2, 3]); +# }) +# } +``` + +After: + +```rust +# use pyo3::prelude::*; +# use pyo3::types::PyTuple; +# fn main() { +# Python::attach(|py| { +// For example, for PyTuple. Many such APIs have been changed. +let tup = PyTuple::new(py, [1, 2, 3]); +# }) +# } +``` + +#### `IntoPyDict` trait adjusted for removal of `gil-refs` + +As part of this API simplification, the `IntoPyDict` trait has had a small breaking change: `IntoPyDict::into_py_dict_bound` method has been renamed to `IntoPyDict::into_py_dict`. +It is also now fallible as part of the `IntoPyObject` trait addition. + +If you implemented `IntoPyDict` for your type, you should implement `into_py_dict` instead of `into_py_dict_bound`. +The old name is still available for calling but deprecated. + +Before: + +```rust,ignore +# use pyo3::prelude::*; +# use pyo3::types::{PyDict, IntoPyDict}; +# use std::collections::HashMap; + +struct MyMap(HashMap); + +impl IntoPyDict for MyMap +where + K: ToPyObject, + V: ToPyObject, +{ + fn into_py_dict_bound(self, py: Python<'_>) -> Bound<'_, PyDict> { + let dict = PyDict::new_bound(py); + for (key, value) in self.0 { + dict.set_item(key, value) + .expect("Failed to set_item on dict"); + } + dict + } +} +``` + +After: + +```rust,no_run +# use pyo3::prelude::*; +# use pyo3::types::{PyDict, IntoPyDict}; +# use std::collections::HashMap; + +# #[allow(dead_code)] +struct MyMap(HashMap); + +impl<'py, K, V> IntoPyDict<'py> for MyMap +where + K: IntoPyObject<'py>, + V: IntoPyObject<'py>, +{ + fn into_py_dict(self, py: Python<'py>) -> PyResult> { + let dict = PyDict::new(py); + for (key, value) in self.0 { + dict.set_item(key, value)?; + } + Ok(dict) + } +} +``` + +
+ +## from 0.21.* to 0.22 + +### Deprecation of `gil-refs` feature continues + +
+Click to expand + +Following the introduction of the "Bound" API in PyO3 0.21 and the planned removal of the "GIL Refs" API, all functionality related to GIL Refs is now gated behind the `gil-refs` feature and emits a deprecation warning on use. + +See the 0.21 migration entry for help upgrading. +
+ +### Deprecation of implicit default for trailing optional arguments + +
+Click to expand + +With `pyo3` 0.22 the implicit `None` default for trailing `Option` type argument is deprecated. +To migrate, place a `#[pyo3(signature = (...))]` attribute on affected functions or methods and specify the desired behavior. +The migration warning specifies the corresponding signature to keep the current behavior. +With 0.23 the signature will be required for any function containing `Option` type parameters to prevent accidental and unnoticed changes in behavior. +With 0.24 this restriction will be lifted again and `Option` type arguments will be treated as any other argument _without_ special handling. + +Before: + +```rust,no_run +# #![allow(deprecated, dead_code)] +# use pyo3::prelude::*; +#[pyfunction] +fn increment(x: u64, amount: Option) -> u64 { + x + amount.unwrap_or(1) +} +``` + +After: + +```rust,no_run +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyfunction] +#[pyo3(signature = (x, amount=None))] +fn increment(x: u64, amount: Option) -> u64 { + x + amount.unwrap_or(1) +} +``` + +
+ +### `Py::clone` is now gated behind the `py-clone` feature + +
+Click to expand +If you rely on `impl Clone for Py` to fulfil trait requirements imposed by existing Rust code written without PyO3-based code in mind, the newly introduced feature `py-clone` must be enabled. + +However, take care to note that the behaviour is different from previous versions. +If `Clone` was called without the GIL being held, we tried to delay the application of these reference count increments until PyO3-based code would re-acquire it. +This turned out to be impossible to implement in a sound manner and hence was removed. +Now, if `Clone` is called without the GIL being held, we panic instead for which calling code might not be prepared. + +It is advised to migrate off the `py-clone` feature. +The simplest way to remove dependency on `impl Clone for Py` is to wrap `Py` as `Arc>` and use cloning of the arc. + +Related to this, we also added a `pyo3_disable_reference_pool` conditional compilation flag which removes the infrastructure necessary to apply delayed reference count decrements implied by `impl Drop for Py`. +They do not appear to be a soundness hazard as they should lead to memory leaks in the worst case. +However, the global synchronization adds significant overhead to cross the Python-Rust boundary. +Enabling this feature will remove these costs and make the `Drop` implementation abort the process if called without the GIL being held instead. +
+ +### Require explicit opt-in for comparison for simple enums + +
+Click to expand + +With `pyo3` 0.22 the new `#[pyo3(eq)]` options allows automatic implementation of Python equality using Rust's `PartialEq`. +Previously simple enums automatically implemented equality in terms of their discriminants. +To make PyO3 more consistent, this automatic equality implementation is deprecated in favour of having opt-ins for all `#[pyclass]` types. +Similarly, simple enums supported comparison with integers, which is not covered by Rust's `PartialEq` derive, so has been split out into the `#[pyo3(eq_int)]` attribute. + +To migrate, place a `#[pyo3(eq, eq_int)]` attribute on simple enum classes. + +Before: + +```rust,no_run +# #![allow(deprecated, dead_code)] +# use pyo3::prelude::*; +#[pyclass] +enum SimpleEnum { + VariantA, + VariantB = 42, +} +``` + +After: + +```rust,no_run +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] +enum SimpleEnum { + VariantA, + VariantB = 42, +} +``` + +
+ +### `PyType::name` reworked to better match Python `__name__` + +
+Click to expand + +This function previously would try to read directly from Python type objects' C API field (`tp_name`), in which case it would return a `Cow::Borrowed`. +However the contents of `tp_name` don't have well-defined semantics. + +Instead `PyType::name()` now returns the equivalent of Python `__name__` and returns `PyResult>`. + +The closest equivalent to PyO3 0.21's version of `PyType::name()` has been introduced as a new function `PyType::fully_qualified_name()`, +which is equivalent to `__module__` and `__qualname__` joined as `module.qualname`. + +Before: + +```rust,ignore +# #![allow(deprecated, dead_code)] +# use pyo3::prelude::*; +# use pyo3::types::{PyBool}; +# fn main() -> PyResult<()> { +Python::with_gil(|py| { + let bool_type = py.get_type_bound::(); + let name = bool_type.name()?.into_owned(); + println!("Hello, {}", name); + + let mut name_upper = bool_type.name()?; + name_upper.to_mut().make_ascii_uppercase(); + println!("Hello, {}", name_upper); + + Ok(()) +}) +# } +``` + +After: + +```rust,ignore +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::types::{PyBool}; +# fn main() -> PyResult<()> { +Python::with_gil(|py| { + let bool_type = py.get_type_bound::(); + let name = bool_type.name()?; + println!("Hello, {}", name); + + // (if the full dotted path was desired, switch from `name()` to `fully_qualified_name()`) + let mut name_upper = bool_type.fully_qualified_name()?.to_string(); + name_upper.make_ascii_uppercase(); + println!("Hello, {}", name_upper); + + Ok(()) +}) +# } +``` + +
+ ## from 0.20.* to 0.21 -PyO3 0.21 introduces a new `Bound<'py, T>` smart pointer which replaces the existing "GIL Refs" API to interact with Python objects. For example, in PyO3 0.20 the reference `&'py PyAny` would be used to interact with Python objects. In PyO3 0.21 the updated type is `Bound<'py, PyAny>`. Making this change moves Rust ownership semantics out of PyO3's internals and into user code. This change fixes [a known soundness edge case of interaction with gevent](https://github.com/PyO3/pyo3/issues/3668) as well as improves CPU and [memory performance](https://github.com/PyO3/pyo3/issues/1056). For a full history of discussion see https://github.com/PyO3/pyo3/issues/3382. +
+Click to expand + +PyO3 0.21 introduces a new `Bound<'py, T>` smart pointer which replaces the existing "GIL Refs" API to interact with Python objects. +For example, in PyO3 0.20 the reference `&'py PyAny` would be used to interact with Python objects. +In PyO3 0.21 the updated type is `Bound<'py, PyAny>`. +Making this change moves Rust ownership semantics out of PyO3's internals and into user code. +This change fixes [a known soundness edge case of interaction with gevent](https://github.com/PyO3/pyo3/issues/3668) as well as improves CPU and [memory performance](https://github.com/PyO3/pyo3/issues/1056). +For a full history of discussion see . +For a full history of discussion see . -The "GIL Ref" `&'py PyAny` and similar types such as `&'py PyDict` continue to be available as a deprecated API. Due to the advantages of the new API it is advised that all users make the effort to upgrade as soon as possible. +The "GIL Ref" `&'py PyAny` and similar types such as `&'py PyDict` continue to be available as a deprecated API. +Due to the advantages of the new API it is advised that all users make the effort to upgrade as soon as possible. In addition to the major API type overhaul, PyO3 has needed to make a few small breaking adjustments to other APIs to close correctness and soundness gaps. The recommended steps to update to PyO3 0.21 is as follows: - 1. Enable the `gil-refs` feature to silence deprecations related to the API change - 2. Fix all other PyO3 0.21 migration steps - 3. Disable the `gil-refs` feature and migrate off the deprecated APIs + +1. Enable the `gil-refs` feature to silence deprecations related to the API change +2. Fix all other PyO3 0.21 migration steps +3. Disable the `gil-refs` feature and migrate off the deprecated APIs The following sections are laid out in this order. +
### Enable the `gil-refs` feature -To make the transition for the PyO3 ecosystem away from the GIL Refs API as smooth as possible, in PyO3 0.21 no APIs consuming or producing GIL Refs have been altered. Instead, variants using `Bound` smart pointers have been introduced, for example `PyTuple::new_bound` which returns `Bound` is the replacement form of `PyTuple::new`. The GIL Ref APIs have been deprecated, but to make migration easier it is possible to disable these deprecation warnings by enabling the `gil-refs` feature. +
+Click to expand + +To make the transition for the PyO3 ecosystem away from the GIL Refs API as smooth as possible, in PyO3 0.21 no APIs consuming or producing GIL Refs have been altered. +Instead, variants using `Bound` smart pointers have been introduced, for example `PyTuple::new_bound` which returns `Bound` is the replacement form of `PyTuple::new`. +The GIL Ref APIs have been deprecated, but to make migration easier it is possible to disable these deprecation warnings by enabling the `gil-refs` feature. > The one single exception where an existing API was changed in-place is the `pyo3::intern!` macro. Almost all uses of this macro did not need to update code to account it changing to return `&Bound` immediately, and adding an `intern_bound!` replacement was perceived as adding more work for users. @@ -42,17 +818,24 @@ After: pyo3 = { version = "0.21", features = ["gil-refs"] } ``` +
+ ### `PyTypeInfo` and `PyTryFrom` have been adjusted -The `PyTryFrom` trait has aged poorly, its [`try_from`] method now conflicts with `try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`. +
+Click to expand + +The `PyTryFrom` trait has aged poorly, its `try_from` method now conflicts with `TryFrom::try_from` in the 2021 edition prelude. +A lot of its functionality was also duplicated with `PyTypeInfo`. -To tighten up the PyO3 traits as part of the deprecation of the GIL Refs API the `PyTypeInfo` trait has had a simpler companion `PyTypeCheck`. The methods [`PyAny::downcast`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast) and [`PyAny::downcast_exact`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast_exact) no longer use `PyTryFrom` as a bound, instead using `PyTypeCheck` and `PyTypeInfo` respectively. +To tighten up the PyO3 traits as part of the deprecation of the GIL Refs API the `PyTypeInfo` trait has had a simpler companion `PyTypeCheck`. +The methods `PyAny::downcast` and `PyAny::downcast_exact` no longer use `PyTryFrom` as a bound, instead using `PyTypeCheck` and `PyTypeInfo` respectively. To migrate, switch all type casts to use `obj.downcast()` instead of `try_from(obj)` (and similar for `downcast_exact`). Before: -```rust +```rust,ignore # #![allow(deprecated)] # use pyo3::prelude::*; # use pyo3::types::{PyInt, PyList}; @@ -67,7 +850,7 @@ Python::with_gil(|py| { After: -```rust +```rust,ignore # use pyo3::prelude::*; # use pyo3::types::{PyInt, PyList}; # fn main() -> PyResult<()> { @@ -82,14 +865,21 @@ Python::with_gil(|py| { # } ``` +
+ ### `Iter(A)NextOutput` are deprecated -The `__next__` and `__anext__` magic methods can now return any type convertible into Python objects directly just like all other `#[pymethods]`. The `IterNextOutput` used by `__next__` and `IterANextOutput` used by `__anext__` are subsequently deprecated. Most importantly, this change allows returning an awaitable from `__anext__` without non-sensically wrapping it into `Yield` or `Some`. Only the return types `Option` and `Result, E>` are still handled in a special manner where `Some(val)` yields `val` and `None` stops iteration. +
+Click to expand + +The `__next__` and `__anext__` magic methods can now return any type convertible into Python objects directly just like all other `#[pymethods]`. +The `IterNextOutput` used by `__next__` and `IterANextOutput` used by `__anext__` are subsequently deprecated. +Most importantly, this change allows returning an awaitable from `__anext__` without non-sensically wrapping it into `Yield` or `Some`. +Only the return types `Option` and `Result, E>` are still handled in a special manner where `Some(val)` yields `val` and `None` stops iteration. Starting with an implementation of a Python iterator using `IterNextOutput`, e.g. -```rust -#![allow(deprecated)] +```rust,ignore use pyo3::prelude::*; use pyo3::iter::IterNextOutput; @@ -113,7 +903,7 @@ impl PyClassIter { If returning `"done"` via `StopIteration` is not really required, this should be written as -```rust +```rust,no_run use pyo3::prelude::*; #[pyclass] @@ -134,11 +924,12 @@ impl PyClassIter { } ``` -This form also has additional benefits: It has already worked in previous PyO3 versions, it matches the signature of Rust's [`Iterator` trait](https://doc.rust-lang.org/stable/std/iter/trait.Iterator.html) and it allows using a fast path in CPython which completely avoids the cost of raising a `StopIteration` exception. Note that using [`Option::transpose`](https://doc.rust-lang.org/stable/std/option/enum.Option.html#method.transpose) and the `Result, E>` variant, this form can also be used to wrap fallible iterators. +This form also has additional benefits: It has already worked in previous PyO3 versions, it matches the signature of Rust's [`Iterator` trait](https://doc.rust-lang.org/stable/std/iter/trait.Iterator.html) and it allows using a fast path in CPython which completely avoids the cost of raising a `StopIteration` exception. +Note that using [`Option::transpose`](https://doc.rust-lang.org/stable/std/option/enum.Option.html#method.transpose) and the `Result, E>` variant, this form can also be used to wrap fallible iterators. Alternatively, the implementation can also be done as it would in Python itself, i.e. by "raising" a `StopIteration` exception -```rust +```rust,no_run use pyo3::prelude::*; use pyo3::exceptions::PyStopIteration; @@ -162,7 +953,7 @@ impl PyClassIter { Finally, an asynchronous iterator can directly return an awaitable without confusing wrapping -```rust +```rust,no_run use pyo3::prelude::*; #[pyclass] @@ -190,7 +981,9 @@ struct PyClassAsyncIter { impl PyClassAsyncIter { fn __anext__(&mut self) -> PyClassAwaitable { self.number += 1; - PyClassAwaitable { number: self.number } + PyClassAwaitable { + number: self.number, + } } fn __aiter__(slf: Py) -> Py { @@ -199,34 +992,102 @@ impl PyClassAsyncIter { } ``` +
+ ### `PyType::name` has been renamed to `PyType::qualname` -`PyType::name` has been renamed to `PyType::qualname` to indicate that it does indeed return the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name), matching the `__qualname__` attribute. The newly added `PyType::name` yields the full name including the module name now which corresponds to `__module__.__name__` on the level of attributes. +
+Click to expand + +`PyType::name` has been renamed to `PyType::qualname` to indicate that it does indeed return the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name), matching the `__qualname__` attribute. +The newly added `PyType::name` yields the full name including the module name now which corresponds to `__module__.__name__` on the level of attributes. +
+ +### `PyCell` has been deprecated + +
+Click to expand + +Interactions with Python objects implemented in Rust no longer need to go though `PyCell`. +Instead interactions with Python object now consistently go through `Bound` or `Py` independently of whether `T` is native Python object or a `#[pyclass]` implemented in Rust. +Use `Bound::new` or `Py::new` respectively to create and `Bound::borrow(_mut)` / `Py::borrow(_mut)` to borrow the Rust object. +
+ +### Migrating from the GIL Refs API to `Bound` + +
+Click to expand -### Migrating from the GIL-Refs API to `Bound` +To minimise breakage of code using the GIL Refs API, the `Bound` smart pointer has been introduced by adding complements to all functions which accept or return GIL Refs. +This allows code to migrate by replacing the deprecated APIs with the new ones. -To minimise breakage of code using the GIL-Refs API, the `Bound` smart pointer has been introduced by adding complements to all functions which accept or return GIL Refs. This allows code to migrate by replacing the deprecated APIs with the new ones. +To identify what to migrate, temporarily switch off the `gil-refs` feature to see deprecation warnings on [almost](#cases-where-pyo3-cannot-emit-gil-ref-deprecation-warnings) all uses of APIs accepting and producing GIL Refs . +Over one or more PRs it should be possible to follow the deprecation hints to update code. +Depending on your development environment, switching off the `gil-refs` feature may introduce [some very targeted breakages](#deactivating-the-gil-refs-feature), so you may need to fixup those first. For example, the following APIs have gained updated variants: -- `PyList::new`, `PyTyple::new` and similar constructors have replacements `PyList::new_bound`, `PyTuple::new_bound` etc. + +- `PyList::new`, `PyTuple::new` and similar constructors have replacements `PyList::new_bound`, `PyTuple::new_bound` etc. - `FromPyObject::extract` has a new `FromPyObject::extract_bound` (see the section below) - The `PyTypeInfo` trait has had new `_bound` methods added to accept / return `Bound`. Because the new `Bound` API brings ownership out of the PyO3 framework and into user code, there are a few places where user code is expected to need to adjust while switching to the new API: + - Code will need to add the occasional `&` to borrow the new smart pointer as `&Bound` to pass these types around (or use `.clone()` at the very small cost of increasing the Python reference count) - `Bound` and `Bound` cannot support indexing with `list[0]`, you should use `list.get_item(0)` instead. -- `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". Similarly `Bound::get_borrowed_item` is more efficient than `Bound::get_item` for the same reason. -- `&Bound` does not implement `FromPyObject` (although it might be possible to do this in the future once the GIL Refs API is completely removed). Use `bound_any.downcast::()` instead of `bound_any.extract::<&Bound>()`. -- To convert between `&PyAny` and `&Bound` you can use the `as_borrowed()` method: +- `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. + The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". + Similarly `Bound::get_borrowed_item` is more efficient than `Bound::get_item` for the same reason. +- `&Bound` does not implement `FromPyObject` (although it might be possible to do this in the future once the GIL Refs API is completely removed). + Use `bound_any.downcast::()` instead of `bound_any.extract::<&Bound>()`. +- `Bound::to_str` now borrows from the `Bound` rather than from the `'py` lifetime, so code will need to store the smart pointer as a value in some cases where previously `&PyString` was just used as a temporary. (There are some more details relating to this in [the section below](#deactivating-the-gil-refs-feature).) +- `.extract::<&str>()` now borrows from the source Python object. + The simplest way to update is to change to `.extract::()`, which retains ownership of the Python reference. + See more information [in the section on deactivating the `gil-refs` feature](#deactivating-the-gil-refs-feature). + +To convert between `&PyAny` and `&Bound` use the `as_borrowed()` method: ```rust,ignore let gil_ref: &PyAny = ...; let bound: &Bound = &gil_ref.as_borrowed(); ``` +To convert between `Py` and `Bound` use the `bind()` / `into_bound()` methods, and `as_unbound()` / `unbind()` to go back from `Bound` to `Py`. + +```rust,ignore +let obj: Py = ...; +let bound: &Bound<'py, PyList> = obj.bind(py); +let bound: Bound<'py, PyList> = obj.into_bound(py); + +let obj: &Py = bound.as_unbound(); +let obj: Py = bound.unbind(); +``` + +
+ +⚠️ Warning: dangling pointer trap 💣 + +> Because of the ownership changes, code which uses `.as_ptr()` to convert `&PyAny` and other GIL Refs to a `*mut pyo3_ffi::PyObject` should take care to avoid creating dangling pointers now that `Bound` carries ownership. +> +> For example, the following pattern with `Option<&PyAny>` can easily create a dangling pointer when migrating to the `Bound` smart pointer: +> +> ```rust,ignore +> let opt: Option<&PyAny> = ...; +> let p: *mut ffi::PyObject = opt.map_or(std::ptr::null_mut(), |any| any.as_ptr()); +> ``` +> +> The correct way to migrate this code is to use `.as_ref()` to avoid dropping the `Bound` in the `map_or` closure: +> +> ```rust,ignore +> let opt: Option> = ...; +> let p: *mut ffi::PyObject = opt.as_ref().map_or(std::ptr::null_mut(), Bound::as_ptr); +> ``` +
+ #### Migrating `FromPyObject` implementations -`FromPyObject` has had a new method `extract_bound` which takes `&Bound<'py, PyAny>` as an argument instead of `&PyAny`. Both `extract` and `extract_bound` have been given default implementations in terms of the other, to avoid breaking code immediately on update to 0.21. +`FromPyObject` has had a new method `extract_bound` which takes `&Bound<'py, PyAny>` as an argument instead of `&PyAny`. +Both `extract` and `extract_bound` have been given default implementations in terms of the other, to avoid breaking code immediately on update to 0.21. All implementations of `FromPyObject` should be switched from `extract` to `extract_bound`. @@ -252,17 +1113,117 @@ impl<'py> FromPyObject<'py> for MyType { The expectation is that in 0.22 `extract_bound` will have the default implementation removed and in 0.23 `extract` will be removed. +#### Cases where PyO3 cannot emit GIL Ref deprecation warnings +Despite a large amount of deprecations warnings produced by PyO3 to aid with the transition from GIL Refs to the Bound API, there are a few cases where PyO3 cannot automatically warn on uses of GIL Refs. +It is worth checking for these cases manually after the deprecation warnings have all been addressed: + +- Individual implementations of the `FromPyObject` trait cannot be deprecated, so PyO3 cannot warn about uses of code patterns like `.extract<&PyAny>()` which produce a GIL Ref. +- GIL Refs in `#[pyfunction]` arguments emit a warning, but if the GIL Ref is wrapped inside another container such as `Vec<&PyAny>` then PyO3 cannot warn against this. +- The `wrap_pyfunction!(function)(py)` deferred argument form of the `wrap_pyfunction` macro taking `py: Python<'py>` produces a GIL Ref, and due to limitations in type inference PyO3 cannot warn against this specific case. + +
+ +### Deactivating the `gil-refs` feature + +
+Click to expand + +As a final step of migration, deactivating the `gil-refs` feature will set up code for best performance and is intended to set up a forward-compatible API for PyO3 0.22. + +At this point code that needed to manage GIL Ref memory can safely remove uses of `GILPool` (which are constructed by calls to `Python::new_pool` and `Python::with_pool`). +Deprecation warnings will highlight these cases. + +There is just one case of code that changes upon disabling these features: `FromPyObject` trait implementations for types that borrow directly from the input data cannot be implemented by PyO3 without GIL Refs (while the GIL Refs API is in the process of being removed). +The main types affected are `&str`, `Cow<'_, str>`, `&[u8]`, `Cow<'_, u8>`. + +To make PyO3's core functionality continue to work while the GIL Refs API is in the process of being removed, disabling the `gil-refs` feature moves the implementations of `FromPyObject` for `&str`, `Cow<'_, str>`, `&[u8]`, `Cow<'_, u8>` to a new temporary trait `FromPyObjectBound`. +This trait is the expected future form of `FromPyObject` and has an additional lifetime `'a` to enable these types to borrow data from Python objects. + +PyO3 0.21 has introduced the [`PyBackedStr`]({{#PYO3_DOCS_URL}}/pyo3/pybacked/struct.PyBackedStr.html) and [`PyBackedBytes`]({{#PYO3_DOCS_URL}}/pyo3/pybacked/struct.PyBackedBytes.html) types to help with this case. +The easiest way to avoid lifetime challenges from extracting `&str` is to use these. +For more complex types like `Vec<&str>`, is now impossible to extract directly from a Python object and `Vec` is the recommended upgrade path. + +A key thing to note here is because extracting to these types now ties them to the input lifetime, some extremely common patterns may need to be split into multiple Rust lines. +For example, the following snippet of calling `.extract::<&str>()` directly on the result of `.getattr()` needs to be adjusted when deactivating the `gil-refs` feature. + +Before: + +```rust,ignore +# #[cfg(feature = "gil-refs")] { +# use pyo3::prelude::*; +# use pyo3::types::{PyList, PyType}; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +#[allow(deprecated)] // GIL Ref API +let obj: &'py PyType = py.get_type::(); +let name: &'py str = obj.getattr("__name__")?.extract()?; +assert_eq!(name, "list"); +# Ok(()) +# } +# Python::with_gil(example).unwrap(); +# } +``` + +After: + +```rust,ignore +# #[cfg(any(not(Py_LIMITED_API), Py_3_10))] { +# use pyo3::prelude::*; +# use pyo3::types::{PyList, PyType}; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +let obj: Bound<'py, PyType> = py.get_type_bound::(); +let name_obj: Bound<'py, PyAny> = obj.getattr("__name__")?; +// the lifetime of the data is no longer `'py` but the much shorter +// lifetime of the `name_obj` smart pointer above +let name: &'_ str = name_obj.extract()?; +assert_eq!(name, "list"); +# Ok(()) +# } +# Python::with_gil(example).unwrap(); +# } +``` + +To avoid needing to worry about lifetimes at all, it is also possible to use the new `PyBackedStr` type, which stores a reference to the Python `str` without a lifetime attachment. +In particular, `PyBackedStr` helps for `abi3` builds for Python older than 3.10. +Due to limitations in the `abi3` CPython API for those older versions, PyO3 cannot offer a `FromPyObjectBound` implementation for `&str` on those versions. +The easiest way to migrate for older `abi3` builds is to replace any cases of `.extract::<&str>()` with `.extract::()`. +Alternatively, use `.extract::>()`, `.extract::()` to copy the data into Rust. + +The following example uses the same snippet as those just above, but this time the final extracted type is `PyBackedStr`: + +```rust,ignore +# use pyo3::prelude::*; +# use pyo3::types::{PyList, PyType}; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +use pyo3::pybacked::PyBackedStr; +let obj: Bound<'py, PyType> = py.get_type_bound::(); +let name: PyBackedStr = obj.getattr("__name__")?.extract()?; +assert_eq!(&*name, "list"); +# Ok(()) +# } +# Python::with_gil(example).unwrap(); +``` + +
## from 0.19.* to 0.20 ### Drop support for older technologies -PyO3 0.20 has increased minimum Rust version to 1.56. This enables use of newer language features and simplifies maintenance of the project. +
+Click to expand + +PyO3 0.20 has increased minimum Rust version to 1.56. +This enables use of newer language features and simplifies maintenance of the project. +
### `PyDict::get_item` now returns a `Result` -`PyDict::get_item` in PyO3 0.19 and older was implemented using a Python API which would suppress all exceptions and return `None` in those cases. This included errors in `__hash__` and `__eq__` implementations of the key being looked up. +
+Click to expand + +`PyDict::get_item` in PyO3 0.19 and older was implemented using a Python API which would suppress all exceptions and return `None` in those cases. +This included errors in `__hash__` and `__eq__` implementations of the key being looked up. Newer recommendations by the Python core developers advise against using these APIs which suppress exceptions, instead allowing exceptions to bubble upwards. `PyDict::get_item_with_error` already implemented this recommended behavior, so that API has been renamed to `PyDict::get_item`. @@ -282,7 +1243,10 @@ Python::with_gil(|py| { // `b` is not in the dictionary assert!(dict.get_item("b").is_none()); // `dict` is not hashable, so this fails with a `TypeError` - assert!(dict.get_item_with_error(dict).unwrap_err().is_instance_of::(py)); + assert!(dict + .get_item_with_error(dict) + .unwrap_err() + .is_instance_of::(py)); }); # } ``` @@ -303,16 +1267,26 @@ Python::with_gil(|py| -> PyResult<()> { // `b` is not in the dictionary assert!(dict.get_item("b")?.is_none()); // `dict` is not hashable, so this fails with a `TypeError` - assert!(dict.get_item(dict).unwrap_err().is_instance_of::(py)); + assert!(dict + .get_item(dict) + .unwrap_err() + .is_instance_of::(py)); Ok(()) }); # } ``` +
+ ### Required arguments are no longer accepted after optional arguments -[Trailing `Option` arguments](./function/signature.md#trailing-optional-arguments) have an automatic default of `None`. To avoid unwanted changes when modifying function signatures, in PyO3 0.18 it was deprecated to have a required argument after an `Option` argument without using `#[pyo3(signature = (...))]` to specify the intended defaults. In PyO3 0.20, this becomes a hard error. +
+Click to expand + +Trailing `Option` arguments have an automatic default of `None`. +To avoid unwanted changes when modifying function signatures, in PyO3 0.18 it was deprecated to have a required argument after an `Option` argument without using `#[pyo3(signature = (...))]` to specify the intended defaults. +In PyO3 0.20, this becomes a hard error. Before: @@ -325,7 +1299,7 @@ fn x_or_y(x: Option, y: u64) -> u64 { After: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; @@ -336,9 +1310,15 @@ fn x_or_y(x: Option, y: u64) -> u64 { } ``` +
+ ### Remove deprecated function forms -In PyO3 0.18 the `#[args]` attribute for `#[pymethods]`, and directly specifying the function signature in `#[pyfunction]`, was deprecated. This functionality has been removed in PyO3 0.20. +
+Click to expand + +In PyO3 0.18 the `#[args]` attribute for `#[pymethods]`, and directly specifying the function signature in `#[pyfunction]`, was deprecated. +This functionality has been removed in PyO3 0.20. Before: @@ -352,7 +1332,7 @@ fn add(a: u64, b: u64) -> u64 { After: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; @@ -363,23 +1343,38 @@ fn add(a: u64, b: u64) -> u64 { } ``` +
+ ### `IntoPyPointer` trait removed +
+Click to expand + The trait `IntoPyPointer`, which provided the `into_ptr` method on many types, has been removed. `into_ptr` is now available as an inherent method on all types that previously implemented this trait. +
### `AsPyPointer` now `unsafe` trait +
+Click to expand + The trait `AsPyPointer` is now `unsafe trait`, meaning any external implementation of it must be marked as `unsafe impl`, and ensure that they uphold the invariant of returning valid pointers. +
## from 0.18.* to 0.19 ### Access to `Python` inside `__traverse__` implementations are now forbidden -During `__traverse__` implementations for Python's Garbage Collection it is forbidden to do anything other than visit the members of the `#[pyclass]` being traversed. This means making Python function calls or other API calls are forbidden. +
+Click to expand + +During `__traverse__` implementations for Python's Garbage Collection it is forbidden to do anything other than visit the members of the `#[pyclass]` being traversed. +This means making Python function calls or other API calls are forbidden. Previous versions of PyO3 would allow access to `Python` (e.g. via `Python::with_gil`), which could cause the Python interpreter to crash or otherwise confuse the garbage collection algorithm. -Attempts to acquire the GIL will now panic. See [#3165](https://github.com/PyO3/pyo3/issues/3165) for more detail. +Attempts to acquire the GIL will now panic. +See [#3165](https://github.com/PyO3/pyo3/issues/3165) for more detail. ```rust,ignore # use pyo3::prelude::*; @@ -394,11 +1389,16 @@ impl SomeClass { } ``` +
+ ### Smarter `anyhow::Error` / `eyre::Report` conversion when inner error is "simple" `PyErr` +
+Click to expand + When converting from `anyhow::Error` or `eyre::Report` to `PyErr`, if the inner error is a "simple" `PyErr` (with no source error), then the inner error will be used directly as the `PyErr` instead of wrapping it in a new `PyRuntimeError` with the original information converted into a string. -```rust +```rust,ignore # #[cfg(feature = "anyhow")] # #[allow(dead_code)] # mod anyhow_only { @@ -432,10 +1432,14 @@ Before, the above code would have printed `RuntimeError('ValueError: original er After, the same code will print `ValueError: original error message`, which is more straightforward. However, if the `anyhow::Error` or `eyre::Report` has a source, then the original exception will still be wrapped in a `PyRuntimeError`. +
### The deprecated `Python::acquire_gil` was removed and `Python::with_gil` must be used instead -While the API provided by [`Python::acquire_gil`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html#method.acquire_gil) seems convenient, it is somewhat brittle as the design of the GIL token [`Python`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html) relies on proper nesting and panics if not used correctly, e.g. +
+Click to expand + +While the API provided by [`Python::acquire_gil`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html#method.acquire_gil) seems convenient, it is somewhat brittle as the design of the [`Python`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html) token relies on proper nesting and panics if not used correctly, e.g. ```rust,ignore # #![allow(dead_code, deprecated)] @@ -465,9 +1469,9 @@ drop(first); drop(second); ``` -The replacement is [`Python::with_gil`]() which is more cumbersome but enforces the proper nesting by design, e.g. +The replacement is [`Python::with_gil`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html#method.with_gil) which is more cumbersome but enforces the proper nesting by design, e.g. -```rust +```rust,ignore # #![allow(dead_code)] # use pyo3::prelude::*; @@ -486,13 +1490,13 @@ impl Object { } } -// It either forces us to release the GIL before aquiring it again. +// It either forces us to release the GIL before acquiring it again. let first = Python::with_gil(|py| Object::new(py)); let second = Python::with_gil(|py| Object::new(py)); drop(first); drop(second); -// Or it ensure releasing the inner lock before the outer one. +// Or it ensures releasing the inner lock before the outer one. Python::with_gil(|py| { let first = Object::new(py); let second = Python::with_gil(|py| Object::new(py)); @@ -501,19 +1505,26 @@ Python::with_gil(|py| { }); ``` -Furthermore, `Python::acquire_gil` provides ownership of a `GILGuard` which can be freely stored and passed around. This is usually not helpful as it may keep the lock held for a long time thereby blocking progress in other parts of the program. Due to the generative lifetime attached to the GIL token supplied by `Python::with_gil`, the problem is avoided as the GIL token can only be passed down the call chain. Often, this issue can also be avoided entirely as any GIL-bound reference `&'py PyAny` implies access to a GIL token `Python<'py>` via the [`PyAny::py`](https://docs.rs/pyo3/latest/pyo3/types/struct.PyAny.html#method.py) method. +Furthermore, `Python::acquire_gil` provides ownership of a `GILGuard` which can be freely stored and passed around. +This is usually not helpful as it may keep the lock held for a long time thereby blocking progress in other parts of the program. +Due to the generative lifetime attached to the Python token supplied by `Python::with_gil`, the problem is avoided as the Python token can only be passed down the call chain. +Often, this issue can also be avoided entirely as any GIL-bound reference `&'py PyAny` implies access to a Python token `Python<'py>` via the [`PyAny::py`](https://docs.rs/pyo3/0.22.5/pyo3/types/struct.PyAny.html#method.py) method. +
## from 0.17.* to 0.18 ### Required arguments after `Option<_>` arguments will no longer be automatically inferred +
+Click to expand + In `#[pyfunction]` and `#[pymethods]`, if a "required" function input such as `i32` came after an `Option<_>` input, then the `Option<_>` would be implicitly treated as required. (All trailing `Option<_>` arguments were treated as optional with a default value of `None`). Starting with PyO3 0.18, this is deprecated and a future PyO3 version will require a [`#[pyo3(signature = (...))]` option](./function/signature.md) to explicitly declare the programmer's intention. Before, x in the below example would be required to be passed from Python code: -```rust,compile_fail +```rust,compile_fail,ignore # #![allow(dead_code)] # use pyo3::prelude::*; @@ -523,7 +1534,7 @@ fn required_argument_after_option(x: Option, y: i32) {} After, specify the intended Python signature explicitly: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; @@ -536,11 +1547,18 @@ fn required_argument_after_option_a(x: Option, y: i32) {} fn required_argument_after_option_b(x: Option, y: i32) {} ``` +
+ ### `__text_signature__` is now automatically generated for `#[pyfunction]` and `#[pymethods]` +
+Click to expand + The [`#[pyo3(text_signature = "...")]` option](./function/signature.md#making-the-function-signature-available-to-python) was previously the only supported way to set the `__text_signature__` attribute on generated Python functions. -PyO3 is now able to automatically populate `__text_signature__` for all functions automatically based on their Rust signature (or the [new `#[pyo3(signature = (...))]` option](./function/signature.md)). These automatically-generated `__text_signature__` values will currently only render `...` for all default values. Many `#[pyo3(text_signature = "...")]` options can be removed from functions when updating to PyO3 0.18, however in cases with default values a manual implementation may still be preferred for now. +PyO3 is now able to automatically populate `__text_signature__` for all functions automatically based on their Rust signature (or the [new `#[pyo3(signature = (...))]` option](./function/signature.md)). +These automatically-generated `__text_signature__` values will currently only render `...` for all default values. +Many `#[pyo3(text_signature = "...")]` options can be removed from functions when updating to PyO3 0.18, however in cases with default values a manual implementation may still be preferred for now. As examples: @@ -558,7 +1576,7 @@ fn simple_function(a: i32, b: i32, c: i32) {} fn function_with_defaults(a: i32, b: i32, c: i32) {} # fn main() { -# Python::with_gil(|py| { +# Python::attach(|py| { # let simple = wrap_pyfunction!(simple_function, py).unwrap(); # assert_eq!(simple.getattr("__text_signature__").unwrap().to_string(), "(a, b, c)"); # let defaulted = wrap_pyfunction!(function_with_defaults, py).unwrap(); @@ -567,31 +1585,30 @@ fn function_with_defaults(a: i32, b: i32, c: i32) {} # } ``` +
+ ## from 0.16.* to 0.17 ### Type checks have been changed for `PyMapping` and `PySequence` types +
+Click to expand + Previously the type checks for `PyMapping` and `PySequence` (implemented in `PyTryFrom`) used the Python C-API functions `PyMapping_Check` and `PySequence_Check`. Unfortunately these functions are not sufficient for distinguishing such types, leading to inconsistent behavior (see [pyo3/pyo3#2072](https://github.com/PyO3/pyo3/issues/2072)). -PyO3 0.17 changes these downcast checks to explicitly test if the type is a -subclass of the corresponding abstract base class `collections.abc.Mapping` or -`collections.abc.Sequence`. Note this requires calling into Python, which may -incur a performance penalty over the previous method. If this performance -penalty is a problem, you may be able to perform your own checks and use -`try_from_unchecked` (unsafe). +PyO3 0.17 changes these downcast checks to explicitly test if the type is a subclass of the corresponding abstract base class `collections.abc.Mapping` or `collections.abc.Sequence`. +Note this requires calling into Python, which may incur a performance penalty over the previous method. +If this performance penalty is a problem, you may be able to perform your own checks and use `try_from_unchecked` (unsafe). -Another side-effect is that a pyclass defined in Rust with PyO3 will need to -be _registered_ with the corresponding Python abstract base class for -downcasting to succeed. `PySequence::register` and `PyMapping:register` have -been added to make it easy to do this from Rust code. These are equivalent to -calling `collections.abc.Mapping.register(MappingPyClass)` or -`collections.abc.Sequence.register(SequencePyClass)` from Python. +Another side-effect is that a pyclass defined in Rust with PyO3 will need to be _registered_ with the corresponding Python abstract base class for downcasting to succeed. `PySequence::register` and `PyMapping:register` have been added to make it easy to do this from Rust code. +These are equivalent to calling `collections.abc.Mapping.register(MappingPyClass)` or `collections.abc.Sequence.register(SequencePyClass)` from Python. For example, for a mapping class defined in Rust: + ```rust,compile_fail use pyo3::prelude::*; use std::collections::HashMap; @@ -611,6 +1628,7 @@ impl Mapping { ``` You must register the class with `collections.abc.Mapping` before the downcast will work: + ```rust,compile_fail let m = Py::new(py, Mapping { index }).unwrap(); assert!(m.as_ref(py).downcast::().is_err()); @@ -619,17 +1637,26 @@ assert!(m.as_ref(py).downcast::().is_ok()); ``` Note that this requirement may go away in the future when a pyclass is able to inherit from the abstract base class directly (see [pyo3/pyo3#991](https://github.com/PyO3/pyo3/issues/991)). +
### The `multiple-pymethods` feature now requires Rust 1.62 -Due to limitations in the `inventory` crate which the `multiple-pymethods` feature depends on, this feature now -requires Rust 1.62. For more information see [dtolnay/inventory#32](https://github.com/dtolnay/inventory/issues/32). +
+Click to expand + +Due to limitations in the `inventory` crate which the `multiple-pymethods` feature depends on, this feature now requires Rust 1.62. +For more information see [dtolnay/inventory#32](https://github.com/dtolnay/inventory/issues/32). +
### Added `impl IntoPy> for &str` +
+Click to expand + This may cause inference errors. Before: + ```rust,compile_fail # use pyo3::prelude::*; # @@ -643,7 +1670,8 @@ Python::with_gil(|py| { After, some type annotations may be necessary: -```rust +```rust,ignore +# #![allow(deprecated)] # use pyo3::prelude::*; # # fn main() { @@ -653,13 +1681,24 @@ Python::with_gil(|py| { # } ``` +
+ ### The `pyproto` feature is now disabled by default -In preparation for removing the deprecated `#[pyproto]` attribute macro in a future PyO3 version, it is now gated behind an opt-in feature flag. This also gives a slight saving to compile times for code which does not use the deprecated macro. +
+Click to expand + +In preparation for removing the deprecated `#[pyproto]` attribute macro in a future PyO3 version, it is now gated behind an opt-in feature flag. +This also gives a slight saving to compile times for code which does not use the deprecated macro. +
### `PyTypeObject` trait has been deprecated -The `PyTypeObject` trait already was near-useless; almost all functionality was already on the `PyTypeInfo` trait, which `PyTypeObject` had a blanket implementation based upon. In PyO3 0.17 the final method, `PyTypeObject::type_object` was moved to `PyTypeInfo::type_object`. +
+Click to expand + +The `PyTypeObject` trait already was near-useless; almost all functionality was already on the `PyTypeInfo` trait, which `PyTypeObject` had a blanket implementation based upon. +In PyO3 0.17 the final method, `PyTypeObject::type_object` was moved to `PyTypeInfo::type_object`. To migrate, update trait bounds and imports from `PyTypeObject` to `PyTypeInfo`. @@ -688,25 +1727,52 @@ fn get_type_object(py: Python<'_>) -> &PyType { # Python::with_gil(|py| { get_type_object::(py); }); ``` +
+ ### `impl IntoPy for [T; N]` now requires `T: IntoPy` rather than `T: ToPyObject` -If this leads to errors, simply implement `IntoPy`. Because pyclasses already implement `IntoPy`, you probably don't need to worry about this. +
+Click to expand + +If this leads to errors, simply implement `IntoPy`. +Because pyclasses already implement `IntoPy`, you probably don't need to worry about this. +
### Each `#[pymodule]` can now only be initialized once per process -To make PyO3 modules sound in the presence of Python sub-interpreters, for now it has been necessary to explicitly disable the ability to initialize a `#[pymodule]` more than once in the same process. Attempting to do this will now raise an `ImportError`. +
+Click to expand + +To make PyO3 modules sound in the presence of Python sub-interpreters, for now it has been necessary to explicitly disable the ability to initialize a `#[pymodule]` more than once in the same process. +Attempting to do this will now raise an `ImportError`. +
## from 0.15.* to 0.16 + + ### Drop support for older technologies -PyO3 0.16 has increased minimum Rust version to 1.48 and minimum Python version to 3.7. This enables use of newer language features (enabling some of the other additions in 0.16) and simplifies maintenance of the project. + + +
+Click to expand + +PyO3 0.16 has increased minimum Rust version to 1.48 and minimum Python version to 3.7. +This enables use of newer language features (enabling some of the other additions in 0.16) and simplifies maintenance of the project. +
### `#[pyproto]` has been deprecated -In PyO3 0.15, the `#[pymethods]` attribute macro gained support for implementing "magic methods" such as `__str__` (aka "dunder" methods). This implementation was not quite finalized at the time, with a few edge cases to be decided upon. The existing `#[pyproto]` attribute macro was left untouched, because it covered these edge cases. +
+Click to expand + +In PyO3 0.15, the `#[pymethods]` attribute macro gained support for implementing "magic methods" such as `__str__` (aka "dunder" methods). +This implementation was not quite finalized at the time, with a few edge cases to be decided upon. +The existing `#[pyproto]` attribute macro was left untouched, because it covered these edge cases. -In PyO3 0.16, the `#[pymethods]` implementation has been completed and is now the preferred way to implement magic methods. To allow the PyO3 project to move forward, `#[pyproto]` has been deprecated (with expected removal in PyO3 0.18). +In PyO3 0.16, the `#[pymethods]` implementation has been completed and is now the preferred way to implement magic methods. +To allow the PyO3 project to move forward, `#[pyproto]` has been deprecated (with expected removal in PyO3 0.18). Migration from `#[pyproto]` to `#[pymethods]` is straightforward; copying the existing methods directly from the `#[pyproto]` trait implementation is all that is needed in most cases. @@ -756,8 +1822,13 @@ impl MyClass { } ``` +
+ ### Removed `PartialEq` for object wrappers +
+Click to expand + The Python object wrappers `Py` and `PyAny` had implementations of `PartialEq` so that `object_a == object_b` would compare the Python objects for pointer equality, which corresponds to the `is` operator, not the `==` operator in @@ -768,14 +1839,21 @@ wrapper type for `object_a` and `object_b`; you can now directly compare a To check for Python object equality (the Python `==` operator), use the new method `eq()`. +
### Container magic methods now match Python behavior -In PyO3 0.15, `__getitem__`, `__setitem__` and `__delitem__` in `#[pymethods]` would generate only the _mapping_ implementation for a `#[pyclass]`. To match the Python behavior, these methods now generate both the _mapping_ **and** _sequence_ implementations. +
+Click to expand -This means that classes implementing these `#[pymethods]` will now also be treated as sequences, same as a Python `class` would be. Small differences in behavior may result: - - PyO3 will allow instances of these classes to be cast to `PySequence` as well as `PyMapping`. - - Python will provide a default implementation of `__iter__` (if the class did not have one) which repeatedly calls `__getitem__` with integers (starting at 0) until an `IndexError` is raised. +In PyO3 0.15, `__getitem__`, `__setitem__` and `__delitem__` in `#[pymethods]` would generate only the _mapping_ implementation for a `#[pyclass]`. +To match the Python behavior, these methods now generate both the _mapping_ **and** _sequence_ implementations. + +This means that classes implementing these `#[pymethods]` will now also be treated as sequences, same as a Python `class` would be. +Small differences in behavior may result: + +- PyO3 will allow instances of these classes to be cast to `PySequence` as well as `PyMapping`. +- Python will provide a default implementation of `__iter__` (if the class did not have one) which repeatedly calls `__getitem__` with integers (starting at 0) until an `IndexError` is raised. To explain this in detail, consider the following Python class: @@ -793,14 +1871,21 @@ class ExampleContainer: This class implements a Python [sequence](https://docs.python.org/3/glossary.html#term-sequence). -The `__len__` and `__getitem__` methods are also used to implement a Python [mapping](https://docs.python.org/3/glossary.html#term-mapping). In the Python C-API, these methods are not shared: the sequence `__len__` and `__getitem__` are defined by the `sq_length` and `sq_item` slots, and the mapping equivalents are `mp_length` and `mp_subscript`. There are similar distinctions for `__setitem__` and `__delitem__`. +The `__len__` and `__getitem__` methods are also used to implement a Python [mapping](https://docs.python.org/3/glossary.html#term-mapping). +In the Python C-API, these methods are not shared: the sequence `__len__` and `__getitem__` are defined by the `sq_length` and `sq_item` slots, and the mapping equivalents are `mp_length` and `mp_subscript`. +There are similar distinctions for `__setitem__` and `__delitem__`. -Because there is no such distinction from Python, implementing these methods will fill the mapping and sequence slots simultaneously. A Python class with `__len__` implemented, for example, will have both the `sq_length` and `mp_length` slots filled. +Because there is no such distinction from Python, implementing these methods will fill the mapping and sequence slots simultaneously. +A Python class with `__len__` implemented, for example, will have both the `sq_length` and `mp_length` slots filled. The PyO3 behavior in 0.16 has been changed to be closer to this Python behavior by default. +
### `wrap_pymodule!` and `wrap_pyfunction!` now respect privacy correctly +
+Click to expand + Prior to PyO3 0.16 the `wrap_pymodule!` and `wrap_pyfunction!` macros could use modules and functions whose defining `fn` was not reachable according Rust privacy rules. For example, the following code was legal before 0.16, but in 0.16 is rejected because the `wrap_pymodule!` macro cannot access the `private_submodule` function: @@ -827,7 +1912,7 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { To fix it, make the private submodule visible, e.g. with `pub` or `pub(crate)`. -```rust +```rust,ignore mod foo { use pyo3::prelude::*; @@ -848,10 +1933,15 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { } ``` +
+ ## from 0.14.* to 0.15 ### Changes in sequence indexing +
+Click to expand + For all types that take sequence indices (`PyList`, `PyTuple` and `PySequence`), the API has been made consistent to only take `usize` indices, for consistency with Rust's indexing conventions. Negative indices, which were only @@ -862,7 +1952,7 @@ Further, the `get_item` methods now always return a `PyResult` instead of panicking on invalid indices. The `Index` trait has been implemented instead, and provides the same panic behavior as on Rust vectors. -Note that *slice* indices (accepted by `PySequence::get_slice` and other) still +Note that _slice_ indices (accepted by `PySequence::get_slice` and other) still inherit the Python behavior of clamping the indices to the actual length, and not panicking/returning an error on out of range indices. @@ -870,7 +1960,7 @@ An additional advantage of using Rust's indexing conventions for these types is that these types can now also support Rust's indexing operators as part of a consistent API: -```rust +```rust,ignore #![allow(deprecated)] use pyo3::{Python, types::PyList}; @@ -880,21 +1970,37 @@ Python::with_gil(|py| { }); ``` +
+ ## from 0.13.* to 0.14 ### `auto-initialize` feature is now opt-in +
+Click to expand + For projects embedding Python in Rust, PyO3 no longer automatically initializes a Python interpreter on the first call to `Python::with_gil` (or `Python::acquire_gil`) unless the [`auto-initialize` feature](features.md#auto-initialize) is enabled. +
### New `multiple-pymethods` feature -`#[pymethods]` have been reworked with a simpler default implementation which removes the dependency on the `inventory` crate. This reduces dependencies and compile times for the majority of users. +
+Click to expand + +`#[pymethods]` have been reworked with a simpler default implementation which removes the dependency on the `inventory` crate. +This reduces dependencies and compile times for the majority of users. -The limitation of the new default implementation is that it cannot support multiple `#[pymethods]` blocks for the same `#[pyclass]`. If you need this functionality, you must enable the `multiple-pymethods` feature which will switch `#[pymethods]` to the inventory-based implementation. +The limitation of the new default implementation is that it cannot support multiple `#[pymethods]` blocks for the same `#[pyclass]`. +If you need this functionality, you must enable the `multiple-pymethods` feature which will switch `#[pymethods]` to the inventory-based implementation. +
### Deprecated `#[pyproto]` methods -Some protocol (aka `__dunder__`) methods such as `__bytes__` and `__format__` have been possible to implement two ways in PyO3 for some time: via a `#[pyproto]` (e.g. `PyObjectProtocol` for the methods listed here), or by writing them directly in `#[pymethods]`. This is only true for a handful of the `#[pyproto]` methods (for technical reasons to do with the way PyO3 currently interacts with the Python C-API). +
+Click to expand + +Some protocol (aka `__dunder__`) methods such as `__bytes__` and `__format__` have been possible to implement two ways in PyO3 for some time: via a `#[pyproto]` (e.g. `PyObjectProtocol` for the methods listed here), or by writing them directly in `#[pymethods]`. +This is only true for a handful of the `#[pyproto]` methods (for technical reasons to do with the way PyO3 currently interacts with the Python C-API). In the interest of having only one way to do things, the `#[pyproto]` forms of these methods have been deprecated. @@ -919,7 +2025,7 @@ impl PyObjectProtocol for MyClass { After: -```rust +```rust,no_run use pyo3::prelude::*; #[pyclass] @@ -933,76 +2039,113 @@ impl MyClass { } ``` +
+ ## from 0.12.* to 0.13 ### Minimum Rust version increased to Rust 1.45 -PyO3 `0.13` makes use of new Rust language features stabilized between Rust 1.40 and Rust 1.45. If you are using a Rust compiler older than Rust 1.45, you will need to update your toolchain to be able to continue using PyO3. +
+Click to expand + +PyO3 `0.13` makes use of new Rust language features stabilized between Rust 1.40 and Rust 1.45. +If you are using a Rust compiler older than Rust 1.45, you will need to update your toolchain to be able to continue using PyO3. +
### Runtime changes to support the CPython limited API -In PyO3 `0.13` support was added for compiling against the CPython limited API. This had a number of implications for _all_ PyO3 users, described here. +
+Click to expand -The largest of these is that all types created from PyO3 are what CPython calls "heap" types. The specific implications of this are: +In PyO3 `0.13` support was added for compiling against the CPython limited API. +This had a number of implications for _all_ PyO3 users, described here. + +The largest of these is that all types created from PyO3 are what CPython calls "heap" types. +The specific implications of this are: - If you wish to subclass one of these types _from Rust_ you must mark it `#[pyclass(subclass)]`, as you would if you wished to allow subclassing it from Python code. - Type objects are now mutable - Python code can set attributes on them. - `__module__` on types without `#[pyclass(module="mymodule")]` no longer returns `builtins`, it now raises `AttributeError`. +
## from 0.11.* to 0.12 ### `PyErr` has been reworked -In PyO3 `0.12` the `PyErr` type has been re-implemented to be significantly more compatible with -the standard Rust error handling ecosystem. Specifically `PyErr` now implements -`Error + Send + Sync`, which are the standard traits used for error types. +
+Click to expand + +In PyO3 `0.12` the `PyErr` type has been re-implemented to be significantly more compatible with the standard Rust error handling ecosystem. +Specifically `PyErr` now implements `Error + Send + Sync`, which are the standard traits used for error types. -While this has necessitated the removal of a number of APIs, the resulting `PyErr` type should now -be much more easier to work with. The following sections list the changes in detail and how to -migrate to the new APIs. +While this has necessitated the removal of a number of APIs, the resulting `PyErr` type should now be much more easier to work with. +The following sections list the changes in detail and how to migrate to the new APIs. +
#### `PyErr::new` and `PyErr::from_type` now require `Send + Sync` for their argument -For most uses no change will be needed. If you are trying to construct `PyErr` from a value that is -not `Send + Sync`, you will need to first create the Python object and then use -`PyErr::from_instance`. +
+Click to expand + +For most uses no change will be needed. +If you are trying to construct `PyErr` from a value that is not `Send + Sync`, you will need to first create the Python object and then use `PyErr::from_instance`. Similarly, any types which implemented `PyErrArguments` will now need to be `Send + Sync`. +
#### `PyErr`'s contents are now private +
+Click to expand + It is no longer possible to access the fields `.ptype`, `.pvalue` and `.ptraceback` of a `PyErr`. You should instead now use the new methods `PyErr::ptype`, `PyErr::pvalue` and `PyErr::ptraceback`. +
#### `PyErrValue` and `PyErr::from_value` have been removed +
+Click to expand + As these were part the internals of `PyErr` which have been reworked, these APIs no longer exist. If you used this API, it is recommended to use `PyException::new_err` (see [the section on Exception types](#exception-types-have-been-reworked)). +
#### `Into>` for `PyErr` has been removed -This implementation was redundant. Just construct the `Result::Err` variant directly. +
+Click to expand + +This implementation was redundant. +Just construct the `Result::Err` variant directly. Before: + ```rust,compile_fail let result: PyResult<()> = PyErr::new::("error message").into(); ``` After (also using the new reworked exception types; see the following section): -```rust + +```rust,no_run # use pyo3::{PyResult, exceptions::PyTypeError}; let result: PyResult<()> = Err(PyTypeError::new_err("error message")); ``` +
+ ### Exception types have been reworked -Previously exception types were zero-sized marker types purely used to construct `PyErr`. In PyO3 -0.12, these types have been replaced with full definitions and are usable in the same way as `PyAny`, `PyDict` etc. This -makes it possible to interact with Python exception objects. +
+Click to expand -The new types also have names starting with the "Py" prefix. For example, before: +Previously exception types were zero-sized marker types purely used to construct `PyErr`. +In PyO3 0.12, these types have been replaced with full definitions and are usable in the same way as `PyAny`, `PyDict` etc. This makes it possible to interact with Python exception objects. + +The new types also have names starting with the "Py" prefix. +For example, before: ```rust,ignore let err: PyErr = TypeError::py_err("error message"); @@ -1029,15 +2172,21 @@ assert_eq!( # }).unwrap(); ``` +
+ ### `FromPy` has been removed -To simplify the PyO3 conversion traits, the `FromPy` trait has been removed. Previously there were -two ways to define the to-Python conversion for a type: -`FromPy for PyObject` and `IntoPy for T`. + +
+Click to expand + +To simplify the PyO3 conversion traits, the `FromPy` trait has been removed. +Previously there were two ways to define the to-Python conversion for a type: `FromPy for PyObject` and `IntoPy for T`. Now there is only one way to define the conversion, `IntoPy`, so downstream crates may need to adjust accordingly. Before: + ```rust,compile_fail # use pyo3::prelude::*; struct MyPyObjectWrapper(PyObject); @@ -1050,10 +2199,13 @@ impl FromPy for PyObject { ``` After -```rust + +```rust,ignore # use pyo3::prelude::*; +# #[allow(dead_code)] struct MyPyObjectWrapper(PyObject); +# #[allow(deprecated)] impl IntoPy for MyPyObjectWrapper { fn into_py(self, _py: Python<'_>) -> PyObject { self.0 @@ -1064,6 +2216,7 @@ impl IntoPy for MyPyObjectWrapper { Similarly, code which was using the `FromPy` trait can be trivially rewritten to use `IntoPy`. Before: + ```rust,compile_fail # use pyo3::prelude::*; # Python::with_gil(|py| { @@ -1072,26 +2225,39 @@ let obj = PyObject::from_py(1.234, py); ``` After: -```rust + +```rust,ignore +# #![allow(deprecated)] # use pyo3::prelude::*; # Python::with_gil(|py| { let obj: PyObject = 1.234.into_py(py); # }) ``` +
+ ### `PyObject` is now a type alias of `Py` -This should change very little from a usage perspective. If you implemented traits for both -`PyObject` and `Py`, you may find you can just remove the `PyObject` implementation. + +
+Click to expand + +This should change very little from a usage perspective. +If you implemented traits for both `PyObject` and `Py`, you may find you can just remove the `PyObject` implementation. +
### `AsPyRef` has been removed -As `PyObject` has been changed to be just a type alias, the only remaining implementor of `AsPyRef` -was `Py`. This removed the need for a trait, so the `AsPyRef::as_ref` method has been moved to -`Py::as_ref`. + +
+Click to expand + +As `PyObject` has been changed to be just a type alias, the only remaining implementor of `AsPyRef` was `Py`. +This removed the need for a trait, so the `AsPyRef::as_ref` method has been moved to `Py::as_ref`. This should require no code changes except removing `use pyo3::AsPyRef` for code which did not use `pyo3::prelude::*`. Before: + ```rust,ignore use pyo3::{AsPyRef, Py, types::PyList}; # pyo3::Python::with_gil(|py| { @@ -1101,6 +2267,7 @@ let list_ref: &PyList = list_py.as_ref(py); ``` After: + ```rust,ignore use pyo3::{Py, types::PyList}; # pyo3::Python::with_gil(|py| { @@ -1109,12 +2276,24 @@ let list_ref: &PyList = list_py.as_ref(py); # }) ``` +
+ ## from 0.10.* to 0.11 ### Stable Rust -PyO3 now supports the stable Rust toolchain. The minimum required version is 1.39.0. + +
+Click to expand + +PyO3 now supports the stable Rust toolchain. +The minimum required version is 1.39.0. +
### `#[pyclass]` structs must now be `Send` or `unsendable` + +
+Click to expand + Because `#[pyclass]` structs can be sent between threads by the Python interpreter, they must implement `Send` or declared as `unsendable` (by `#[pyclass(unsendable)]`). Note that `unsendable` is added in PyO3 `0.11.1` and `Send` is always required in PyO3 `0.11.0`. @@ -1123,10 +2302,11 @@ This may "break" some code which previously was accepted, even though it could b There can be two fixes: 1. If you think that your `#[pyclass]` actually must be `Send`able, then let's implement `Send`. - A common, safer way is using thread-safe types. E.g., `Arc` instead of `Rc`, `Mutex` instead of - `RefCell`, and `Box` instead of `Box`. + A common, safer way is using thread-safe types. + E.g., `Arc` instead of `Rc`, `Mutex` instead of `RefCell`, and `Box` instead of `Box`. Before: + ```rust,compile_fail use pyo3::prelude::*; use std::rc::Rc; @@ -1140,7 +2320,8 @@ There can be two fixes: ``` After: - ```rust + + ```rust,ignore # #![allow(dead_code)] use pyo3::prelude::*; use std::sync::{Arc, Mutex}; @@ -1152,64 +2333,79 @@ There can be two fixes: } ``` - In situations where you cannot change your `#[pyclass]` to automatically implement `Send` - (e.g., when it contains a raw pointer), you can use `unsafe impl Send`. + In situations where you cannot change your `#[pyclass]` to automatically implement `Send` (e.g., when it contains a raw pointer), you can use `unsafe impl Send`. In such cases, care should be taken to ensure the struct is actually thread safe. See [the Rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html) for more. -2. If you think that your `#[pyclass]` should not be accessed by another thread, you can use - `unsendable` flag. A class marked with `unsendable` panics when accessed by another thread, - making it thread-safe to expose an unsendable object to the Python interpreter. +2. If you think that your `#[pyclass]` should not be accessed by another thread, you can use `unsendable` flag. + A class marked with `unsendable` panics when accessed by another thread, making it thread-safe to expose an unsendable object to the Python interpreter. Before: + ```rust,compile_fail use pyo3::prelude::*; #[pyclass] struct Unsendable { - pointers: Vec<*mut std::os::raw::c_char>, + pointers: Vec<*mut std::ffi::c_char>, } ``` After: - ```rust + + ```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; #[pyclass(unsendable)] struct Unsendable { - pointers: Vec<*mut std::os::raw::c_char>, + pointers: Vec<*mut std::ffi::c_char>, } ``` +
+ ### All `PyObject` and `Py` methods now take `Python` as an argument -Previously, a few methods such as `Object::get_refcnt` did not take `Python` as an argument (to -ensure that the Python GIL was held by the current thread). Technically, this was not sound. + +
+Click to expand + +Previously, a few methods such as `Object::get_refcnt` did not take `Python` as an argument (to ensure that the Python GIL was held by the current thread). +Technically, this was not sound. To migrate, just pass a `py` argument to any calls to these methods. Before: + ```rust,compile_fail -# pyo3::Python::with_gil(|py| { +# pyo3::Python::attach(|py| { py.None().get_refcnt(); # }) ``` After: + ```rust -# pyo3::Python::with_gil(|py| { +# pyo3::Python::attach(|py| { py.None().get_refcnt(py); # }) ``` +
+ ## from 0.9.* to 0.10 ### `ObjectProtocol` is removed + +
+Click to expand + All methods are moved to [`PyAny`]. And since now all native types (e.g., `PyList`) implements `Deref`, all you need to do is remove `ObjectProtocol` from your code. Or if you use `ObjectProtocol` by `use pyo3::prelude::*`, you have to do nothing. Before: + ```rust,compile_fail,ignore use pyo3::ObjectProtocol; @@ -1221,6 +2417,7 @@ assert_eq!(hi.len().unwrap(), 5); ``` After: + ```rust,ignore # pyo3::Python::with_gil(|py| { let obj = py.eval("lambda: 'Hi :)'", None, None).unwrap(); @@ -1229,17 +2426,29 @@ assert_eq!(hi.len().unwrap(), 5); # }) ``` +
+ ### No `#![feature(specialization)]` in user code + +
+Click to expand + While PyO3 itself still requires specialization and nightly Rust, now you don't have to use `#![feature(specialization)]` in your crate. +
## from 0.8.* to 0.9 ### `#[new]` interface + +
+Click to expand + [`PyRawObject`](https://docs.rs/pyo3/0.8.5/pyo3/type_object/struct.PyRawObject.html) is now removed and our syntax for constructors has changed. Before: + ```rust,compile_fail #[pyclass] struct MyClass {} @@ -1254,7 +2463,8 @@ impl MyClass { ``` After: -```rust + +```rust,no_run # use pyo3::prelude::*; #[pyclass] struct MyClass {} @@ -1269,10 +2479,15 @@ impl MyClass { ``` Basically you can return `Self` or `Result` directly. -For more, see [the constructor section](class.html#constructor) of this guide. +For more, see [the constructor section](class.md#constructor) of this guide. +
### PyCell -PyO3 0.9 introduces [`PyCell`], which is a [`RefCell`]-like object wrapper + +
+Click to expand + +PyO3 0.9 introduces `PyCell`, which is a [`RefCell`]-like object wrapper for ensuring Rust's rules regarding aliasing of references are upheld. For more detail, see the [Rust Book's section on Rust's rules of references](https://doc.rust-lang.org/book/ch04-02-references-and-borrowing.html#the-rules-of-references) @@ -1282,6 +2497,7 @@ Python exceptions will automatically be raised when your functions are used in a rules of references. Here is an example. + ```rust # use pyo3::prelude::*; @@ -1300,8 +2516,8 @@ impl Names { self.names.append(&mut other.names) } } -# Python::with_gil(|py| { -# let names = PyCell::new(py, Names::new()).unwrap(); +# Python::attach(|py| { +# let names = Py::new(py, Names::new()).unwrap(); # pyo3::py_run!(py, names, r" # try: # names.merge(names) @@ -1311,6 +2527,7 @@ impl Names { # "); # }) ``` + `Names` has a `merge` method, which takes `&mut self` and another argument of type `&mut Self`. Given this `#[pyclass]`, calling `names.merge(names)` in Python raises a [`PyBorrowMutError`] exception, since it requires two mutable borrows of `names`. @@ -1318,14 +2535,16 @@ a [`PyBorrowMutError`] exception, since it requires two mutable borrows of `name However, for `#[pyproto]` and some functions, you need to manually fix the code. #### Object creation + In 0.8 object creation was done with `PyRef::new` and `PyRefMut::new`. In 0.9 these have both been removed. To upgrade code, please use -[`PyCell::new`]({{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyCell.html#method.new) instead. +`PyCell::new` instead. If you need [`PyRef`] or [`PyRefMut`], just call `.borrow()` or `.borrow_mut()` on the newly-created `PyCell`. Before: + ```rust,compile_fail # use pyo3::prelude::*; # #[pyclass] @@ -1336,7 +2555,8 @@ let obj_ref = PyRef::new(py, MyClass {}).unwrap(); ``` After: -```rust + +```rust,ignore # use pyo3::prelude::*; # #[pyclass] # struct MyClass {} @@ -1347,12 +2567,14 @@ let obj_ref = obj.borrow(); ``` #### Object extraction + For `PyClass` types `T`, `&T` and `&mut T` no longer have [`FromPyObject`] implementations. Instead you should extract `PyRef` or `PyRefMut`, respectively. If `T` implements `Clone`, you can extract `T` itself. In addition, you can also extract `&PyCell`, though you rarely need it. Before: + ```compile_fail let obj: &PyAny = create_obj(); let obj_ref: &MyClass = obj.extract().unwrap(); @@ -1360,6 +2582,7 @@ let obj_ref_mut: &mut MyClass = obj.extract().unwrap(); ``` After: + ```rust,ignore # use pyo3::prelude::*; # use pyo3::types::IntoPyDict; @@ -1380,14 +2603,15 @@ let obj_ref_mut: PyRefMut<'_, MyClass> = obj.extract().unwrap(); # }) ``` - #### `#[pyproto]` + Most of the arguments to methods in `#[pyproto]` impls require a [`FromPyObject`] implementation. So if your protocol methods take `&T` or `&mut T` (where `T: PyClass`), please use [`PyRef`] or [`PyRefMut`] instead. Before: + ```rust,compile_fail # use pyo3::prelude::*; # use pyo3::class::PySequenceProtocol; @@ -1406,6 +2630,7 @@ impl PySequenceProtocol for ByteSequence { ``` After: + ```rust,compile_fail # use pyo3::prelude::*; # use pyo3::class::PySequenceProtocol; @@ -1423,9 +2648,45 @@ impl PySequenceProtocol for ByteSequence { } ``` +
+ + + [`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html [`PyAny`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html -[`PyCell`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyCell.html [`PyBorrowMutError`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyBorrowMutError.html [`PyRef`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html [`PyRefMut`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html diff --git a/guide/src/module.md b/guide/src/module.md index 3d984f60d39..79d7fbdad32 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -2,7 +2,8 @@ You can create a module using `#[pymodule]`: -```rust +```rust,no_run +# mod declarative_module_basic_test { use pyo3::prelude::*; #[pyfunction] @@ -12,19 +13,28 @@ fn double(x: usize) -> usize { /// This module is implemented in Rust. #[pymodule] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(double, m)?)?; - Ok(()) +mod my_extension { + use pyo3::prelude::*; + + #[pymodule_export] + use super::double; // The double function is made available from Python, works also with classes + + #[pyfunction] // Inline definition of a pyfunction, also made availlable to Python + fn triple(x: usize) -> usize { + x * 3 + } } +# } ``` -The `#[pymodule]` procedural macro takes care of exporting the initialization function of your -module to Python. +The `#[pymodule]` procedural macro takes care of creating the initialization function of your +module and exposing it to Python. -The module's name defaults to the name of the Rust function. You can override the module name by -using `#[pyo3(name = "custom_name")]`: +The module's name defaults to the name of the Rust module. +You can override the module name by using `#[pyo3(name = "custom_name")]`: -```rust +```rust,no_run +# mod declarative_module_custom_name_test { use pyo3::prelude::*; #[pyfunction] @@ -32,27 +42,26 @@ fn double(x: usize) -> usize { x * 2 } -#[pymodule] -#[pyo3(name = "custom_name")] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(double, m)?)?; - Ok(()) +#[pymodule(name = "custom_name")] +mod my_extension { + #[pymodule_export] + use super::double; } +# } ``` -The name of the module must match the name of the `.so` or `.pyd` -file. Otherwise, you will get an import error in Python with the following message: -`ImportError: dynamic module does not define module export function (PyInit_name_of_your_module)` +The name of the module must match the name of the `.so` or `.pyd` file. +Otherwise, you will get an import error in Python with the following message: `ImportError: dynamic module does not define module export function (PyInit_name_of_your_module)` To import the module, either: - - copy the shared library as described in [Manual builds](building_and_distribution.html#manual-builds), or - - use a tool, e.g. `maturin develop` with [maturin](https://github.com/PyO3/maturin) or + +- copy the shared library as described in [Manual builds](building-and-distribution.md#manual-builds), or +- use a tool, e.g. `maturin develop` with [maturin](https://github.com/PyO3/maturin) or `python setup.py develop` with [setuptools-rust](https://github.com/PyO3/setuptools-rust). ## Documentation -The [Rust doc comments](https://doc.rust-lang.org/stable/book/ch03-04-comments.html) of the module -initialization function will be applied automatically as the Python docstring of your module. +The [Rust doc comments](https://doc.rust-lang.org/stable/book/ch03-04-comments.html) of the Rust module will be applied automatically as the Python docstring of your module. For example, building off of the above code, this will print `This module is implemented in Rust.`: @@ -64,44 +73,105 @@ print(my_extension.__doc__) ## Python submodules -You can create a module hierarchy within a single extension module by using -[`PyModule.add_submodule()`]({{#PYO3_DOCS_URL}}/pyo3/prelude/struct.PyModule.html#method.add_submodule). -For example, you could define the modules `parent_module` and `parent_module.child_module`. +You can create a module hierarchy within a single extension module by just `use`ing modules like functions or classes. +For example, you could define the modules `parent_module` and `parent_module.child_module`: ```rust use pyo3::prelude::*; #[pymodule] -fn parent_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { - register_child_module(py, m)?; - Ok(()) +mod parent_module { + #[pymodule_export] + use super::child_module; } -fn register_child_module(py: Python<'_>, parent_module: &PyModule) -> PyResult<()> { - let child_module = PyModule::new(py, "child_module")?; - child_module.add_function(wrap_pyfunction!(func, child_module)?)?; - parent_module.add_submodule(child_module)?; - Ok(()) +#[pymodule] +mod child_module { + #[pymodule_export] + use super::func; } #[pyfunction] fn func() -> String { "func".to_string() } - -# Python::with_gil(|py| { -# use pyo3::wrap_pymodule; -# use pyo3::types::IntoPyDict; -# let parent_module = wrap_pymodule!(parent_module)(py); -# let ctx = [("parent_module", parent_module)].into_py_dict_bound(py); # -# py.run_bound("assert parent_module.child_module.func() == 'func'", None, Some(&ctx)).unwrap(); -# }) +# fn main() { +# Python::attach(|py| { +# use pyo3::wrap_pymodule; +# use pyo3::types::IntoPyDict; +# let parent_module = wrap_pymodule!(parent_module)(py); +# let ctx = [("parent_module", parent_module)].into_py_dict(py).unwrap(); +# +# py.run(c"assert parent_module.child_module.func() == 'func'", None, Some(&ctx)).unwrap(); +# }) +} ``` -Note that this does not define a package, so this won’t allow Python code to directly import -submodules by using `from parent_module import child_module`. For more information, see -[#759](https://github.com/PyO3/pyo3/issues/759) and -[#1517](https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021). +Note that this does not define a package, so this won’t allow Python code to directly import submodules by using `from parent_module import child_module`. +For more information, see [#759](https://github.com/PyO3/pyo3/issues/759) and [#1517](https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021). + +You can provide the `submodule` argument to `#[pymodule()]` for modules that are not top-level modules in order for them to properly generate the `#[pyclass]` `module` attribute automatically. + +## Inline declaration + +It is possible to declare functions, classes, sub-modules and constants inline in a module: + +For example: + +```rust,no_run +# mod declarative_module_test { +#[pyo3::pymodule] +mod my_extension { + use pyo3::prelude::*; -It is not necessary to add `#[pymodule]` on nested modules, which is only required on the top-level module. + #[pymodule_export] + const PI: f64 = std::f64::consts::PI; // Exports PI constant as part of the module + + #[pyfunction] // This will be part of the module + fn double(x: usize) -> usize { + x * 2 + } + + #[pyclass] // This will be part of the module + struct Unit; + + #[pymodule] + mod submodule { + // This is a submodule + use pyo3::prelude::*; + + #[pyclass] + struct Nested; + } +} +# } +``` + +In this case, `#[pymodule]` macro automatically sets the `module` attribute of the `#[pyclass]` macros declared inside of it with its name. +For nested modules, the name of the parent module is automatically added. +In the previous example, the `Nested` class will have for `module` `my_extension.submodule`. + +## Procedural initialization + +If the macros provided by PyO3 are not enough, it is possible to run code at the module initialization: + +```rust,no_run +# mod procedural_module_test { +#[pyo3::pymodule] +mod my_extension { + use pyo3::prelude::*; + + #[pyfunction] + fn double(x: usize) -> usize { + x * 2 + } + + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + // Arbitrary code to run at the module initialization + m.add("double2", m.getattr("double")?) + } +} +# } +``` diff --git a/guide/src/parallelism.md b/guide/src/parallelism.md index 195106b2420..6f36e1cf956 100644 --- a/guide/src/parallelism.md +++ b/guide/src/parallelism.md @@ -1,8 +1,17 @@ # Parallelism -CPython has the infamous [Global Interpreter Lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock), which prevents several threads from executing Python bytecode in parallel. This makes threading in Python a bad fit for [CPU-bound](https://stackoverflow.com/questions/868568/) tasks and often forces developers to accept the overhead of multiprocessing. +Historically, CPython was limited by the [global interpreter lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) (GIL), which only allowed a single thread to drive the Python interpreter at a time. +This made threading in Python a bad fit for [CPU-bound](https://en.wikipedia.org/wiki/CPU-bound) tasks and often forced developers to accept the overhead of multiprocessing. + +Rust is well-suited to multithreaded code, and libraries like [`rayon`] can help you leverage safe parallelism with minimal effort. +The [`Python::detach`] method can be used to allow the Python interpreter to do other work while the Rust work is ongoing. + +To enable full parallelism in your application, consider also using [free-threaded Python](./free-threading.md) which is supported since Python 3.14. + +## Parallelism under the Python GIL + +Let's take a look at our [word-count](https://github.com/PyO3/pyo3/blob/main/examples/word-count/src/lib.rs) example, where we have a `search` function that utilizes the [`rayon`] crate to count words in parallel. -In PyO3 parallelism can be easily achieved in Rust-only code. Let's take a look at our [word-count](https://github.com/PyO3/pyo3/blob/main/examples/word-count/src/lib.rs) example, where we have a `search` function that utilizes the [rayon](https://github.com/rayon-rs/rayon) crate to count words in parallel. ```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; @@ -31,7 +40,9 @@ fn search(contents: &str, needle: &str) -> usize { } ``` -But let's assume you have a long running Rust function which you would like to execute several times in parallel. For the sake of example let's take a sequential version of the word count: +But let's assume you have a long running Rust function which you would like to execute several times in parallel. +For the sake of example let's take a sequential version of the word count: + ```rust,no_run # #![allow(dead_code)] # fn count_line(line: &str, needle: &str) -> usize { @@ -49,7 +60,9 @@ fn search_sequential(contents: &str, needle: &str) -> usize { } ``` -To enable parallel execution of this function, the [`Python::allow_threads`] method can be used to temporarily release the GIL, thus allowing other Python threads to run. We then have a function exposed to the Python runtime which calls `search_sequential` inside a closure passed to [`Python::allow_threads`] to enable true parallelism: +To enable parallel execution of this function, the [`Python::detach`] method can be used to temporarily release the GIL, thus allowing other Python threads to run. +We then have a function exposed to the Python runtime which calls `search_sequential` inside a closure passed to [`Python::detach`] to enable true parallelism: + ```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; @@ -68,23 +81,24 @@ To enable parallel execution of this function, the [`Python::allow_threads`] met # contents.lines().map(|line| count_line(line, needle)).sum() # } #[pyfunction] -fn search_sequential_allow_threads(py: Python<'_>, contents: &str, needle: &str) -> usize { - py.allow_threads(|| search_sequential(contents, needle)) +fn search_sequential_detached(py: Python<'_>, contents: &str, needle: &str) -> usize { + py.detach(|| search_sequential(contents, needle)) } ``` Now Python threads can use more than one CPU core, resolving the limitation which usually makes multi-threading in Python only good for IO-bound tasks: + ```Python from concurrent.futures import ThreadPoolExecutor -from word_count import search_sequential_allow_threads +from word_count import search_sequential_detached executor = ThreadPoolExecutor(max_workers=2) future_1 = executor.submit( - word_count.search_sequential_allow_threads, contents, needle + word_count.search_sequential_detached, contents, needle ) future_2 = executor.submit( - word_count.search_sequential_allow_threads, contents, needle + word_count.search_sequential_detached, contents, needle ) result_1 = future_1.result() result_2 = future_2.result() @@ -104,6 +118,7 @@ We are using `pytest-benchmark` to benchmark four word count functions: The benchmark script can be found [here](https://github.com/PyO3/pyo3/blob/main/examples/word-count/tests/test_word_count.py), and we can run `nox` in the `word-count` folder to benchmark these functions. While the results of the benchmark of course depend on your machine, the relative results should be similar to this (mid 2020): + ```text -------------------------------------------------------------------------------------------------- benchmark: 4 tests ------------------------------------------------------------------------------------------------- Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations @@ -117,4 +132,54 @@ test_word_count_python_sequential 27.3985 (15.82) 45.452 You can see that the Python threaded version is not much slower than the Rust sequential version, which means compared to an execution on a single CPU core the speed has doubled. -[`Python::allow_threads`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.allow_threads +## Sharing Python objects between Rust threads + +In the example above we made a Python interface to a low-level rust function, and then leveraged the python `threading` module to run the low-level function in parallel. +It is also possible to spawn threads in Rust that acquire the GIL and operate on Python objects. +However, care must be taken to avoid writing code that deadlocks with the GIL in these cases. + +- Note: This example is meant to illustrate how to drop and re-acquire the GIL + to avoid creating deadlocks. Unless the spawned threads subsequently + release the GIL or you are using the free-threaded build of CPython, you + will not see any speedups due to multi-threaded parallelism using `rayon` + to parallelize code that acquires and holds the GIL for the entire + execution of the spawned thread. + +In the example below, we share a `Vec` of User ID objects defined using the +`pyclass` macro and spawn threads to process the collection of data into a `Vec` +of booleans based on a predicate using a `rayon` parallel iterator: + +```rust,no_run +use pyo3::prelude::*; + +// These traits let us use int_par_iter and map +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +#[pyclass] +struct UserID { + id: i64, +} + +let allowed_ids: Vec = Python::attach(|outer_py| { + let instances: Vec> = (0..10).map(|x| Py::new(outer_py, UserID { id: x }).unwrap()).collect(); + outer_py.detach(|| { + instances.par_iter().map(|instance| { + Python::attach(|inner_py| { + instance.borrow(inner_py).id > 5 + }) + }).collect() + }) +}); +assert!(allowed_ids.into_iter().filter(|b| *b).count() == 4); +``` + +It's important to note that there is an `outer_py` Python token as well as an `inner_py` token. +Sharing Python tokens between threads is not allowed and threads must individually attach to the interpreter to access data wrapped by a Python object. + +It's also important to see that this example uses [`Python::detach`] to wrap the code that spawns OS threads via `rayon`. +If this example didn't use `detach`, a `rayon` worker thread would block on acquiring the GIL while a thread that owns the GIL spins forever waiting for the result of the `rayon` thread. +Calling `detach` allows the GIL to be released in the thread collecting the results from the worker threads. +You should always call `detach` in situations that spawn worker threads, but especially so in cases where worker threads need to acquire the GIL, to prevent deadlocks. + +[`Python::detach`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.detach +[`rayon`]: https://github.com/rayon-rs/rayon diff --git a/guide/src/performance.md b/guide/src/performance.md index 23fb59c4e90..86912829797 100644 --- a/guide/src/performance.md +++ b/guide/src/performance.md @@ -2,28 +2,29 @@ To achieve the best possible performance, it is useful to be aware of several tricks and sharp edges concerning PyO3's API. -## `extract` versus `downcast` +## `extract` versus `cast` -Pythonic API implemented using PyO3 are often polymorphic, i.e. they will accept `&PyAny` and try to turn this into multiple more concrete types to which the requested operation is applied. This often leads to chains of calls to `extract`, e.g. +Pythonic API implemented using PyO3 are often polymorphic, i.e. they will accept `&Bound<'_, PyAny>` and try to turn this into multiple more concrete types to which the requested operation is applied. +This often leads to chains of calls to `extract`, e.g. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::{exceptions::PyTypeError, types::PyList}; -fn frobnicate_list(list: &PyList) -> PyResult<&PyAny> { +fn frobnicate_list<'py>(list: &Bound<'_, PyList>) -> PyResult> { todo!() } -fn frobnicate_vec(vec: Vec<&PyAny>) -> PyResult<&PyAny> { +fn frobnicate_vec<'py>(vec: Vec>) -> PyResult> { todo!() } #[pyfunction] -fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { - if let Ok(list) = value.extract::<&PyList>() { - frobnicate_list(list) - } else if let Ok(vec) = value.extract::>() { +fn frobnicate<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(list) = value.extract::>() { + frobnicate_list(&list) + } else if let Ok(vec) = value.extract::>>() { frobnicate_vec(vec) } else { Err(PyTypeError::new_err("Cannot frobnicate that type.")) @@ -31,21 +32,23 @@ fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { } ``` -This suboptimal as the `FromPyObject` trait requires `extract` to have a `Result` return type. For native types like `PyList`, it faster to use `downcast` (which `extract` calls internally) when the error value is ignored. This avoids the costly conversion of a `PyDowncastError` to a `PyErr` required to fulfil the `FromPyObject` contract, i.e. +This suboptimal as the `FromPyObject` trait requires `extract` to have a `Result` return type. +For native types like `PyList`, it faster to use `cast` (which `extract` calls internally) when the error value is ignored. +This avoids the costly conversion of a `PyDowncastError` to a `PyErr` required to fulfil the `FromPyObject` contract, i.e. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::{exceptions::PyTypeError, types::PyList}; -# fn frobnicate_list(list: &PyList) -> PyResult<&PyAny> { todo!() } -# fn frobnicate_vec(vec: Vec<&PyAny>) -> PyResult<&PyAny> { todo!() } +# fn frobnicate_list<'py>(list: &Bound<'_, PyList>) -> PyResult> { todo!() } +# fn frobnicate_vec<'py>(vec: Vec>) -> PyResult> { todo!() } # #[pyfunction] -fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { - // Use `downcast` instead of `extract` as turning `PyDowncastError` into `PyErr` is quite costly. - if let Ok(list) = value.downcast::() { +fn frobnicate<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + // Use `cast` instead of `extract` as turning `PyDowncastError` into `PyErr` is quite costly. + if let Ok(list) = value.cast::() { frobnicate_list(list) - } else if let Ok(vec) = value.extract::>() { + } else if let Ok(vec) = value.extract::>>() { frobnicate_vec(vec) } else { Err(PyTypeError::new_err("Cannot frobnicate that type.")) @@ -53,42 +56,125 @@ fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { } ``` -## Access to GIL-bound reference implies access to GIL token +## Access to Bound implies access to Python token -Calling `Python::with_gil` is effectively a no-op when the GIL is already held, but checking that this is the case still has a cost. If an existing GIL token can not be accessed, for example when implementing a pre-existing trait, but a GIL-bound reference is available, this cost can be avoided by exploiting that access to GIL-bound reference gives zero-cost access to a GIL token via `PyAny::py`. +Calling `Python::attach` is effectively a no-op when we're already attached to the interpreter, but checking that this is the case still has a cost. +If an existing Python token can not be accessed, for example when implementing a pre-existing trait, but a Python-bound reference is available, this cost can be avoided by exploiting that access to Python-bound reference gives zero-cost access to a Python token via `Bound::py`. For example, instead of writing -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyList; struct Foo(Py); -struct FooRef<'a>(&'a PyList); +struct FooBound<'py>(Bound<'py, PyList>); -impl PartialEq for FooRef<'_> { +impl PartialEq for FooBound<'_> { fn eq(&self, other: &Foo) -> bool { - Python::with_gil(|py| self.0.len() == other.0.as_ref(py).len()) + Python::attach(|py| { + let len = other.0.bind(py).len(); + self.0.len() == len + }) } } ``` -use more efficient +use the more efficient -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyList; # struct Foo(Py); -# struct FooRef<'a>(&'a PyList); +# struct FooBound<'py>(Bound<'py, PyList>); # -impl PartialEq for FooRef<'_> { +impl PartialEq for FooBound<'_> { fn eq(&self, other: &Foo) -> bool { - // Access to `&'a PyAny` implies access to `Python<'a>`. + // Access to `&Bound<'py, PyAny>` implies access to `Python<'py>`. let py = self.0.py(); - self.0.len() == other.0.as_ref(py).len() + let len = other.0.bind(py).len(); + self.0.len() == len } } ``` + +## Calling Python callables (`__call__`) + +CPython support multiple calling protocols: [`tp_call`] and [`vectorcall`]. [`vectorcall`] is a more efficient protocol unlocking faster calls. +PyO3 will try to dispatch Python `call`s using the [`vectorcall`] calling convention to archive maximum performance if possible and falling back to [`tp_call`] otherwise. +This is implemented using the (internal) `PyCallArgs` trait. +It defines how Rust types can be used as Python `call` arguments. +This trait is currently implemented for + +- Rust tuples, where each member implements `IntoPyObject`, +- `Bound<'_, PyTuple>` +- `Py` + +Rust tuples may make use of [`vectorcall`] where as `Bound<'_, PyTuple>` and `Py` can only use [`tp_call`]. +For maximum performance prefer using Rust tuples as arguments. + +[`tp_call`]: https://docs.python.org/3/c-api/call.html#the-tp-call-protocol +[`vectorcall`]: https://docs.python.org/3/c-api/call.html#the-vectorcall-protocol + +## Detach from the interpreter for long-running Rust-only work + +When executing Rust code which does not need to interact with the Python interpreter, use [`Python::detach`] to allow the Python interpreter to proceed without waiting for the current thread. + +On the GIL-enabled build, this is crucial for best performance as only a single thread may ever be attached at a time. + +On the free-threaded build, this is still best practice as there are several "stop the world" events (such as garbage collection) where all threads attached to the Python interpreter are forced to wait. + +As a rule of thumb, attaching and detaching from the Python interpreter takes less than a millisecond, so any work which is expected to take multiple milliseconds can likely benefit from detaching from the interpreter. + +[`Python::detach`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.detach + +## Disable the global reference pool + +PyO3 uses global mutable state to keep track of deferred reference count updates implied by `impl Drop for Py` being called without being attached to the interpreter. +The necessary synchronization to obtain and apply these reference count updates when PyO3-based code next attaches to the interpreter is somewhat expensive and can become a significant part of the cost of crossing the Python-Rust boundary. + +This functionality can be avoided by setting the `pyo3_disable_reference_pool` conditional compilation flag. +This removes the global reference pool and the associated costs completely. +However, it does _not_ remove the `Drop` implementation for `Py` which is necessary to interoperate with existing Rust code written without PyO3-based code in mind. +To stay compatible with the wider Rust ecosystem in these cases, we keep the implementation but abort when `Drop` is called without being attached to the interpreter. +If `pyo3_leak_on_drop_without_reference_pool` is additionally enabled, objects dropped without being attached to Python will be leaked instead which is always sound but might have determinal effects like resource exhaustion in the long term. + +This limitation is important to keep in mind when this setting is used, especially when embedding Python code into a Rust application as it is quite easy to accidentally drop a `Py` (or types containing it like `PyErr`, `PyBackedStr` or `PyBackedBytes`) returned from `Python::attach` without making sure to re-attach beforehand. +For example, the following code + +```rust,ignore +# use pyo3::prelude::*; +# use pyo3::types::PyList; +let numbers: Py = Python::attach(|py| PyList::empty(py).unbind()); + +Python::attach(|py| { + numbers.bind(py).append(23).unwrap(); +}); + +Python::attach(|py| { + numbers.bind(py).append(42).unwrap(); +}); +``` + +will abort if the list not explicitly disposed via + +```rust +# use pyo3::prelude::*; +# use pyo3::types::PyList; +let numbers: Py = Python::attach(|py| PyList::empty(py).unbind()); + +Python::attach(|py| { + numbers.bind(py).append(23).unwrap(); +}); + +Python::attach(|py| { + numbers.bind(py).append(42).unwrap(); +}); + +Python::attach(move |py| { + drop(numbers); +}); +``` diff --git a/guide/src/python-from-rust.md b/guide/src/python-from-rust.md new file mode 100644 index 00000000000..87a80b95a72 --- /dev/null +++ b/guide/src/python-from-rust.md @@ -0,0 +1,65 @@ +# Calling Python in Rust code + +This chapter of the guide documents some ways to interact with Python code from Rust. + +Below is an introduction to the `'py` lifetime and some general remarks about how PyO3's API reasons about Python code. + +The subchapters also cover the following topics: + +- Python object types available in PyO3's API +- How to work with Python exceptions +- How to call Python functions +- How to execute existing Python code + +## The `'py` lifetime + +To safely interact with the Python interpreter a Rust thread must be [attached] to the Python interpreter. +PyO3 has a `Python<'py>` token that is used to prove that these conditions are met. +Its lifetime `'py` is a central part of PyO3's API. + +The `Python<'py>` token serves three purposes: + +- It provides global APIs for the Python interpreter, such as [`py.eval()`][eval] and [`py.import()`][import]. +- It can be passed to functions that require a proof of attachment, such as [`Py::clone_ref`][clone_ref]. +- Its lifetime `'py` is used to bind many of PyO3's types to the Python interpreter, such as [`Bound<'py, T>`][Bound]. + +PyO3's types that are bound to the `'py` lifetime, for example `Bound<'py, T>`, all contain a `Python<'py>` token. +This means they have full access to the Python interpreter and offer a complete API for interacting with Python objects. + +Consult [PyO3's API documentation][obtaining-py] to learn how to acquire one of these tokens. + +### The Global Interpreter Lock + +Prior to the introduction of free-threaded Python (first available in 3.13, fully supported in 3.14), the Python interpreter was made thread-safe by the [global interpreter lock]. +This ensured that only one Python thread can use the Python interpreter and its API at the same time. +Historically, Rust code was able to use the GIL as a synchronization guarantee, but the introduction of free-threaded Python removed this possibility. + +The [`pyo3::sync`] module offers synchronization tools which abstract over both Python builds. + +To enable any parallelism on the GIL-enabled build, and best throughput on the free-threaded build, non-Python operations (system calls and native Rust code) should consider detaching from the Python interpreter to allow other work to proceed. +See [the section on parallelism](parallelism.md) for how to do that using PyO3's API. + +## Python's memory model + +Python's memory model differs from Rust's memory model in two key ways: + +- There is no concept of ownership; all Python objects are shared and usually implemented via reference counting +- There is no concept of exclusive (`&mut`) references; any reference can mutate a Python object + +PyO3's API reflects this by providing [smart pointer][smart-pointers] types, `Py`, `Bound<'py, T>`, and (the very rarely used) `Borrowed<'a, 'py, T>`. +These smart pointers all use Python reference counting. +See the [subchapter on types](./types.md) for more detail on these types. + +Because of the lack of exclusive `&mut` references, PyO3's APIs for Python objects, for example [`PyListMethods::append`], use shared references. +This is safe because Python objects have internal mechanisms to prevent data races (as of time of writing, the Python GIL). + +[attached]: https://docs.python.org/3.14/glossary.html#term-attached-thread-state +[global interpreter lock]: https://docs.python.org/3/c-api/init.html#thread-state-and-the-global-interpreter-lock +[smart-pointers]: https://doc.rust-lang.org/book/ch15-00-smart-pointers.html +[obtaining-py]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#obtaining-a-python-token +[`pyo3::sync`]: {{#PYO3_DOCS_URL}}/pyo3/sync/index.html +[eval]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval +[import]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.import +[clone_ref]: {{#PYO3_DOCS_URL}}/pyo3/prelude/struct.Py.html#method.clone_ref +[Bound]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html +[`PyListMethods::append`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyListMethods.html#tymethod.append diff --git a/guide/src/python-from-rust/calling-existing-code.md b/guide/src/python-from-rust/calling-existing-code.md new file mode 100644 index 00000000000..b54aee76dcf --- /dev/null +++ b/guide/src/python-from-rust/calling-existing-code.md @@ -0,0 +1,403 @@ +# Executing existing Python code + +If you already have some existing Python code that you need to execute from Rust, the following FAQs can help you select the right PyO3 functionality for your situation: + +## Want to access Python APIs? Then use `PyModule::import` + +[`PyModule::import`] can be used to get handle to a Python module from Rust. +You can use this to import and use any Python module available in your environment. + +```rust +use pyo3::prelude::*; + +fn main() -> PyResult<()> { + Python::attach(|py| { + let builtins = PyModule::import(py, "builtins")?; + let total: i32 = builtins + .getattr("sum")? + .call1((vec![1, 2, 3],))? + .extract()?; + assert_eq!(total, 6); + Ok(()) + }) +} +``` + +[`PyModule::import`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.import + +## Want to run just an expression? Then use `eval` + +[`Python::eval`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval) is +a method to execute a [Python expression](https://docs.python.org/3/reference/expressions.html) +and return the evaluated value as a `Bound<'py, PyAny>` object. + +```rust +use pyo3::prelude::*; + +# fn main() -> Result<(), ()> { +Python::attach(|py| { + let result = py + .eval(c"[i * 10 for i in range(5)]", None, None) + .map_err(|e| { + e.print_and_set_sys_last_vars(py); + })?; + let res: Vec = result.extract().unwrap(); + assert_eq!(res, vec![0, 10, 20, 30, 40]); + Ok(()) +}) +# } +``` + +## Want to run statements? Then use `run` + +[`Python::run`] is a method to execute one or more +[Python statements](https://docs.python.org/3/reference/simple_stmts.html). +This method returns nothing (like any Python statement), but you can get +access to manipulated objects via the `locals` dict. + +You can also use the [`py_run!`] macro, which is a shorthand for [`Python::run`]. +Since [`py_run!`] panics on exceptions, we recommend you use this macro only for +quickly testing your Python extensions. + +```rust +use pyo3::prelude::*; +use pyo3::py_run; + +# fn main() { +#[pyclass] +struct UserData { + id: u32, + name: String, +} + +#[pymethods] +impl UserData { + fn as_tuple(&self) -> (u32, String) { + (self.id, self.name.clone()) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("User {}(id: {})", self.name, self.id)) + } +} + +Python::attach(|py| { + let userdata = UserData { + id: 34, + name: "Yu".to_string(), + }; + let userdata = Py::new(py, userdata).unwrap(); + let userdata_as_tuple = (34, "Yu"); + py_run!(py, userdata userdata_as_tuple, r#" +assert repr(userdata) == "User Yu(id: 34)" +assert userdata.as_tuple() == userdata_as_tuple + "#); +}) +# } +``` + +## You have a Python file or code snippet? Then use `PyModule::from_code` + +[`PyModule::from_code`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.from_code) +can be used to generate a Python module which can then be used just as if it was imported with +`PyModule::import`. + +**Warning**: This will compile and execute code. **Never** pass untrusted code +to this function! + +```rust +use pyo3::{prelude::*, types::IntoPyDict}; + +# fn main() -> PyResult<()> { +Python::attach(|py| { + let activators = PyModule::from_code( + py, + cr#" +def relu(x): + """see https://en.wikipedia.org/wiki/Rectifier_(neural_networks)""" + return max(0.0, x) + +def leaky_relu(x, slope=0.01): + return x if x >= 0 else x * slope + "#, + c"activators.py", + c"activators", + )?; + + let relu_result: f64 = activators.getattr("relu")?.call1((-1.0,))?.extract()?; + assert_eq!(relu_result, 0.0); + + let kwargs = [("slope", 0.2)].into_py_dict(py)?; + let lrelu_result: f64 = activators + .getattr("leaky_relu")? + .call((-1.0,), Some(&kwargs))? + .extract()?; + assert_eq!(lrelu_result, -0.2); +# Ok(()) +}) +# } +``` + +## Want to embed Python in Rust with additional modules? + +Python maintains the `sys.modules` dict as a cache of all imported modules. +An import in Python will first attempt to lookup the module from this dict, +and if not present will use various strategies to attempt to locate and load +the module. + +The [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) macro can be used to add additional `#[pymodule]` modules to an embedded Python interpreter. +The macro **must** be invoked _before_ initializing Python. + +As an example, the below adds the module `foo` to the embedded interpreter: + +```rust +use pyo3::prelude::*; + +#[pymodule] +mod foo { + use pyo3::prelude::*; + + #[pyfunction] + fn add_one(x: i64) -> i64 { + x + 1 + } +} + +fn main() -> PyResult<()> { + pyo3::append_to_inittab!(foo); + Python::attach(|py| Python::run(py, c"import foo; foo.add_one(6)", None, None)) +} +``` + +If `append_to_inittab` cannot be used due to constraints in the program, +an alternative is to create a module using [`PyModule::new`] +and insert it manually into `sys.modules`: + +```rust +use pyo3::prelude::*; +use pyo3::types::PyDict; + +#[pyfunction] +pub fn add_one(x: i64) -> i64 { + x + 1 +} + +fn main() -> PyResult<()> { + Python::attach(|py| { + // Create new module + let foo_module = PyModule::new(py, "foo")?; + foo_module.add_function(wrap_pyfunction!(add_one, &foo_module)?)?; + + // Import and get sys.modules + let sys = PyModule::import(py, "sys")?; + let py_modules: Bound<'_, PyDict> = sys.getattr("modules")?.cast_into()?; + + // Insert foo into sys.modules + py_modules.set_item("foo", foo_module)?; + + // Now we can import + run our python code + Python::run(py, c"import foo; foo.add_one(6)", None, None) + }) +} +``` + +## Include multiple Python files + +You can include a file at compile time by using +[`std::include_str`](https://doc.rust-lang.org/std/macro.include_str.html) macro. + +Or you can load a file at runtime by using +[`std::fs::read_to_string`](https://doc.rust-lang.org/std/fs/fn.read_to_string.html) function. + +Many Python files can be included and loaded as modules. +If one file depends on another you must preserve correct order while declaring `PyModule`. + +Example directory structure: + +```text +. +├── Cargo.lock +├── Cargo.toml +├── python_app +│ ├── app.py +│ └── utils +│ └── foo.py +└── src + └── main.rs +``` + +`python_app/app.py`: + +```python +from utils.foo import bar + + +def run(): + return bar() +``` + +`python_app/utils/foo.py`: + +```python +def bar(): + return "baz" +``` + +The example below shows: + +- how to include content of `app.py` and `utils/foo.py` into your rust binary +- how to call function `run()` (declared in `app.py`) that needs function + imported from `utils/foo.py` + +`src/main.rs`: + +```rust,ignore +use pyo3::prelude::*; +use pyo3_ffi::c_str; + +fn main() -> PyResult<()> { + let py_foo = c_str!(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/python_app/utils/foo.py" + ))); + let py_app = c_str!(include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/app.py"))); + let from_python = Python::attach(|py| -> PyResult> { + PyModule::from_code(py, py_foo, c"foo.py", c"utils.foo")?; + let app: Py = PyModule::from_code(py, py_app, c"app.py", c"")? + .getattr("run")? + .into(); + app.call0(py) + }); + + println!("py: {}", from_python?); + Ok(()) +} +``` + +The example below shows: + +- how to load content of `app.py` at runtime so that it sees its dependencies + automatically +- how to call function `run()` (declared in `app.py`) that needs function + imported from `utils/foo.py` + +It is recommended to use absolute paths because then your binary can be run +from anywhere as long as your `app.py` is in the expected directory (in this example +that directory is `/usr/share/python_app`). + +`src/main.rs`: + +```rust,no_run +use pyo3::prelude::*; +use pyo3::types::PyList; +use std::fs; +use std::path::Path; +use std::ffi::CString; + +fn main() -> PyResult<()> { + let path = Path::new("/usr/share/python_app"); + let py_app = CString::new(fs::read_to_string(path.join("app.py"))?)?; + let from_python = Python::attach(|py| -> PyResult> { + let syspath = py + .import("sys")? + .getattr("path")? + .cast_into::()?; + syspath.insert(0, path)?; + let app: Py = PyModule::from_code(py, py_app.as_c_str(), c"app.py", c"")? + .getattr("run")? + .into(); + app.call0(py) + }); + + println!("py: {}", from_python?); + Ok(()) +} +``` + +## Need to use a context manager from Rust? + +Use context managers by directly invoking `__enter__` and `__exit__`. + +```rust +use pyo3::prelude::*; + +fn main() { + Python::attach(|py| { + let custom_manager = PyModule::from_code( + py, + cr#" +class House(object): + def __init__(self, address): + self.address = address + def __enter__(self): + print(f"Welcome to {self.address}!") + def __exit__(self, type, value, traceback): + if type: + print(f"Sorry you had {type} trouble at {self.address}") + else: + print(f"Thank you for visiting {self.address}, come again soon!") + + "#, + c"house.py", + c"house", + ) + .unwrap(); + + let house_class = custom_manager.getattr("House").unwrap(); + let house = house_class.call1(("123 Main Street",)).unwrap(); + + house.call_method0("__enter__").unwrap(); + + let result = py.eval(c"undefined_variable + 1", None, None); + + // If the eval threw an exception we'll pass it through to the context manager. + // Otherwise, __exit__ is called with empty arguments (Python "None"). + match result { + Ok(_) => { + let none = py.None(); + house + .call_method1("__exit__", (&none, &none, &none)) + .unwrap(); + } + Err(e) => { + house + .call_method1( + "__exit__", + ( + e.get_type(py), + e.value(py), + e.traceback(py), + ), + ) + .unwrap(); + } + } + }) +} +``` + +## Handling system signals/interrupts (Ctrl-C) + +The best way to handle system signals when running Rust code is to periodically call `Python::check_signals` to handle any signals captured by Python's signal handler. +See also [the FAQ entry](../faq.md#ctrl-c-doesnt-do-anything-while-my-rust-code-is-executing). + +Alternatively, set Python's `signal` module to take the default action for a signal: + +```rust +use pyo3::prelude::*; + +# fn main() -> PyResult<()> { +Python::attach(|py| -> PyResult<()> { + let signal = py.import("signal")?; + // Set SIGINT to have the default action + signal + .getattr("signal")? + .call1((signal.getattr("SIGINT")?, signal.getattr("SIG_DFL")?))?; + Ok(()) +}) +# } +``` + +[`py_run!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.py_run.html +[`Python::run`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.run +[`PyModule::new`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.new diff --git a/guide/src/python-from-rust/function-calls.md b/guide/src/python-from-rust/function-calls.md new file mode 100644 index 00000000000..6574b86a356 --- /dev/null +++ b/guide/src/python-from-rust/function-calls.md @@ -0,0 +1,115 @@ +# Calling Python functions + +The `Bound<'py, T>` smart pointer (such as `Bound<'py, PyAny>`, `Bound<'py, PyList>`, or `Bound<'py, MyClass>`) can be used to call Python functions. + +PyO3 offers two APIs to make function calls: + +- [`call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) - call any callable Python object. +- [`call_method`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method) - call a method on the Python object. + +Both of these APIs take `args` and `kwargs` arguments (for positional and keyword arguments respectively). +There are variants for less complex calls: + +- [`call1`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call1) and [`call_method1`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method1) to call only with positional `args`. +- [`call0`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call0) and [`call_method0`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method0) to call with no arguments. + +For convenience the [`Py`](../types.md#pyt) smart pointer also exposes these same six API methods, but needs a `Python` token as an additional first argument to prove the thread is attached to the Python interpreter. + +The example below calls a Python function behind a `Py` reference: + +```rust +use pyo3::prelude::*; +use pyo3::types::PyTuple; + +fn main() -> PyResult<()> { + let arg1 = "arg1"; + let arg2 = "arg2"; + let arg3 = "arg3"; + + Python::attach(|py| { + let fun: Py = PyModule::from_code( + py, + c"def example(*args, **kwargs): + if args != (): + print('called with args', args) + if kwargs != {}: + print('called with kwargs', kwargs) + if args == () and kwargs == {}: + print('called with no arguments')", + c"example.py", + c"", + )? + .getattr("example")? + .into(); + + // call object without any arguments + fun.call0(py)?; + + // pass object with Rust tuple of positional arguments + let args = (arg1, arg2, arg3); + fun.call1(py, args)?; + + // call object with Python tuple of positional arguments + let args = PyTuple::new(py, &[arg1, arg2, arg3])?; + fun.call1(py, args)?; + Ok(()) + }) +} +``` + +## Creating keyword arguments + +For the `call` and `call_method` APIs, `kwargs` are `Option<&Bound<'py, PyDict>>`, so can either be `None` or `Some(&dict)`. +You can use the [`IntoPyDict`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.IntoPyDict.html) trait to convert other dict-like containers, e.g. `HashMap` or `BTreeMap`, as well as tuples with up to 10 elements and `Vec`s where each element is a two-element tuple. +To pass keyword arguments of different types, construct a `PyDict` object. + +```rust +use pyo3::prelude::*; +use pyo3::types::{PyDict, IntoPyDict}; +use std::collections::HashMap; + +fn main() -> PyResult<()> { + let key1 = "key1"; + let val1 = 1; + let key2 = "key2"; + let val2 = 2; + + Python::attach(|py| { + let fun: Py = PyModule::from_code( + py, + c"def example(*args, **kwargs): + if args != (): + print('called with args', args) + if kwargs != {}: + print('called with kwargs', kwargs) + if args == () and kwargs == {}: + print('called with no arguments')", + c"example.py", + c"", + )? + .getattr("example")? + .into(); + + // call object with PyDict + let kwargs = [(key1, val1)].into_py_dict(py)?; + fun.call(py, (), Some(&kwargs))?; + + // pass arguments as Vec + let kwargs = vec![(key1, val1), (key2, val2)]; + fun.call(py, (), Some(&kwargs.into_py_dict(py)?))?; + + // pass arguments as HashMap + let mut kwargs = HashMap::<&str, i32>::new(); + kwargs.insert(key1, 1); + fun.call(py, (), Some(&kwargs.into_py_dict(py)?))?; + + // pass arguments of different types as PyDict + let kwargs = PyDict::new(py); + kwargs.set_item(key1, val1)?; + kwargs.set_item(key2, "string")?; + fun.call(py, (), Some(&kwargs))?; + + Ok(()) + }) +} +``` diff --git a/guide/src/python_typing_hints.md b/guide/src/python-typing-hints.md similarity index 50% rename from guide/src/python_typing_hints.md rename to guide/src/python-typing-hints.md index 43a15a21a6e..7f6661a2743 100644 --- a/guide/src/python_typing_hints.md +++ b/guide/src/python-typing-hints.md @@ -1,20 +1,25 @@ # Typing and IDE hints for your Python package -PyO3 provides an easy to use interface to code native Python libraries in Rust. The accompanying Maturin allows you to build and publish them as a package. Yet, for a better user experience, Python libraries should provide typing hints and documentation for all public entities, so that IDEs can show them during development and type analyzing tools such as `mypy` can use them to properly verify the code. +PyO3 provides an easy to use interface to code native Python libraries in Rust. +The accompanying Maturin allows you to build and publish them as a package. +Yet, for a better user experience, Python libraries should provide typing hints and documentation for all public entities, so that IDEs can show them during development and type analyzing tools such as `mypy` can use them to properly verify the code. Currently the best solution for the problem is to manually maintain `*.pyi` files and ship them along with the package. -There is a sketch of a roadmap towards completing [the `experimental-inspect` feature](./features.md#experimental-inspect) which may eventually lead to automatic type annotations generated by PyO3. This needs more testing and implementation, please see [issue #2454](https://github.com/PyO3/pyo3/issues/2454). +PyO3 is working on automated their generation. +See the [type stub generation](type-stub.md) documentation for a description of the current state of automated generation. ## Introduction to `pyi` files -`pyi` files (an abbreviation for `Python Interface`) are called "stub files" in most of the documentation related to them. A very good definition of what it is can be found in [old MyPy documentation](https://github.com/python/mypy/wiki/Creating-Stubs-For-Python-Modules): +`pyi` files (an abbreviation for `Python Interface`) are called "stub files" in most of the documentation related to them. +A very good definition of what it is can be found in [old MyPy documentation](https://github.com/python/mypy/wiki/Creating-Stubs-For-Python-Modules): > A stubs file only contains a description of the public interface of the module without any implementations. There is also [extensive documentation on type stubs on the official Python typing documentation](https://typing.readthedocs.io/en/latest/source/stubs.html). -Most Python developers probably already encountered them when trying to use their IDE's "Go to Definition" function on any builtin type. For example, the definitions of a few standard exceptions look like this: +Most Python developers probably already encountered them when trying to use their IDE's "Go to Definition" function on any builtin type. +For example, the definitions of a few standard exceptions look like this: ```python class BaseException(object): @@ -37,11 +42,12 @@ class StopIteration(Exception): value: Any ``` -As we can see, those are not full definitions containing implementation, but just a description of the interface. It is usually all that the user of the library needs. +As we can see, those are not full definitions containing implementation, but just a description of the interface. +It is usually all that the user of the library needs. ### What do the PEPs say? -At the time of writing this documentation, the `pyi` files are referenced in three PEPs. +At the time of writing this documentation, the `pyi` files are referenced in four PEPs. [PEP8 - Style Guide for Python Code - #Function Annotations](https://www.python.org/dev/peps/pep-0008/#function-annotations) (last point) recommends all third party library creators to provide stub files as the source of knowledge about the package for type checker tools. @@ -53,29 +59,37 @@ At the time of writing this documentation, the `pyi` files are referenced in thr It contains a specification for them (highly recommended reading, since it contains at least one thing that is not used in normal Python code) and also some general information about where to store the stub files. -[PEP561 - Distributing and Packaging Type Information](https://www.python.org/dev/peps/pep-0561/) describes in detail how to build packages that will enable type checking. In particular it contains information about how the stub files must be distributed in order for type checkers to use them. +[PEP561 - Distributing and Packaging Type Information](https://www.python.org/dev/peps/pep-0561/) describes in detail how to build packages that will enable type checking. +In particular it contains information about how the stub files must be distributed in order for type checkers to use them. + +[PEP560 - Core support for typing module and generic types](https://www.python.org/dev/peps/pep-0560/) describes the details on how Python's type system internally supports generics, including both runtime behavior and integration with static type checkers. ## How to do it? [PEP561](https://www.python.org/dev/peps/pep-0561/) recognizes three ways of distributing type information: -* `inline` - the typing is placed directly in source (`py`) files; -* `separate package with stub files` - the typing is placed in `pyi` files distributed in their own, separate package; -* `in-package stub files` - the typing is placed in `pyi` files distributed in the same package as source files. +- `inline` - the typing is placed directly in source (`py`) files; +- `separate package with stub files` - the typing is placed in `pyi` files distributed in their own, separate package; +- `in-package stub files` - the typing is placed in `pyi` files distributed in the same package as source files. -The first way is tricky with PyO3 since we do not have `py` files. When it has been investigated and necessary changes are implemented, this document will be updated. +The first way is tricky with PyO3 since we do not have `py` files. +When it has been investigated and necessary changes are implemented, this document will be updated. -The second way is easy to do, and the whole work can be fully separated from the main library code. The example repo for the package with stub files can be found in [PEP561 references section](https://www.python.org/dev/peps/pep-0561/#references): [Stub package repository](https://github.com/ethanhs/stub-package) +The second way is easy to do, and the whole work can be fully separated from the main library code. +The example repo for the package with stub files can be found in [PEP561 references section](https://www.python.org/dev/peps/pep-0561/#references): [Stub package repository](https://github.com/ethanhs/stub-package) The third way is described below. ### Including `pyi` files in your PyO3/Maturin build package -When source files are in the same package as stub files, they should be placed next to each other. We need a way to do that with Maturin. Also, in order to mark our package as typing-enabled we need to add an empty file named `py.typed` to the package. +When source files are in the same package as stub files, they should be placed next to each other. +We need a way to do that with Maturin. +Also, in order to mark our package as typing-enabled we need to add an empty file named `py.typed` to the package. #### If you do not have other Python files -If you do not need to add any other Python files apart from `pyi` to the package, Maturin provides a way to do most of the work for you. As documented in the [Maturin Guide](https://github.com/PyO3/maturin/#mixed-rustpython-projects), the only thing you need to do is to create a stub file for your module named `.pyi` in your project root and Maturin will do the rest. +If you do not need to add any other Python files apart from `pyi` to the package, Maturin provides a way to do most of the work for you. +As documented in the [Maturin Guide](https://github.com/PyO3/maturin/#mixed-rustpython-projects), the only thing you need to do is to create a stub file for your module named `.pyi` in your project root and Maturin will do the rest. ```text my-rust-project/ @@ -90,7 +104,9 @@ For an example `pyi` file see the [`my_project.pyi` content](#my_projectpyi-cont #### If you need other Python files -If you need to add other Python files apart from `pyi` to the package, you can do it also, but that requires some more work. Maturin provides an easy way to add files to a package ([documentation](https://github.com/PyO3/maturin/blob/0dee40510083c03607834c821eea76964140a126/Readme.md#mixed-rustpython-projects)). You just need to create a folder with the name of your module next to the `Cargo.toml` file (for customization see documentation linked above). +If you need to add other Python files apart from `pyi` to the package, you can do it also, but that requires some more work. +Maturin provides an easy way to add files to a package ([documentation](https://github.com/PyO3/maturin/blob/0dee40510083c03607834c821eea76964140a126/Readme.md#mixed-rustpython-projects)). +You just need to create a folder with the name of your module next to the `Cargo.toml` file (for customization see documentation linked above). The folder structure would be: @@ -112,7 +128,9 @@ Let's go a little bit more into detail regarding the files inside the package fo ##### `__init__.py` content -As we now specify our own package content, we have to provide the `__init__.py` file, so the folder is treated as a package and we can import things from it. We can always use the same content that Maturin creates for us if we do not specify a Python source folder. For PyO3 bindings it would be: +As we now specify our own package content, we have to provide the `__init__.py` file, so the folder is treated as a package and we can import things from it. +We can always use the same content that Maturin creates for us if we do not specify a Python source folder. +For PyO3 bindings it would be: ```python from .my_project import * @@ -125,7 +143,8 @@ That way everything that is exposed by our native module can be imported directl As stated in [PEP561](https://www.python.org/dev/peps/pep-0561/): > Package maintainers who wish to support type checking of their code MUST add a marker file named py.typed to their package supporting typing. This marker applies recursively: if a top-level package includes it, all its sub-packages MUST support type checking as well. -If we do not include that file, some IDEs might still use our `pyi` files to show hints, but the type checkers might not. MyPy will raise an error in this situation: +If we do not include that file, some IDEs might still use our `pyi` files to show hints, but the type checkers might not. +MyPy will raise an error in this situation: ```text error: Skipping analyzing "my_project": found module but no type hints or library stubs @@ -135,7 +154,8 @@ The file is just a marker file, so it should be empty. ##### `my_project.pyi` content -Our module stub file. This document does not aim at describing how to write them, since you can find a lot of documentation on it, starting from the already quoted [PEP484](https://www.python.org/dev/peps/pep-0484/#stub-files). +Our module stub file. +This document does not aim at describing how to write them, since you can find a lot of documentation on it, starting from the already quoted [PEP484](https://www.python.org/dev/peps/pep-0484/#stub-files). The example can look like this: @@ -165,3 +185,75 @@ class Car: :return: the name of the color our great algorithm thinks is the best for this car """ ``` + +### Supporting Generics + +Type annotations can also be made generic in Python. +They are useful for working with different types while maintaining type safety. +Usually, generic classes inherit from the `typing.Generic` metaclass. + +Take for example the following `.pyi` file that specifies a `Car` that can +accept multiple types of wheels: + +```python +from typing import Generic, TypeVar + +W = TypeVar('W') + +class Car(Generic[W]): + def __init__(self, wheels: list[W]) -> None: ... + + def get_wheels(self) -> list[W]: ... + + def change_wheel(self, wheel_number: int, wheel: W) -> None: ... +``` + +This way, the end-user can specify the type with variables such as `truck: Car[SteelWheel] = ...` +and `f1_car: Car[AlloyWheel] = ...`. + +There is also a special syntax for specifying generic types in Python 3.12+: + +```python +class Car[W]: + def __init__(self, wheels: list[W]) -> None: ... + + def get_wheels(self) -> list[W]: ... +``` + +#### Runtime Behaviour + +Stub files (`pyi`) are only useful for static type checkers and ignored at runtime. +Therefore, PyO3 classes do not inherit from `typing.Generic` even if specified in the stub files. + +This can cause some runtime issues, as annotating a variable like `f1_car: Car[AlloyWheel] = ...` +can make Python call magic methods that are not defined. + +To overcome this limitation, implementers can pass the `generic` parameter to `pyclass` in Rust: + +```rust ignore +#[pyclass(generic)] +``` + +#### Advanced Users + +`#[pyclass(generic)]` implements a very simple runtime behavior that accepts any generic argument. +Advanced users can opt to manually implement [`__class_geitem__`](https://docs.python.org/3/reference/datamodel.html#emulating-generic-types) for the generic class to have more control. + +```rust ignore +impl MyClass { + #[classmethod] + #[pyo3(signature = (key, /))] + pub fn __class_getitem__( + cls: &Bound<'_, PyType>, + key: &Bound<'_, PyAny>, + ) -> PyResult> { + /* implementation details */ + } +} +``` + +Note that [`pyo3::types::PyGenericAlias`][pygenericalias] can be helpful when implementing +`__class_getitem__` as it can create [`types.GenericAlias`][genericalias] objects from Rust. + +[pygenericalias]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyGenericAlias.html +[genericalias]: https://docs.python.org/3/library/types.html#types.GenericAlias diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md deleted file mode 100644 index 1ac1a7c217a..00000000000 --- a/guide/src/python_from_rust.md +++ /dev/null @@ -1,512 +0,0 @@ -# Calling Python in Rust code - -This chapter of the guide documents some ways to interact with Python code from Rust: - - How to call Python functions - - How to execute existing Python code - -## Calling Python functions - -Any Python-native object reference (such as `&PyAny`, `&PyList`, or `&PyCell`) can be used to call Python functions. - -PyO3 offers two APIs to make function calls: - -* [`call`]({{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods#tymethod.call) - call any callable Python object. -* [`call_method`]({{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods#tymethod.call_method) - call a method on the Python object. - -Both of these APIs take `args` and `kwargs` arguments (for positional and keyword arguments respectively). There are variants for less complex calls: - -* [`call1`]({{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods#tymethod.call1) and [`call_method1`]({{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods#tymethod.call_method1) to call only with positional `args`. -* [`call0`]({{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods#tymethod.call0) and [`call_method0`]({{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods#tymethod.call_method0) to call with no arguments. - -For convenience the [`Py`](types.html#pyt-and-pyobject) smart pointer also exposes these same six API methods, but needs a `Python` token as an additional first argument to prove the GIL is held. - -The example below calls a Python function behind a `PyObject` (aka `Py`) reference: - -```rust -use pyo3::prelude::*; -use pyo3::types::PyTuple; - -fn main() -> PyResult<()> { - let arg1 = "arg1"; - let arg2 = "arg2"; - let arg3 = "arg3"; - - Python::with_gil(|py| { - let fun: Py = PyModule::from_code( - py, - "def example(*args, **kwargs): - if args != (): - print('called with args', args) - if kwargs != {}: - print('called with kwargs', kwargs) - if args == () and kwargs == {}: - print('called with no arguments')", - "", - "", - )? - .getattr("example")? - .into(); - - // call object without any arguments - fun.call0(py)?; - - // call object with PyTuple - let args = PyTuple::new_bound(py, &[arg1, arg2, arg3]); - fun.call1(py, args)?; - - // pass arguments as rust tuple - let args = (arg1, arg2, arg3); - fun.call1(py, args)?; - Ok(()) - }) -} -``` - -### Creating keyword arguments - -For the `call` and `call_method` APIs, `kwargs` can be `None` or `Some(&PyDict)`. You can use the [`IntoPyDict`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.IntoPyDict.html) trait to convert other dict-like containers, e.g. `HashMap` or `BTreeMap`, as well as tuples with up to 10 elements and `Vec`s where each element is a two-element tuple. - -```rust -use pyo3::prelude::*; -use pyo3::types::IntoPyDict; -use std::collections::HashMap; - -fn main() -> PyResult<()> { - let key1 = "key1"; - let val1 = 1; - let key2 = "key2"; - let val2 = 2; - - Python::with_gil(|py| { - let fun: Py = PyModule::from_code( - py, - "def example(*args, **kwargs): - if args != (): - print('called with args', args) - if kwargs != {}: - print('called with kwargs', kwargs) - if args == () and kwargs == {}: - print('called with no arguments')", - "", - "", - )? - .getattr("example")? - .into(); - - // call object with PyDict - let kwargs = [(key1, val1)].into_py_dict_bound(py); - fun.call_bound(py, (), Some(&kwargs))?; - - // pass arguments as Vec - let kwargs = vec![(key1, val1), (key2, val2)]; - fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; - - // pass arguments as HashMap - let mut kwargs = HashMap::<&str, i32>::new(); - kwargs.insert(key1, 1); - fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; - - Ok(()) - }) -} -``` - -
- -During PyO3's [migration from "GIL Refs" to the `Bound` smart pointer](./migration.md#migrating-from-the-gil-refs-api-to-boundt), [`Py::call`]({{#PYO3_DOCS_URL}}/pyo3/struct.py#method.call) is temporarily named `call_bound` (and `call_method` is temporarily `call_method_bound`). - -(This temporary naming is only the case for the `Py` smart pointer. The methods on the `&PyAny` GIL Ref such as `call` have not been given replacements, and the methods on the `Bound` smart pointer such as [`Bound::call`]({#PYO3_DOCS_URL}}/pyo3/prelude/trait.pyanymethods#tymethod.call) already use follow the newest API conventions.) - -
- -## Executing existing Python code - -If you already have some existing Python code that you need to execute from Rust, the following FAQs can help you select the right PyO3 functionality for your situation: - -### Want to access Python APIs? Then use `PyModule::import`. - -[`Pymodule::import`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.import) can -be used to get handle to a Python module from Rust. You can use this to import and use any Python -module available in your environment. - -```rust -use pyo3::prelude::*; - -fn main() -> PyResult<()> { - Python::with_gil(|py| { - let builtins = PyModule::import(py, "builtins")?; - let total: i32 = builtins - .getattr("sum")? - .call1((vec![1, 2, 3],))? - .extract()?; - assert_eq!(total, 6); - Ok(()) - }) -} -``` - -### Want to run just an expression? Then use `eval`. - -[`Python::eval`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval) is -a method to execute a [Python expression](https://docs.python.org/3.7/reference/expressions.html) -and return the evaluated value as a `&PyAny` object. - -```rust -use pyo3::prelude::*; - -# fn main() -> Result<(), ()> { -Python::with_gil(|py| { - let result = py - .eval_bound("[i * 10 for i in range(5)]", None, None) - .map_err(|e| { - e.print_and_set_sys_last_vars(py); - })?; - let res: Vec = result.extract().unwrap(); - assert_eq!(res, vec![0, 10, 20, 30, 40]); - Ok(()) -}) -# } -``` - -### Want to run statements? Then use `run`. - -[`Python::run`] is a method to execute one or more -[Python statements](https://docs.python.org/3.7/reference/simple_stmts.html). -This method returns nothing (like any Python statement), but you can get -access to manipulated objects via the `locals` dict. - -You can also use the [`py_run!`] macro, which is a shorthand for [`Python::run`]. -Since [`py_run!`] panics on exceptions, we recommend you use this macro only for -quickly testing your Python extensions. - -```rust -use pyo3::prelude::*; -use pyo3::{PyCell, py_run}; - -# fn main() { -#[pyclass] -struct UserData { - id: u32, - name: String, -} - -#[pymethods] -impl UserData { - fn as_tuple(&self) -> (u32, String) { - (self.id, self.name.clone()) - } - - fn __repr__(&self) -> PyResult { - Ok(format!("User {}(id: {})", self.name, self.id)) - } -} - -Python::with_gil(|py| { - let userdata = UserData { - id: 34, - name: "Yu".to_string(), - }; - let userdata = PyCell::new(py, userdata).unwrap(); - let userdata_as_tuple = (34, "Yu"); - py_run!(py, userdata userdata_as_tuple, r#" -assert repr(userdata) == "User Yu(id: 34)" -assert userdata.as_tuple() == userdata_as_tuple - "#); -}) -# } -``` - -## You have a Python file or code snippet? Then use `PyModule::from_code`. - -[`PyModule::from_code`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.from_code) -can be used to generate a Python module which can then be used just as if it was imported with -`PyModule::import`. - -**Warning**: This will compile and execute code. **Never** pass untrusted code -to this function! - -```rust -use pyo3::{ - prelude::*, - types::{IntoPyDict, PyModule}, -}; - -# fn main() -> PyResult<()> { -Python::with_gil(|py| { - let activators = PyModule::from_code( - py, - r#" -def relu(x): - """see https://en.wikipedia.org/wiki/Rectifier_(neural_networks)""" - return max(0.0, x) - -def leaky_relu(x, slope=0.01): - return x if x >= 0 else x * slope - "#, - "activators.py", - "activators", - )?; - - let relu_result: f64 = activators.getattr("relu")?.call1((-1.0,))?.extract()?; - assert_eq!(relu_result, 0.0); - - let kwargs = [("slope", 0.2)].into_py_dict_bound(py); - let lrelu_result: f64 = activators - .getattr("leaky_relu")? - .call((-1.0,), Some(kwargs.as_gil_ref()))? - .extract()?; - assert_eq!(lrelu_result, -0.2); -# Ok(()) -}) -# } -``` - -### Want to embed Python in Rust with additional modules? - -Python maintains the `sys.modules` dict as a cache of all imported modules. -An import in Python will first attempt to lookup the module from this dict, -and if not present will use various strategies to attempt to locate and load -the module. - -The [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) -macro can be used to add additional `#[pymodule]` modules to an embedded -Python interpreter. The macro **must** be invoked _before_ initializing Python. - -As an example, the below adds the module `foo` to the embedded interpreter: - -```rust -use pyo3::prelude::*; - -#[pyfunction] -fn add_one(x: i64) -> i64 { - x + 1 -} - -#[pymodule] -fn foo(_py: Python<'_>, foo_module: &PyModule) -> PyResult<()> { - foo_module.add_function(wrap_pyfunction!(add_one, foo_module)?)?; - Ok(()) -} - -fn main() -> PyResult<()> { - pyo3::append_to_inittab!(foo); - Python::with_gil(|py| Python::run_bound(py, "import foo; foo.add_one(6)", None, None)) -} -``` - -If `append_to_inittab` cannot be used due to constraints in the program, -an alternative is to create a module using [`PyModule::new`] -and insert it manually into `sys.modules`: - -```rust -use pyo3::prelude::*; -use pyo3::types::PyDict; - -#[pyfunction] -pub fn add_one(x: i64) -> i64 { - x + 1 -} - -fn main() -> PyResult<()> { - Python::with_gil(|py| { - // Create new module - let foo_module = PyModule::new(py, "foo")?; - foo_module.add_function(wrap_pyfunction!(add_one, foo_module)?)?; - - // Import and get sys.modules - let sys = PyModule::import(py, "sys")?; - let py_modules: &PyDict = sys.getattr("modules")?.downcast()?; - - // Insert foo into sys.modules - py_modules.set_item("foo", foo_module)?; - - // Now we can import + run our python code - Python::run_bound(py, "import foo; foo.add_one(6)", None, None) - }) -} -``` - -### Include multiple Python files - -You can include a file at compile time by using -[`std::include_str`](https://doc.rust-lang.org/std/macro.include_str.html) macro. - -Or you can load a file at runtime by using -[`std::fs::read_to_string`](https://doc.rust-lang.org/std/fs/fn.read_to_string.html) function. - -Many Python files can be included and loaded as modules. If one file depends on -another you must preserve correct order while declaring `PyModule`. - -Example directory structure: -```text -. -├── Cargo.lock -├── Cargo.toml -├── python_app -│ ├── app.py -│ └── utils -│ └── foo.py -└── src - └── main.rs -``` - -`python_app/app.py`: -```python -from utils.foo import bar - - -def run(): - return bar() -``` - -`python_app/utils/foo.py`: -```python -def bar(): - return "baz" -``` - -The example below shows: -* how to include content of `app.py` and `utils/foo.py` into your rust binary -* how to call function `run()` (declared in `app.py`) that needs function - imported from `utils/foo.py` - -`src/main.rs`: -```rust,ignore -use pyo3::prelude::*; - -fn main() -> PyResult<()> { - let py_foo = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/python_app/utils/foo.py" - )); - let py_app = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/app.py")); - let from_python = Python::with_gil(|py| -> PyResult> { - PyModule::from_code(py, py_foo, "utils.foo", "utils.foo")?; - let app: Py = PyModule::from_code(py, py_app, "", "")? - .getattr("run")? - .into(); - app.call0(py) - }); - - println!("py: {}", from_python?); - Ok(()) -} -``` - -The example below shows: -* how to load content of `app.py` at runtime so that it sees its dependencies - automatically -* how to call function `run()` (declared in `app.py`) that needs function - imported from `utils/foo.py` - -It is recommended to use absolute paths because then your binary can be run -from anywhere as long as your `app.py` is in the expected directory (in this example -that directory is `/usr/share/python_app`). - -`src/main.rs`: -```rust,no_run -use pyo3::prelude::*; -use pyo3::types::PyList; -use std::fs; -use std::path::Path; - -fn main() -> PyResult<()> { - let path = Path::new("/usr/share/python_app"); - let py_app = fs::read_to_string(path.join("app.py"))?; - let from_python = Python::with_gil(|py| -> PyResult> { - let syspath = py.import_bound("sys")?.getattr("path")?.downcast_into::()?; - syspath.insert(0, &path)?; - let app: Py = PyModule::from_code(py, &py_app, "", "")? - .getattr("run")? - .into(); - app.call0(py) - }); - - println!("py: {}", from_python?); - Ok(()) -} -``` - - -[`Python::run`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.run -[`py_run!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.py_run.html - -## Need to use a context manager from Rust? - -Use context managers by directly invoking `__enter__` and `__exit__`. - -```rust -use pyo3::prelude::*; -use pyo3::types::PyModule; - -fn main() { - Python::with_gil(|py| { - let custom_manager = PyModule::from_code( - py, - r#" -class House(object): - def __init__(self, address): - self.address = address - def __enter__(self): - print(f"Welcome to {self.address}!") - def __exit__(self, type, value, traceback): - if type: - print(f"Sorry you had {type} trouble at {self.address}") - else: - print(f"Thank you for visiting {self.address}, come again soon!") - - "#, - "house.py", - "house", - ) - .unwrap(); - - let house_class = custom_manager.getattr("House").unwrap(); - let house = house_class.call1(("123 Main Street",)).unwrap(); - - house.call_method0("__enter__").unwrap(); - - let result = py.eval_bound("undefined_variable + 1", None, None); - - // If the eval threw an exception we'll pass it through to the context manager. - // Otherwise, __exit__ is called with empty arguments (Python "None"). - match result { - Ok(_) => { - let none = py.None(); - house - .call_method1("__exit__", (&none, &none, &none)) - .unwrap(); - } - Err(e) => { - house - .call_method1("__exit__", (e.get_type_bound(py), e.value(py), e.traceback_bound(py))) - .unwrap(); - } - } - }) -} -``` - -## Handling system signals/interrupts (Ctrl-C) - -The best way to handle system signals when running Rust code is to periodically call `Python::check_signals` to handle any signals captured by Python's signal handler. See also [the FAQ entry](./faq.md#ctrl-c-doesnt-do-anything-while-my-rust-code-is-executing). - -Alternatively, set Python's `signal` module to take the default action for a signal: - -```rust -use pyo3::prelude::*; - -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - let signal = py.import_bound("signal")?; - // Set SIGINT to have the default action - signal - .getattr("signal")? - .call1((signal.getattr("SIGINT")?, signal.getattr("SIG_DFL")?))?; - Ok(()) -}) -# } -``` - - -[`PyModule::new`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.new diff --git a/guide/src/rust-from-python.md b/guide/src/rust-from-python.md new file mode 100644 index 00000000000..3b525d399df --- /dev/null +++ b/guide/src/rust-from-python.md @@ -0,0 +1,13 @@ +# Using Rust from Python + +This chapter of the guide is dedicated to explaining how to wrap Rust code into Python objects. + +PyO3 uses Rust's "procedural macros" to provide a powerful yet simple API to denote what Rust code should map into Python objects. + +PyO3 can create three types of Python objects: + +- Python modules, via the `#[pymodule]` macro +- Python functions, via the `#[pyfunction]` macro +- Python classes, via the `#[pyclass]` macro (plus `#[pymethods]` to define methods for those classes) + +The following subchapters go through each of these in turn. diff --git a/guide/src/trait_bounds.md b/guide/src/trait-bounds.md similarity index 84% rename from guide/src/trait_bounds.md rename to guide/src/trait-bounds.md index e0dd988412f..578019c8dfc 100644 --- a/guide/src/trait_bounds.md +++ b/guide/src/trait-bounds.md @@ -1,17 +1,19 @@ # Using in Python a Rust function with trait bounds -PyO3 allows for easy conversion from Rust to Python for certain functions and classes (see the [conversion table](conversions/tables.html). +PyO3 allows for easy conversion from Rust to Python for certain functions and classes (see the [conversion table](conversions/tables.md)). However, it is not always straightforward to convert Rust code that requires a given trait implementation as an argument. This tutorial explains how to convert a Rust function that takes a trait as argument for use in Python with classes implementing the same methods as the trait. Why is this useful? -### Pros +## Pros + - Make your Rust code available to Python users - Code complex algorithms in Rust with the help of the borrow checker ### Cons + - Not as fast as native Rust (type conversion has to be performed and one part of the code runs in Python) - You need to adapt your code to expose it @@ -22,7 +24,7 @@ Let's work with the following basic example of an implementation of a optimizati Let's say we have a function `solve` that operates on a model and mutates its state. The argument of the function can be any model that implements the `Model` trait : -```rust +```rust,no_run # #![allow(dead_code)] pub trait Model { fn set_variables(&mut self, inputs: &Vec); @@ -34,9 +36,12 @@ pub fn solve(model: &mut T) { println!("Magic solver that mutates the model into a resolved state"); } ``` + Let's assume we have the following constraints: + - We cannot change that code as it runs on many Rust models. - We also have many Python models that cannot be solved as this solver is not available in that language. + Rewriting it in Python would be cumbersome and error-prone, as everything is already available in Rust. How could we expose this solver to Python thanks to PyO3 ? @@ -44,7 +49,7 @@ How could we expose this solver to Python thanks to PyO3 ? ## Implementation of the trait bounds for the Python class If a Python class implements the same three methods as the `Model` trait, it seems logical it could be adapted to use the solver. -However, it is not possible to pass a `PyObject` to it as it does not implement the Rust trait (even if the Python model has the required methods). +However, it is not possible to pass a `Py` to it as it does not implement the Rust trait (even if the Python model has the required methods). In order to implement the trait, we must write a wrapper around the calls in Rust to the Python model. The method signatures must be the same as the trait, keeping in mind that the Rust trait cannot be changed for the purpose of making the code available in Python. @@ -63,9 +68,10 @@ class Model: The following wrapper will call the Python model from Rust, using a struct to hold the model as a `PyAny` object: -```rust +```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; +use pyo3::types::PyList; # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); @@ -80,21 +86,19 @@ struct UserModel { impl Model for UserModel { fn set_variables(&mut self, var: &Vec) { println!("Rust calling Python to set the variables"); - Python::with_gil(|py| { - let values: Vec = var.clone(); - let list: PyObject = values.into_py(py); - let py_model = self.model.as_ref(py); - py_model - .call_method("set_variables", (list,), None) + Python::attach(|py| { + self.model + .bind(py) + .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) .unwrap(); }) } fn get_results(&self) -> Vec { println!("Rust calling Python to get the results"); - Python::with_gil(|py| { + Python::attach(|py| { self.model - .as_ref(py) + .bind(py) .call_method("get_results", (), None) .unwrap() .extract() @@ -104,9 +108,9 @@ impl Model for UserModel { fn compute(&mut self) { println!("Rust calling Python to perform the computation"); - Python::with_gil(|py| { + Python::attach(|py| { self.model - .as_ref(py) + .bind(py) .call_method("compute", (), None) .unwrap(); }) @@ -117,8 +121,9 @@ impl Model for UserModel { Now that this bit is implemented, let's expose the model wrapper to Python. Let's add the PyO3 annotations and add a constructor: -```rust +```rust,no_run # #![allow(dead_code)] +# fn main() {} # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); # fn compute(&mut self); @@ -131,12 +136,6 @@ struct UserModel { model: Py, } -#[pymodule] -fn trait_exposure(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) -} - #[pymethods] impl UserModel { #[new] @@ -144,6 +143,12 @@ impl UserModel { UserModel { model } } } + +#[pymodule] +mod trait_exposure { + #[pymodule_export] + use super::UserModel; +} ``` Now we add the PyO3 annotations to the trait implementation: @@ -155,16 +160,17 @@ impl Model for UserModel { } ``` -However, the previous code will not compile. The compilation error is the following one: -`error: #[pymethods] cannot be used on trait impl blocks` +However, the previous code will not compile. +The compilation error is the following one: `error: #[pymethods] cannot be used on trait impl blocks` That's a bummer! However, we can write a second wrapper around these functions to call them directly. This wrapper will also perform the type conversions between Python and Rust. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; +# use pyo3::types::PyList; # # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); @@ -180,21 +186,18 @@ This wrapper will also perform the type conversions between Python and Rust. # impl Model for UserModel { # fn set_variables(&mut self, var: &Vec) { # println!("Rust calling Python to set the variables"); -# Python::with_gil(|py| { -# let values: Vec = var.clone(); -# let list: PyObject = values.into_py(py); -# let py_model = self.model.as_ref(py); -# py_model -# .call_method("set_variables", (list,), None) +# Python::attach(|py| { +# self.model.bind(py) +# .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) # .unwrap(); # }) # } # # fn get_results(&self) -> Vec { # println!("Rust calling Python to get the results"); -# Python::with_gil(|py| { +# Python::attach(|py| { # self.model -# .as_ref(py) +# .bind(py) # .call_method("get_results", (), None) # .unwrap() # .extract() @@ -204,9 +207,9 @@ This wrapper will also perform the type conversions between Python and Rust. # # fn compute(&mut self) { # println!("Rust calling Python to perform the computation"); -# Python::with_gil(|py| { +# Python::attach(|py| { # self.model -# .as_ref(py) +# .bind(py) # .call_method("compute", (), None) # .unwrap(); # }) @@ -232,8 +235,10 @@ impl UserModel { } } ``` + This wrapper handles the type conversion between the PyO3 requirements and the trait. In order to meet PyO3 requirements, this wrapper must: + - return an object of type `PyResult` - use only values, not references in the method signatures @@ -282,7 +287,6 @@ We will now expose the `solve` function, but before, let's talk about types erro What happens if you have type errors when using Python and how can you improve the error messages? - ### Wrong types in Python function arguments Let's assume in the first case that you will use in your Python file `my_rust_model.set_variables(2.0)` instead of `my_rust_model.set_variables([2.0])`. @@ -331,9 +335,10 @@ Let's modify the code performing the type conversion to give a helpful error mes We used in our `get_results` method the following call that performs the type conversion: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; +# use pyo3::types::PyList; # # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); @@ -349,9 +354,9 @@ We used in our `get_results` method the following call that performs the type co impl Model for UserModel { fn get_results(&self) -> Vec { println!("Rust calling Python to get the results"); - Python::with_gil(|py| { + Python::attach(|py| { self.model - .as_ref(py) + .bind(py) .call_method("get_results", (), None) .unwrap() .extract() @@ -360,21 +365,18 @@ impl Model for UserModel { } # fn set_variables(&mut self, var: &Vec) { # println!("Rust calling Python to set the variables"); -# Python::with_gil(|py| { -# let values: Vec = var.clone(); -# let list: PyObject = values.into_py(py); -# let py_model = self.model.as_ref(py); -# py_model -# .call_method("set_variables", (list,), None) +# Python::attach(|py| { +# self.model.bind(py) +# .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) # .unwrap(); # }) # } # # fn compute(&mut self) { # println!("Rust calling Python to perform the computation"); -# Python::with_gil(|py| { +# Python::attach(|py| { # self.model -# .as_ref(py) +# .bind(py) # .call_method("compute", (), None) # .unwrap(); # }) @@ -384,9 +386,10 @@ impl Model for UserModel { Let's break it down in order to perform better error handling: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; +# use pyo3::types::PyList; # # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); @@ -402,10 +405,10 @@ Let's break it down in order to perform better error handling: impl Model for UserModel { fn get_results(&self) -> Vec { println!("Get results from Rust calling Python"); - Python::with_gil(|py| { - let py_result: &PyAny = self + Python::attach(|py| { + let py_result: Bound<'_, PyAny> = self .model - .as_ref(py) + .bind(py) .call_method("get_results", (), None) .unwrap(); @@ -421,21 +424,18 @@ impl Model for UserModel { } # fn set_variables(&mut self, var: &Vec) { # println!("Rust calling Python to set the variables"); -# Python::with_gil(|py| { -# let values: Vec = var.clone(); -# let list: PyObject = values.into_py(py); -# let py_model = self.model.as_ref(py); -# py_model -# .call_method("set_variables", (list,), None) +# Python::attach(|py| { +# let py_model = self.model.bind(py) +# .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) # .unwrap(); # }) # } # # fn compute(&mut self) { # println!("Rust calling Python to perform the computation"); -# Python::with_gil(|py| { +# Python::attach(|py| { # self.model -# .as_ref(py) +# .bind(py) # .call_method("compute", (), None) # .unwrap(); # }) @@ -463,9 +463,11 @@ Because of this, we can write a function wrapper that takes the `UserModel`--whi It is also required to make the struct public. -```rust +```rust,no_run # #![allow(dead_code)] +# fn main() {} use pyo3::prelude::*; +use pyo3::types::PyList; pub trait Model { fn set_variables(&mut self, var: &Vec); @@ -488,13 +490,6 @@ pub struct UserModel { model: Py, } -#[pymodule] -fn trait_exposure(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_function(wrap_pyfunction!(solve_wrapper, m)?)?; - Ok(()) -} - #[pymethods] impl UserModel { #[new] @@ -517,25 +512,29 @@ impl UserModel { } } +#[pymodule] +mod trait_exposure { + #[pymodule_export] + use super::{UserModel, solve_wrapper}; +} + impl Model for UserModel { fn set_variables(&mut self, var: &Vec) { println!("Rust calling Python to set the variables"); - Python::with_gil(|py| { - let values: Vec = var.clone(); - let list: PyObject = values.into_py(py); - let py_model = self.model.as_ref(py); - py_model - .call_method("set_variables", (list,), None) + Python::attach(|py| { + self.model + .bind(py) + .call_method("set_variables", (PyList::new(py, var).unwrap(),), None) .unwrap(); }) } fn get_results(&self) -> Vec { println!("Get results from Rust calling Python"); - Python::with_gil(|py| { - let py_result: &PyAny = self + Python::attach(|py| { + let py_result: Bound<'_, PyAny> = self .model - .as_ref(py) + .bind(py) .call_method("get_results", (), None) .unwrap(); @@ -552,9 +551,9 @@ impl Model for UserModel { fn compute(&mut self) { println!("Rust calling Python to perform the computation"); - Python::with_gil(|py| { + Python::attach(|py| { self.model - .as_ref(py) + .bind(py) .call_method("compute", (), None) .unwrap(); }) diff --git a/guide/src/type-stub.md b/guide/src/type-stub.md new file mode 100644 index 00000000000..8214984a6a7 --- /dev/null +++ b/guide/src/type-stub.md @@ -0,0 +1,87 @@ +# Type stub generation (`*.pyi` files) and introspection + +*This feature is still in active development. See [the related issue](https://github.com/PyO3/pyo3/issues/5137).* + +*For documentation on type stubs and how to use them with stable PyO3, refer to [this page](python-typing-hints.md)* + +PyO3 has a work in progress support to generate [type stub files](https://typing.python.org/en/latest/spec/distributing.html#stub-files). + +It works using: + +1. PyO3 macros (`#[pyclass]`) that generate constant JSON strings that are then included in the built binaries by rustc if the `experimental-inspect` feature is enabled. +2. The `pyo3-introspection` crate that can parse the generated binaries, extract the JSON strings and build stub files from it. +3. \[Not done yet\] Build tools like `maturin` exposing `pyo3-introspection` features in their CLI API. + +For example, the following Rust code + +```rust +#[pymodule] +pub mod example { + use pyo3::prelude::*; + + #[pymodule_export] + pub const CONSTANT: &str = "FOO"; + + #[pyclass(eq)] + #[derive(Eq)] + struct Class { + value: usize + } + + #[pymethods] + impl Class { + #[new] + fn new(value: usize) -> Self { + Self { value } + } + + #[getter] + fn value(&self) -> usize { + self.value + } + } + + #[pyfunction] + #[pyo3(signature = (arg: "list[int]") -> "list[int]")] + fn list_of_int_identity(arg: Bound<'_, PyAny>) -> Bound<'_, PyAny> { + arg + } +} +``` + +will generate the following stub file: + +```python +import typing + +CONSTANT: typing.Final = "FOO" + +class Class: + def __init__(self, value: int) -> None: ... + + @property + def value(self) -> int: ... + + def __eq__(self, other: Class) -> bool: ... + def __ne__(self, other: Class) -> bool: ... + +def list_of_int_identity(arg: list[int]) -> list[int]: ... +``` + +The only piece of added syntax is that the `#[pyo3(signature = ...)]` attribute +can now contain type annotations like `#[pyo3(signature = (arg: "list[int]") -> "list[int]")]` +(note the `""` around type annotations). +This is useful when PyO3 is not able to derive proper type annotations by itself. + +## Constraints and limitations + +- The `experimental-inspect` feature is required to generate the introspection fragments. +- Lots of features are not implemented yet. + See [the related issue](https://github.com/PyO3/pyo3/issues/5137) for a list of them. +- Introspection only works with Python modules declared with an inline Rust module. + Modules declared using a function are not supported. +- `FromPyObject::INPUT_TYPE` and `IntoPyObject::OUTPUT_TYPE` must be implemented for PyO3 to get the proper input/output type annotations to use. +- Because `FromPyObject::INPUT_TYPE` and `IntoPyObject::OUTPUT_TYPE` are `const` it is not possible to build yet smart generic annotations for containers like `concat!("list[", T::OUTPUT_TYPE, "]")`. + See [this tracking issue](https://github.com/rust-lang/rust/issues/76560). +- PyO3 is not able to introspect the content of `#[pymodule]` and `#[pymodule_init]` functions. + If they are present, the module is tagged as incomplete using a fake `def __getattr__(name: str) -> Incomplete: ...` function [following best practices](https://typing.python.org/en/latest/guides/writing_stubs.html#incomplete-stubs). diff --git a/guide/src/types.md b/guide/src/types.md index 882baddb33c..585a895f9e1 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -1,306 +1,371 @@ -# GIL lifetimes, mutability and Python object types +# Python object types -On first glance, PyO3 provides a huge number of different types that can be used -to wrap or refer to Python objects. This page delves into the details and gives -an overview of their intended meaning, with examples when each type is best -used. +PyO3 offers two main sets of types to interact with Python objects. +This section of the guide expands into detail about these types and how to choose which to use. +The first set of types are the [smart pointers][smart-pointers] which all Python objects are wrapped in. +These are `Py`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`. +The [first section below](#pyo3s-smart-pointers) expands on each of these in detail and why there are three of them. -## The Python GIL, mutability, and Rust types +The second set of types are types which fill in the generic parameter `T` of the smart pointers. +The most common is `PyAny`, which represents any Python object (similar to Python's `typing.Any`). +There are also concrete types for many Python built-in types, such as `PyList`, `PyDict`, and `PyTuple`. +User defined `#[pyclass]` types also fit this category. +The [second section below](#concrete-python-types) expands on how to use these types. -Since Python has no concept of ownership, and works solely with boxed objects, -any Python object can be referenced any number of times, and mutation is allowed -from any reference. +## PyO3's smart pointers -The situation is helped a little by the Global Interpreter Lock (GIL), which -ensures that only one thread can use the Python interpreter and its API at the -same time, while non-Python operations (system calls and extension code) can -unlock the GIL. (See [the section on parallelism](parallelism.md) for how to do -that in PyO3.) +PyO3's API offers three generic smart pointers: `Py`, `Bound<'py, T>` and `Borrowed<'a, 'py, T>`. +For each of these the type parameter `T` will be filled by a [concrete Python type](#concrete-python-types). +For example, a Python list object can be represented by `Py`, `Bound<'py, PyList>`, and `Borrowed<'a, 'py, PyList>`. -In PyO3, holding the GIL is modeled by acquiring a token of the type -`Python<'py>`, which serves three purposes: +These smart pointers behave differently due to their lifetime parameters. `Py` has no lifetime parameters, `Bound<'py, T>` has [the `'py` lifetime](./python-from-rust.md#the-py-lifetime) as a parameter, and `Borrowed<'a, 'py, T>` has the `'py` lifetime plus an additional lifetime `'a` to denote the lifetime it is borrowing data for. (You can read more about these lifetimes in the subsections below). -* It provides some global API for the Python interpreter, such as - [`eval`][eval]. -* It can be passed to functions that require a proof of holding the GIL, - such as [`Py::clone_ref`][clone_ref]. -* Its lifetime can be used to create Rust references that implicitly guarantee - holding the GIL, such as [`&'py PyAny`][PyAny]. +Python objects are reference counted, like [`std::sync::Arc`](https://doc.rust-lang.org/stable/std/sync/struct.Arc.html). +A major reason for these smart pointers is to bring Python's reference counting to a Rust API. -The latter two points are the reason why some APIs in PyO3 require the `py: -Python` argument, while others don't. +The recommendation of when to use each of these smart pointers is as follows: -The PyO3 API for Python objects is written such that instead of requiring a -mutable Rust reference for mutating operations such as -[`PyList::append`][PyList_append], a shared reference (which, in turn, can only -be created through `Python<'_>` with a GIL lifetime) is sufficient. +- Use `Bound<'py, T>` for as much as possible, as it offers the most efficient and complete API. +- Use `Py` mostly just for storage inside Rust `struct`s which do not want to or can't add a lifetime parameter for `Bound<'py, T>`. +- `Borrowed<'a, 'py, T>` is almost never used. + It is occasionally present at the boundary between Rust and the Python interpreter, for example when borrowing data from Python tuples (which is safe because they are immutable). -However, Rust structs wrapped as Python objects (called `pyclass` types) usually -*do* need `&mut` access. Due to the GIL, PyO3 *can* guarantee thread-safe access -to them, but it cannot statically guarantee uniqueness of `&mut` references once -an object's ownership has been passed to the Python interpreter, ensuring -references is done at runtime using `PyCell`, a scheme very similar to -`std::cell::RefCell`. +The sections below also explain these smart pointers in a little more detail. -### Accessing the Python GIL +### `Py` -To get hold of a `Python<'py>` token to prove the GIL is held, consult [PyO3's documentation][obtaining-py]. +[`Py`][Py] is the foundational smart pointer in PyO3's API. +The type parameter `T` denotes the type of the Python object. +Very frequently this is `PyAny`, meaning any Python object. -## Object types +Because `Py` is not bound to [the `'py` lifetime](./python-from-rust.md#the-py-lifetime), it is the type to use when storing a Python object inside a Rust `struct` or `enum` which do not want to have a lifetime parameter. +In particular, [`#[pyclass]`][pyclass] types are not permitted to have a lifetime, so `Py` is the correct type to store Python objects inside them. -### [`PyAny`][PyAny] +The lack of binding to the `'py` lifetime also carries drawbacks: -**Represents:** a Python object of unspecified type, restricted to a GIL -lifetime. Currently, `PyAny` can only ever occur as a reference, `&PyAny`. +- Almost all methods on `Py` require a `Python<'py>` token as the first argument +- Other functionality, such as [`Drop`][Drop], needs to check at runtime for attachment to the Python interpreter, at a small performance cost -**Used:** Whenever you want to refer to some Python object and will have the -GIL for the whole duration you need to access that object. For example, -intermediate values and arguments to `pyfunction`s or `pymethod`s implemented -in Rust where any type is allowed. +Because of the drawbacks `Bound<'py, T>` is preferred for many of PyO3's APIs. +In particular, `Bound<'py, T>` is better for function arguments. -Many general methods for interacting with Python objects are on the `PyAny` struct, -such as `getattr`, `setattr`, and `.call`. +To convert a `Py` into a `Bound<'py, T>`, the `Py::bind` and `Py::into_bound` methods are available. `Bound<'py, T>` can be converted back into `Py` using [`Bound::unbind`]. -**Conversions:** +### `Bound<'py, T>` -For a `&PyAny` object reference `any` where the underlying object is a Python-native type such as -a list: +[`Bound<'py, T>`][Bound] is the counterpart to `Py` which is also bound to the `'py` lifetime. +It can be thought of as equivalent to the Rust tuple `(Python<'py>, Py)`. -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyList; -# Python::with_gil(|py| -> PyResult<()> { -#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. -let obj: &PyAny = PyList::empty(py); +By having the binding to the `'py` lifetime, `Bound<'py, T>` can offer the complete PyO3 API at maximum efficiency. +This means that `Bound<'py, T>` should usually be used whenever carrying this lifetime is acceptable, and `Py` otherwise. -// To &PyList with PyAny::downcast -let _: &PyList = obj.downcast()?; +`Bound<'py, T>` engages in Python reference counting. +This means that `Bound<'py, T>` owns a Python object. +Rust code which just wants to borrow a Python object should use a shared reference `&Bound<'py, T>`. +Just like `std::sync::Arc`, using `.clone()` and `drop()` will cheaply increment and decrement the reference count of the object (just in this case, the reference counting is implemented by the Python interpreter itself). -// To Py (aka PyObject) with .into() -let _: Py = obj.into(); +To give an example of how `Bound<'py, T>` is PyO3's primary API type, consider the following Python code: -// To Py with PyAny::extract -let _: Py = obj.extract()?; -# Ok(()) -# }).unwrap(); +```python +def example(): + x = list() # create a Python list + x.append(1) # append the integer 1 to it + y = x # create a second reference to the list + del x # delete the original reference ``` -For a `&PyAny` object reference `any` where the underlying object is a `#[pyclass]`: +Using PyO3's API, and in particular `Bound<'py, PyList>`, this code translates into the following Rust code: ```rust -# use pyo3::prelude::*; -# #[pyclass] #[derive(Clone)] struct MyClass { } -# Python::with_gil(|py| -> PyResult<()> { -let obj: &PyAny = Py::new(py, MyClass {})?.into_ref(py); - -// To &PyCell with PyAny::downcast -let _: &PyCell = obj.downcast()?; +use pyo3::prelude::*; +use pyo3::types::PyList; + +fn example<'py>(py: Python<'py>) -> PyResult<()> { + let x: Bound<'py, PyList> = PyList::empty(py); + x.append(1)?; + let y: Bound<'py, PyList> = x.clone(); // y is a new reference to the same list + drop(x); // release the original reference x + Ok(()) +} +# Python::attach(example).unwrap(); +``` -// To Py (aka PyObject) with .into() -let _: Py = obj.into(); +Or, without the type annotations: -// To Py with PyAny::extract -let _: Py = obj.extract()?; +```rust +use pyo3::prelude::*; +use pyo3::types::PyList; + +fn example(py: Python<'_>) -> PyResult<()> { + let x = PyList::empty(py); + x.append(1)?; + let y = x.clone(); + drop(x); + Ok(()) +} +# Python::attach(example).unwrap(); +``` -// To MyClass with PyAny::extract, if MyClass: Clone -let _: MyClass = obj.extract()?; +#### Function argument lifetimes -// To PyRef<'_, MyClass> or PyRefMut<'_, MyClass> with PyAny::extract -let _: PyRef<'_, MyClass> = obj.extract()?; -let _: PyRefMut<'_, MyClass> = obj.extract()?; -# Ok(()) -# }).unwrap(); -``` +Because the `'py` lifetime often appears in many function arguments as part of the `Bound<'py, T>` smart pointer, the Rust compiler will often require annotations of input and output lifetimes. +This occurs when the function output has at least one lifetime, and there is more than one lifetime present on the inputs. -### `PyTuple`, `PyDict`, and many more +To demonstrate, consider this function which takes accepts Python objects and applies the [Python `+` operation][PyAnyMethods::add] to them: -**Represents:** a native Python object of known type, restricted to a GIL -lifetime just like `PyAny`. +```rust,compile_fail +# use pyo3::prelude::*; +fn add(left: &'_ Bound<'_, PyAny>, right: &'_ Bound<'_, PyAny>) -> PyResult> { + left.add(right) +} +``` -**Used:** Whenever you want to operate with native Python types while holding -the GIL. Like `PyAny`, this is the most convenient form to use for function -arguments and intermediate values. +Because the Python `+` operation might raise an exception, this function returns `PyResult>`. +It doesn't need ownership of the inputs, so it takes `&Bound<'_, PyAny>` shared references. +To demonstrate the point, all lifetimes have used the wildcard `'_` to allow the Rust compiler to attempt to infer them. +Because there are four input lifetimes (two lifetimes of the shared references, and two `'py` lifetimes unnamed inside the `Bound<'_, PyAny>` pointers), the compiler cannot reason about which must be connected to the output. -These types all implement `Deref`, so they all expose the same -methods which can be found on `PyAny`. +The correct way to solve this is to add the `'py` lifetime as a parameter for the function, and name all the `'py` lifetimes inside the `Bound<'py, PyAny>` smart pointers. +For the shared references, it's also fine to reduce `&'_` to just `&`. +The working end result is below: -To see all Python types exposed by `PyO3` you should consult the -[`pyo3::types`][pyo3::types] module. +```rust +# use pyo3::prelude::*; +fn add<'py>( + left: &Bound<'py, PyAny>, + right: &Bound<'py, PyAny>, +) -> PyResult> { + left.add(right) +} +# Python::attach(|py| { +# let s = pyo3::types::PyString::new(py, "s"); +# assert!(add(&s, &s).unwrap().eq("ss").unwrap()); +# }) +``` -**Conversions:** +If naming the `'py` lifetime adds unwanted complexity to the function signature, it is also acceptable to return `Py`, which has no lifetime. +The cost is instead paid by a slight increase in implementation complexity, as seen by the introduction of a call to [`Bound::unbind`]: ```rust # use pyo3::prelude::*; -# use pyo3::types::PyList; -# Python::with_gil(|py| -> PyResult<()> { -#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. -let list = PyList::empty(py); +fn add(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult> { + let output: Bound<'_, PyAny> = left.add(right)?; + Ok(output.unbind()) +} +# Python::attach(|py| { +# let s = pyo3::types::PyString::new(py, "s"); +# assert!(add(&s, &s).unwrap().bind(py).eq("ss").unwrap()); +# }) +``` -// Use methods from PyAny on all Python types with Deref implementation -let _ = list.repr()?; +### `Borrowed<'a, 'py, T>` -// To &PyAny automatically with Deref implementation -let _: &PyAny = list; +[`Borrowed<'a, 'py, T>`][Borrowed] is an advanced type used just occasionally at the edge of interaction with the Python interpreter. +It can be thought of as analogous to the shared reference `&'a Bound<'py, T>`. +The difference is that `Borrowed<'a, 'py, T>` is just a smart pointer rather than a reference-to-a-smart-pointer, which is a helpful reduction in indirection in specific interactions with the Python interpreter. -// To &PyAny explicitly with .as_ref() -let _: &PyAny = list.as_ref(); +`Borrowed<'a, 'py, T>` dereferences to `Bound<'py, T>`, so all methods on `Bound<'py, T>` are available on `Borrowed<'a, 'py, T>`. -// To Py with .into() or Py::from() -let _: Py = list.into(); +An example where `Borrowed<'a, 'py, T>` is used is in [`PyTupleMethods::get_borrowed_item`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyTupleMethods.html#tymethod.get_item): -// To PyObject with .into() or .to_object(py) -let _: PyObject = list.into(); +```rust +use pyo3::prelude::*; +use pyo3::types::PyTuple; + +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +// Create a new tuple with the elements (0, 1, 2) +let t = PyTuple::new(py, [0, 1, 2])?; +for i in 0..=2 { + let entry: Borrowed<'_, 'py, PyAny> = t.get_borrowed_item(i)?; + // `PyAnyMethods::extract` is available on `Borrowed` + // via the dereference to `Bound` + let value: usize = entry.extract()?; + assert_eq!(i, value); +} # Ok(()) -# }).unwrap(); +# } +# Python::attach(example).unwrap(); ``` -### `Py` and `PyObject` +### Casting between smart pointer types -**Represents:** a GIL-independent reference to a Python object. This can be a Python native type -(like `PyTuple`), or a `pyclass` type implemented in Rust. The most commonly-used variant, -`Py`, is also known as `PyObject`. +To convert between `Py` and `Bound<'py, T>` use the `bind()` / `into_bound()` methods. +Use the `as_unbound()` / `unbind()` methods to go back from `Bound<'py, T>` to `Py`. -**Used:** Whenever you want to carry around references to a Python object without caring about a -GIL lifetime. For example, storing Python object references in a Rust struct that outlives the -Python-Rust FFI boundary, or returning objects from functions implemented in Rust back to Python. +```rust,ignore +let obj: Py = ...; +let bound: &Bound<'py, PyAny> = obj.bind(py); +let bound: Bound<'py, PyAny> = obj.into_bound(py); -Can be cloned using Python reference counts with `.clone()`. +let obj: &Py = bound.as_unbound(); +let obj: Py = bound.unbind(); +``` -**Conversions:** +To convert between `Bound<'py, T>` and `Borrowed<'a, 'py, T>` use the `as_borrowed()` method. `Borrowed<'a, 'py, T>` has a deref coercion to `Bound<'py, T>`. +Use the `to_owned()` method to increment the Python reference count and to create a new `Bound<'py, T>` from the `Borrowed<'a, 'py, T>`. -For a `Py`, the conversions are as below: +```rust,ignore +let bound: Bound<'py, PyAny> = ...; +let borrowed: Borrowed<'_, 'py, PyAny> = bound.as_borrowed(); -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyList; -# Python::with_gil(|py| { -let list: Py = PyList::empty_bound(py).unbind(); +// deref coercion +let bound: &Bound<'py, PyAny> = &borrowed; -// To &PyList with Py::as_ref() (borrows from the Py) -let _: &PyList = list.as_ref(py); +// create a new Bound by increase the Python reference count +let bound: Bound<'py, PyAny> = borrowed.to_owned(); +``` -# let list_clone = list.clone(); // Because `.into_ref()` will consume `list`. -// To &PyList with Py::into_ref() (moves the pointer into PyO3's object storage) -let _: &PyList = list.into_ref(py); +To convert between `Py` and `Borrowed<'a, 'py, T>` use the `bind_borrowed()` method. +Use either `as_unbound()` or `.to_owned().unbind()` to go back to `Py` from `Borrowed<'a, 'py, T>`, via `Bound<'py, T>`. -# let list = list_clone; -// To Py (aka PyObject) with .into() -let _: Py = list.into(); -# }) -``` +```rust,ignore +let obj: Py = ...; +let borrowed: Borrowed<'_, 'py, PyAny> = bound.as_borrowed(); -For a `#[pyclass] struct MyClass`, the conversions for `Py` are below: +// via deref coercion to Bound and then using Bound::as_unbound +let obj: &Py = borrowed.as_unbound(); -```rust -# use pyo3::prelude::*; -# Python::with_gil(|py| { -# #[pyclass] struct MyClass { } -# Python::with_gil(|py| -> PyResult<()> { -let my_class: Py = Py::new(py, MyClass { })?; +// via a new Bound by increasing the Python reference count, and unbind it +let obj: Py = borrowed.to_owned().unbind(). +``` -// To &PyCell with Py::as_ref() (borrows from the Py) -let _: &PyCell = my_class.as_ref(py); +## Concrete Python types -# let my_class_clone = my_class.clone(); // Because `.into_ref()` will consume `my_class`. -// To &PyCell with Py::into_ref() (moves the pointer into PyO3's object storage) -let _: &PyCell = my_class.into_ref(py); +In all of `Py`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`, the type parameter `T` denotes the type of the Python object referred to by the smart pointer. -# let my_class = my_class_clone.clone(); -// To Py (aka PyObject) with .into_py(py) -let _: Py = my_class.into_py(py); +This parameter `T` can be filled by: -# let my_class = my_class_clone; -// To PyRef<'_, MyClass> with Py::borrow or Py::try_borrow -let _: PyRef<'_, MyClass> = my_class.try_borrow(py)?; +- [`PyAny`][PyAny], which represents any Python object, +- Native Python types such as `PyList`, `PyTuple`, and `PyDict`, and +- [`#[pyclass]`][pyclass] types defined from Rust -// To PyRefMut<'_, MyClass> with Py::borrow_mut or Py::try_borrow_mut -let _: PyRefMut<'_, MyClass> = my_class.try_borrow_mut(py)?; -# Ok(()) -# }).unwrap(); -# }); -``` +The following subsections covers some further detail about how to work with these types: + +- the APIs that are available for these concrete types, +- how to cast `Bound<'py, T>` to a specific concrete type, and +- how to get Rust data out of a `Bound<'py, T>`. -### `PyCell` +### Using APIs for concrete Python types -**Represents:** a reference to a Rust object (instance of `PyClass`) which is -wrapped in a Python object. The cell part is an analog to stdlib's -[`RefCell`][RefCell] to allow access to `&mut` references. +Each concrete Python type such as `PyAny`, `PyTuple` and `PyDict` exposes its API on the corresponding bound smart pointer `Bound<'py, PyAny>`, `Bound<'py, PyTuple>` and `Bound<'py, PyDict>`. -**Used:** for accessing pure-Rust API of the instance (members and functions -taking `&SomeType` or `&mut SomeType`) while maintaining the aliasing rules of -Rust references. +Each type's API is exposed as a trait: [`PyAnyMethods`], [`PyTupleMethods`], [`PyDictMethods`], and so on for all concrete types. +Using traits rather than associated methods on the `Bound` smart pointer is done for a couple of reasons: -Like PyO3's Python native types, `PyCell` implements `Deref`, -so it also exposes all of the methods on `PyAny`. +- Clarity of documentation: each trait gets its own documentation page in the PyO3 API docs. + If all methods were on the `Bound` smart pointer directly, the vast majority of PyO3's API would be on a single, extremely long, documentation page. +- Consistency: downstream code implementing Rust APIs for existing Python types can also follow this pattern of using a trait. + Downstream code would not be allowed to add new associated methods directly on the `Bound` type. +- Future design: it is hoped that a future Rust with [arbitrary self types](https://github.com/rust-lang/rust/issues/44874) will remove the need for these traits in favour of placing the methods directly on `PyAny`, `PyTuple`, `PyDict`, and so on. -**Conversions:** +These traits are all included in the `pyo3::prelude` module, so with the glob import `use pyo3::prelude::*` the full PyO3 API is made available to downstream code. -`PyCell` can be used to access `&T` and `&mut T` via `PyRef` and `PyRefMut` respectively. +The following function accesses the first item in the input Python list, using the `.get_item()` method from the `PyListMethods` trait: ```rust -# use pyo3::prelude::*; -# #[pyclass] struct MyClass { } -# Python::with_gil(|py| -> PyResult<()> { -let cell: &PyCell = PyCell::new(py, MyClass {})?; - -// To PyRef with .borrow() or .try_borrow() -let py_ref: PyRef<'_, MyClass> = cell.try_borrow()?; -let _: &MyClass = &*py_ref; -# drop(py_ref); - -// To PyRefMut with .borrow_mut() or .try_borrow_mut() -let mut py_ref_mut: PyRefMut<'_, MyClass> = cell.try_borrow_mut()?; -let _: &mut MyClass = &mut *py_ref_mut; -# Ok(()) -# }).unwrap(); +use pyo3::prelude::*; +use pyo3::types::PyList; + +fn get_first_item<'py>(list: &Bound<'py, PyList>) -> PyResult> { + list.get_item(0) +} +# Python::attach(|py| { +# let l = PyList::new(py, ["hello world"]).unwrap(); +# assert!(get_first_item(&l).unwrap().eq("hello world").unwrap()); +# }) ``` -`PyCell` can also be accessed like a Python-native type. +### Casting between Python object types + +To cast `Bound<'py, T>` smart pointers to some other type, use the [`.cast()`][Bound::cast] family of functions. +This converts `&Bound<'py, T>` to a different `&Bound<'py, U>`, without transferring ownership. +There is also [`.cast_into()`][Bound::cast_into] to convert `Bound<'py, T>` to `Bound<'py, U>` with transfer of ownership. +These methods are available for all types `T` which implement the [`PyTypeCheck`] trait. + +Casting to `Bound<'py, PyAny>` can be done with `.as_any()` or `.into_any()`. + +For example, the following snippet shows how to cast `Bound<'py, PyAny>` to `Bound<'py, PyTuple>`: ```rust # use pyo3::prelude::*; -# #[pyclass] struct MyClass { } -# Python::with_gil(|py| -> PyResult<()> { -let cell: &PyCell = PyCell::new(py, MyClass {})?; +# use pyo3::types::PyTuple; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +// create a new Python `tuple`, and use `.into_any()` to erase the type +let obj: Bound<'py, PyAny> = PyTuple::empty(py).into_any(); -// Use methods from PyAny on PyCell with Deref implementation -let _ = cell.repr()?; +// use `.cast()` to cast to `PyTuple` without transferring ownership +let _: &Bound<'py, PyTuple> = obj.cast()?; -// To &PyAny automatically with Deref implementation -let _: &PyAny = cell; - -// To &PyAny explicitly with .as_ref() -let _: &PyAny = cell.as_ref(); +// use `.cast_into()` to cast to `PyTuple` with transfer of ownership +let _: Bound<'py, PyTuple> = obj.cast_into()?; # Ok(()) -# }).unwrap(); +# } +# Python::attach(example).unwrap() ``` -### `PyRef` and `PyRefMut` +Custom [`#[pyclass]`][pyclass] types implement [`PyTypeCheck`], so `.cast()` also works for these types. +The snippet below is the same as the snippet above casting instead to a custom type `MyClass`: -**Represents:** reference wrapper types employed by `PyCell` to keep track of -borrows, analog to `Ref` and `RefMut` used by `RefCell`. +```rust +use pyo3::prelude::*; -**Used:** while borrowing a `PyCell`. They can also be used with `.extract()` -on types like `Py` and `PyAny` to get a reference quickly. +#[pyclass] +struct MyClass {} +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +// create a new Python `tuple`, and use `.into_any()` to erase the type +let obj: Bound<'py, PyAny> = Bound::new(py, MyClass {})?.into_any(); -## Related traits and types +// use `.cast()` to cast to `MyClass` without transferring ownership +let _: &Bound<'py, MyClass> = obj.cast()?; -### `PyClass` +// use `.cast_into()` to cast to `MyClass` with transfer of ownership +let _: Bound<'py, MyClass> = obj.cast_into()?; +# Ok(()) +# } +# Python::attach(example).unwrap() +``` -This trait marks structs defined in Rust that are also usable as Python classes, -usually defined using the `#[pyclass]` macro. +### Extracting Rust data from Python objects -### `PyNativeType` +To extract Rust data from Python objects, use [`.extract()`][PyAnyMethods::extract] instead of `.cast()`. +This method is available for all types which implement the [`FromPyObject`] trait. -This trait marks structs that mirror native Python types, such as `PyList`. +For example, the following snippet extracts a Rust tuple of integers from a Python tuple: +```rust +# use pyo3::prelude::*; +# use pyo3::types::PyTuple; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +// create a new Python `tuple`, and use `.into_any()` to erase the type +let obj: Bound<'py, PyAny> = PyTuple::new(py, [1, 2, 3])?.into_any(); + +// extracting the Python `tuple` to a rust `(i32, i32, i32)` tuple +let (x, y, z) = obj.extract::<(i32, i32, i32)>()?; +assert_eq!((x, y, z), (1, 2, 3)); +# Ok(()) +# } +# Python::attach(example).unwrap() +``` -[eval]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval -[clone_ref]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html#method.clone_ref -[pyo3::types]: {{#PYO3_DOCS_URL}}/pyo3/types/index.html +To avoid copying data, [`#[pyclass]`][pyclass] types can directly reference Rust data stored within the Python objects without needing to `.extract()`. +See the [corresponding documentation in the class section of the guide](./class.md#bound-and-interior-mutability) for more detail. + +[Bound]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html +[`Bound::unbind`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html#method.unbind +[Py]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html +[PyAnyMethods::add]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.add +[PyAnyMethods::extract]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.extract +[Bound::cast]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html#method.cast +[Bound::cast_into]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html#method.cast_into +[`PyTypeCheck`]: {{#PYO3_DOCS_URL}}/pyo3/type_object/trait.PyTypeCheck.html +[`PyAnyMethods`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html +[`PyDictMethods`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyDictMethods.html +[`PyTupleMethods`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyTupleMethods.html +[pyclass]: class.md +[Borrowed]: {{#PYO3_DOCS_URL}}/pyo3/struct.Borrowed.html +[Drop]: https://doc.rust-lang.org/std/ops/trait.Drop.html [PyAny]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html -[PyList_append]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyList.html#method.append -[RefCell]: https://doc.rust-lang.org/std/cell/struct.RefCell.html -[obtaining-py]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#obtaining-a-python-token +[smart-pointers]: https://doc.rust-lang.org/book/ch15-00-smart-pointers.html +[`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html diff --git a/guide/theme/tabs.css b/guide/theme/tabs.css new file mode 100644 index 00000000000..8712b859c0b --- /dev/null +++ b/guide/theme/tabs.css @@ -0,0 +1,25 @@ +.mdbook-tabs { + display: flex; +} + +.mdbook-tab { + background-color: var(--table-alternate-bg); + padding: 0.5rem 1rem; + cursor: pointer; + border: none; + font-size: 1.6rem; + line-height: 1.45em; +} + +.mdbook-tab.active { + background-color: var(--table-header-bg); + font-weight: bold; +} + +.mdbook-tab-content { + padding: 1rem 0rem; +} + +.mdbook-tab-content table { + margin: unset; +} diff --git a/guide/theme/tabs.js b/guide/theme/tabs.js new file mode 100644 index 00000000000..8ba5e878c39 --- /dev/null +++ b/guide/theme/tabs.js @@ -0,0 +1,75 @@ +/** + * Change active tab of tabs. + * + * @param {Element} container + * @param {string} name + */ +const changeTab = (container, name) => { + for (const child of container.children) { + if (!(child instanceof HTMLElement)) { + continue; + } + + if (child.classList.contains('mdbook-tabs')) { + for (const tab of child.children) { + if (!(tab instanceof HTMLElement)) { + continue; + } + + if (tab.dataset.tabname === name) { + tab.classList.add('active'); + } else { + tab.classList.remove('active'); + } + } + } else if (child.classList.contains('mdbook-tab-content')) { + if (child.dataset.tabname === name) { + child.classList.remove('hidden'); + } else { + child.classList.add('hidden'); + } + } + } +}; + +document.addEventListener('DOMContentLoaded', () => { + const tabs = document.querySelectorAll('.mdbook-tab'); + for (const tab of tabs) { + tab.addEventListener('click', () => { + if (!(tab instanceof HTMLElement)) { + return; + } + + if (!tab.parentElement || !tab.parentElement.parentElement) { + return; + } + + const container = tab.parentElement.parentElement; + const name = tab.dataset.tabname; + const global = container.dataset.tabglobal; + + changeTab(container, name); + + if (global) { + localStorage.setItem(`mdbook-tabs-${global}`, name); + + const globalContainers = document.querySelectorAll( + `.mdbook-tabs-container[data-tabglobal="${global}"]` + ); + for (const globalContainer of globalContainers) { + changeTab(globalContainer, name); + } + } + }); + } + + const containers = document.querySelectorAll('.mdbook-tabs-container[data-tabglobal]'); + for (const container of containers) { + const global = container.dataset.tabglobal; + + const name = localStorage.getItem(`mdbook-tabs-${global}`); + if (name && document.querySelector(`.mdbook-tab[data-tabname=${name}]`)) { + changeTab(container, name); + } + } +}); diff --git a/newsfragments/3514.added.md b/newsfragments/3514.added.md deleted file mode 100644 index 7fbf662b2ec..00000000000 --- a/newsfragments/3514.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyMemoryView` type. diff --git a/newsfragments/3532.changed.md b/newsfragments/3532.changed.md deleted file mode 100644 index b65f240931e..00000000000 --- a/newsfragments/3532.changed.md +++ /dev/null @@ -1 +0,0 @@ -- `PyDict::from_sequence` now takes a single argument of type `&PyAny` (previously took two arguments `Python` and `PyObject`). diff --git a/newsfragments/3540.added.md b/newsfragments/3540.added.md deleted file mode 100644 index 2b113193bef..00000000000 --- a/newsfragments/3540.added.md +++ /dev/null @@ -1 +0,0 @@ -Support `async fn` in macros with coroutine implementation \ No newline at end of file diff --git a/newsfragments/3577.added.md b/newsfragments/3577.added.md deleted file mode 100644 index 632274984ec..00000000000 --- a/newsfragments/3577.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement `PyTypeInfo` for `PyEllipsis`, `PyNone` and `PyNotImplemented`. diff --git a/newsfragments/3577.changed.md b/newsfragments/3577.changed.md deleted file mode 100644 index a7e6629d6a5..00000000000 --- a/newsfragments/3577.changed.md +++ /dev/null @@ -1 +0,0 @@ -Deprecate `Py::is_ellipsis` and `PyAny::is_ellipsis` in favour of `any.is(py.Ellipsis())`. diff --git a/newsfragments/3582.added.md b/newsfragments/3582.added.md deleted file mode 100644 index 59659a8819d..00000000000 --- a/newsfragments/3582.added.md +++ /dev/null @@ -1 +0,0 @@ -Support `#[pyclass]` on enums that have non-unit variants. diff --git a/newsfragments/3588.added.md b/newsfragments/3588.added.md deleted file mode 100644 index acddf296a6f..00000000000 --- a/newsfragments/3588.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `__name__`/`__qualname__` attributes to `Coroutine` \ No newline at end of file diff --git a/newsfragments/3599.added.md b/newsfragments/3599.added.md deleted file mode 100644 index 36078fbcdb6..00000000000 --- a/newsfragments/3599.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `coroutine::CancelHandle` to catch coroutine cancellation \ No newline at end of file diff --git a/newsfragments/3600.changed.md b/newsfragments/3600.changed.md deleted file mode 100644 index c8701ef4b25..00000000000 --- a/newsfragments/3600.changed.md +++ /dev/null @@ -1 +0,0 @@ -Split some `PyTypeInfo` functionality into new traits `HasPyGilRef` and `PyTypeCheck`. diff --git a/newsfragments/3601.changed.md b/newsfragments/3601.changed.md deleted file mode 100644 index 413765ecad5..00000000000 --- a/newsfragments/3601.changed.md +++ /dev/null @@ -1 +0,0 @@ -Deprecate `PyTryFrom` and `PyTryInto` traits in favor of `any.downcast()` via the `PyTypeCheck` and `PyTypeInfo` traits. diff --git a/newsfragments/3603.removed.md b/newsfragments/3603.removed.md deleted file mode 100644 index e8f5004e3b9..00000000000 --- a/newsfragments/3603.removed.md +++ /dev/null @@ -1 +0,0 @@ -Remove all functionality deprecated in PyO3 0.19. diff --git a/newsfragments/3609.changed.md b/newsfragments/3609.changed.md deleted file mode 100644 index 7979ea71960..00000000000 --- a/newsfragments/3609.changed.md +++ /dev/null @@ -1 +0,0 @@ -Allow async methods to accept `&self`/`&mut self` \ No newline at end of file diff --git a/newsfragments/3619.fixed.md b/newsfragments/3619.fixed.md deleted file mode 100644 index 690542409f4..00000000000 --- a/newsfragments/3619.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Use portable-atomic to support platforms without 64-bit atomics diff --git a/newsfragments/3632.added.md b/newsfragments/3632.added.md deleted file mode 100644 index d9c954fa0b4..00000000000 --- a/newsfragments/3632.added.md +++ /dev/null @@ -1 +0,0 @@ -Add support for extracting Rust set types from `frozenset`. diff --git a/newsfragments/3638.changed.md b/newsfragments/3638.changed.md deleted file mode 100644 index 6bdafde8422..00000000000 --- a/newsfragments/3638.changed.md +++ /dev/null @@ -1 +0,0 @@ -Values of type `bool` can now be extracted from NumPy's `bool_`. diff --git a/newsfragments/3653.changed.md b/newsfragments/3653.changed.md deleted file mode 100644 index 75fea03cb71..00000000000 --- a/newsfragments/3653.changed.md +++ /dev/null @@ -1 +0,0 @@ -Add `AsRefSource` to `PyNativeType`. diff --git a/newsfragments/3657.changed.md b/newsfragments/3657.changed.md deleted file mode 100644 index 0a519b09d62..00000000000 --- a/newsfragments/3657.changed.md +++ /dev/null @@ -1 +0,0 @@ -Changed `.is_true` to `.is_truthy` on `PyAny` and `Py` to clarify that the test is not based on identity with or equality to the True singleton. diff --git a/newsfragments/3660.changed.md b/newsfragments/3660.changed.md deleted file mode 100644 index 8b4a3f734e1..00000000000 --- a/newsfragments/3660.changed.md +++ /dev/null @@ -1 +0,0 @@ -`PyType::name` is now `PyType::qualname` whereas `PyType::name` efficiently accesses the full name which includes the module name. diff --git a/newsfragments/3661.changed.md b/newsfragments/3661.changed.md deleted file mode 100644 index 8245a6f1a80..00000000000 --- a/newsfragments/3661.changed.md +++ /dev/null @@ -1 +0,0 @@ -The `Iter(A)NextOutput` types are now deprecated and `__(a)next__` can directly return anything which can be converted into Python objects, i.e. awaitables do not need to be wrapped into `IterANextOutput` or `Option` any more. `Option` can still be used as well and returning `None` will trigger the fast path for `__next__`, stopping iteration without having to raise a `StopIteration` exception. diff --git a/newsfragments/3663.changed.md b/newsfragments/3663.changed.md deleted file mode 100644 index 13c07e01f2d..00000000000 --- a/newsfragments/3663.changed.md +++ /dev/null @@ -1 +0,0 @@ -Implements `FromPyObject` on `chrono::DateTime` for all `Tz` and not only `FixedOffset` and `Utc` \ No newline at end of file diff --git a/newsfragments/3664.changed.md b/newsfragments/3664.changed.md deleted file mode 100644 index 3a167d2f9d2..00000000000 --- a/newsfragments/3664.changed.md +++ /dev/null @@ -1 +0,0 @@ -`chrono` conversions are compatible with `abi3` \ No newline at end of file diff --git a/newsfragments/3670.added.md b/newsfragments/3670.added.md deleted file mode 100644 index a524261e9d9..00000000000 --- a/newsfragments/3670.added.md +++ /dev/null @@ -1 +0,0 @@ -`FromPyObject`, `IntoPy` and `ToPyObject` are implemented on `std::duration::Duration` \ No newline at end of file diff --git a/newsfragments/3677.added.md b/newsfragments/3677.added.md deleted file mode 100644 index 3e6bc56d582..00000000000 --- a/newsfragments/3677.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyString::to_cow`. Add `Py::to_str`, `Py::to_cow`, and `Py::to_string_lossy`, as ways to access Python string data safely beyond the GIL lifetime. diff --git a/newsfragments/3679.changed.md b/newsfragments/3679.changed.md deleted file mode 100644 index ab46598ad65..00000000000 --- a/newsfragments/3679.changed.md +++ /dev/null @@ -1 +0,0 @@ -Add lifetime parameter to `PyTzInfoAccess` trait. For the deprecated gil-ref API, the trait is now implemented for `&'py PyTime` and `&'py PyDateTime` instead of `PyTime` and `PyDate`. diff --git a/newsfragments/3686.added.md b/newsfragments/3686.added.md deleted file mode 100644 index f808df3685a..00000000000 --- a/newsfragments/3686.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `Bound` and `Borrowed` smart pointers as a new API for accessing Python objects. diff --git a/newsfragments/3689.changed.md b/newsfragments/3689.changed.md deleted file mode 100644 index 30928e82f64..00000000000 --- a/newsfragments/3689.changed.md +++ /dev/null @@ -1 +0,0 @@ -Calls to `__traverse__` become no-ops for unsendable pyclasses if on the wrong thread, thereby avoiding hard aborts at the cost of potential leakage. diff --git a/newsfragments/3692.added.md b/newsfragments/3692.added.md deleted file mode 100644 index 45cdd5aba28..00000000000 --- a/newsfragments/3692.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyNativeType::as_bound` to convert "GIL refs" to the new `Bound` smart pointer. diff --git a/newsfragments/3692.changed.md b/newsfragments/3692.changed.md deleted file mode 100644 index 9535cbb23db..00000000000 --- a/newsfragments/3692.changed.md +++ /dev/null @@ -1 +0,0 @@ -Include `PyNativeType` in `pyo3::prelude`. diff --git a/newsfragments/3706.added.md b/newsfragments/3706.added.md deleted file mode 100644 index 31db8b96cef..00000000000 --- a/newsfragments/3706.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `FromPyObject::extract_bound` method, which can be implemented to avoid using the GIL Ref API in `FromPyObject` implementations. diff --git a/newsfragments/3707.added.md b/newsfragments/3707.added.md deleted file mode 100644 index bc92e2c0f95..00000000000 --- a/newsfragments/3707.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `gil-refs` feature to allow continued use of the deprecated GIL Refs APIs. diff --git a/newsfragments/3712.added.md b/newsfragments/3712.added.md deleted file mode 100644 index d7390f77c14..00000000000 --- a/newsfragments/3712.added.md +++ /dev/null @@ -1 +0,0 @@ -Added methods to `PyAnyMethods` for binary operators (`add`, `sub`, etc.) diff --git a/newsfragments/3730.added.md b/newsfragments/3730.added.md deleted file mode 100644 index 7e287245eb1..00000000000 --- a/newsfragments/3730.added.md +++ /dev/null @@ -1 +0,0 @@ -`chrono-tz` feature allowing conversion between `chrono_tz::Tz` and `zoneinfo.ZoneInfo` \ No newline at end of file diff --git a/newsfragments/3734.added.md b/newsfragments/3734.added.md deleted file mode 100644 index e58c2038e70..00000000000 --- a/newsfragments/3734.added.md +++ /dev/null @@ -1 +0,0 @@ -Add definition for `PyType_GetModuleByDef` to `pyo3_ffi`. diff --git a/newsfragments/3736.added.md b/newsfragments/3736.added.md deleted file mode 100644 index 0d3a4a08c1a..00000000000 --- a/newsfragments/3736.added.md +++ /dev/null @@ -1 +0,0 @@ -Conversion between `std::time::SystemTime` and `datetime.datetime` \ No newline at end of file diff --git a/newsfragments/3742.changed.md b/newsfragments/3742.changed.md deleted file mode 100644 index b8805abafda..00000000000 --- a/newsfragments/3742.changed.md +++ /dev/null @@ -1 +0,0 @@ -Improve performance of `extract::` (and other integer types) by avoiding call to `__index__()` converting the value to an integer for 3.10+. Gives performance improvement of around 30% for successful extraction. diff --git a/newsfragments/3757.fixed.md b/newsfragments/3757.fixed.md deleted file mode 100644 index 103a634af9f..00000000000 --- a/newsfragments/3757.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Match PyPy 7.3.14 in removing PyPy-only symbol `Py_MAX_NDIMS` in favour of `PyBUF_MAX_NDIM`. diff --git a/newsfragments/3776.changed.md b/newsfragments/3776.changed.md deleted file mode 100644 index 71ffd893a18..00000000000 --- a/newsfragments/3776.changed.md +++ /dev/null @@ -1 +0,0 @@ -Relax bound of `FromPyObject` for `Py` to just `T: PyTypeCheck`. diff --git a/newsfragments/3785.added.md b/newsfragments/3785.added.md deleted file mode 100644 index 6af3bb999f8..00000000000 --- a/newsfragments/3785.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `Py::as_any` and `Py::into_any`. diff --git a/newsfragments/3801.added.md b/newsfragments/3801.added.md deleted file mode 100644 index 78f45032ba2..00000000000 --- a/newsfragments/3801.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyStringMethods::encode_utf8`. diff --git a/newsfragments/3802.added.md b/newsfragments/3802.added.md deleted file mode 100644 index 86b98e9df97..00000000000 --- a/newsfragments/3802.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyBackedStr` and `PyBackedBytes`, as alternatives to `&str` and `&bytes` where a Python object owns the data. diff --git a/newsfragments/3818.fixed.md b/newsfragments/3818.fixed.md deleted file mode 100644 index 76fe01a545c..00000000000 --- a/newsfragments/3818.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix segmentation fault using `datetime` types when an invalid `datetime` module is on sys.path. diff --git a/newsfragments/3821.packaging.md b/newsfragments/3821.packaging.md deleted file mode 100644 index 4bd89355086..00000000000 --- a/newsfragments/3821.packaging.md +++ /dev/null @@ -1 +0,0 @@ -Check maximum version of Python at build time and for versions not yet supported require opt-in to the `abi3` stable ABI by the environment variable `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1`. diff --git a/newsfragments/5438.changed.md b/newsfragments/5438.changed.md new file mode 100644 index 00000000000..bcd7885673d --- /dev/null +++ b/newsfragments/5438.changed.md @@ -0,0 +1,2 @@ +Introspection: introduce `TypeHint` and make use of it to encode type hint annotations. +Rename `PyType{Info,Check}::TYPE_INFO` into `PyType{Info,Check}::TYPE_HINT` diff --git a/newsfragments/5531.packaging.md b/newsfragments/5531.packaging.md new file mode 100644 index 00000000000..8bf82fe545b --- /dev/null +++ b/newsfragments/5531.packaging.md @@ -0,0 +1,2 @@ +Bump MSRV to Rust 1.83. +Bump minimum supported `quote` version to 1.0.37. diff --git a/newsfragments/5538.changed.md b/newsfragments/5538.changed.md new file mode 100644 index 00000000000..56493d0862f --- /dev/null +++ b/newsfragments/5538.changed.md @@ -0,0 +1 @@ +silence a clippy warning on rust 1.83 diff --git a/newsfragments/5539.changed.md b/newsfragments/5539.changed.md new file mode 100644 index 00000000000..2ea40282f2d --- /dev/null +++ b/newsfragments/5539.changed.md @@ -0,0 +1 @@ +Expose types::iterator::PySendResult in public API diff --git a/newsfragments/5542.packaging.md b/newsfragments/5542.packaging.md new file mode 100644 index 00000000000..dd381c6f005 --- /dev/null +++ b/newsfragments/5542.packaging.md @@ -0,0 +1 @@ +Bump supported GraalPy version to 25.0. diff --git a/newsfragments/5545.packaging.md b/newsfragments/5545.packaging.md new file mode 100644 index 00000000000..2591f9d8798 --- /dev/null +++ b/newsfragments/5545.packaging.md @@ -0,0 +1 @@ +Drop `memoffset` dependency. diff --git a/noxfile.py b/noxfile.py index 3981e62e100..fc1d2b50c9b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,24 +1,88 @@ -from contextlib import contextmanager +import io import json import os import re +import shutil import subprocess import sys +import sysconfig +import tarfile import tempfile +from contextlib import ExitStack, contextmanager from functools import lru_cache from glob import glob from pathlib import Path -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, +) -import nox import nox.command -nox.options.sessions = ["test", "clippy", "rustfmt", "ruff", "docs"] +try: + import tomllib as toml +except ImportError: + try: + import toml + except ImportError: + toml = None + +try: + import requests +except ImportError: + requests = None +nox.options.sessions = ["test", "clippy", "rustfmt", "ruff", "rumdl", "docs"] PYO3_DIR = Path(__file__).parent -PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11", "3.12") -PYPY_VERSIONS = ("3.7", "3.8", "3.9", "3.10") +PYO3_TARGET = Path(os.environ.get("CARGO_TARGET_DIR", PYO3_DIR / "target")).absolute() +PYO3_GUIDE_SRC = PYO3_DIR / "guide" / "src" +PYO3_GUIDE_TARGET = PYO3_TARGET / "guide" +PYO3_DOCS_TARGET = PYO3_TARGET / "doc" +FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + + +def _get_output(*args: str) -> str: + return subprocess.run(args, capture_output=True, text=True, check=True).stdout + + +def _parse_supported_interpreter_version( + python_impl: str, # Literal["cpython", "pypy"], TODO update after 3.7 dropped +) -> Tuple[str, str]: + output = _get_output("cargo", "metadata", "--format-version=1", "--no-deps") + cargo_packages = json.loads(output)["packages"] + # Check Python interpreter version support in package metadata + package = "pyo3-ffi" + metadata = next(pkg["metadata"] for pkg in cargo_packages if pkg["name"] == package) + version_info = metadata[python_impl] + assert "min-version" in version_info, f"missing min-version for {python_impl}" + assert "max-version" in version_info, f"missing max-version for {python_impl}" + return version_info["min-version"], version_info["max-version"] + + +def _supported_interpreter_versions( + python_impl: str, # Literal["cpython", "pypy"], TODO update after 3.7 dropped +) -> List[str]: + min_version, max_version = _parse_supported_interpreter_version(python_impl) + major = int(min_version.split(".")[0]) + assert major == 3, f"unsupported Python major version {major}" + min_minor = int(min_version.split(".")[1]) + max_minor = int(max_version.split(".")[1]) + versions = [f"{major}.{minor}" for minor in range(min_minor, max_minor + 1)] + # Add free-threaded builds for 3.13+ + if python_impl == "cpython": + versions += [f"{major}.{minor}t" for minor in range(13, max_minor + 1)] + return versions + + +PY_VERSIONS = _supported_interpreter_versions("cpython") +PYPY_VERSIONS = _supported_interpreter_versions("pypy") @nox.session(venv_backend="none") @@ -32,13 +96,48 @@ def test_rust(session: nox.Session): _run_cargo_test(session, package="pyo3-build-config") _run_cargo_test(session, package="pyo3-macros-backend") _run_cargo_test(session, package="pyo3-macros") - _run_cargo_test(session, package="pyo3-ffi") - _run_cargo_test(session) - _run_cargo_test(session, features="abi3") - if "skip-full" not in session.posargs: - _run_cargo_test(session, features="full") - _run_cargo_test(session, features="abi3 full") + extra_flags = [] + # pypy and graalpy don't have Py_Initialize APIs, so we can only + # build the main tests, not run them + if sys.implementation.name in ("pypy", "graalpy"): + extra_flags.append("--no-run") + + _run_cargo_test(session, package="pyo3-ffi", extra_flags=extra_flags) + + extra_flags.append("--no-default-features") + + for feature_set in _get_feature_sets(): + flags = extra_flags.copy() + + if feature_set is None or "full" not in feature_set: + # doctests require at least the macros feature, which is + # activated by the full feature set + # + # using `--all-targets` makes cargo run everything except doctests + flags.append("--all-targets") + + # We need to pass the feature set to the test command + # so that it can be used in the test code + # (e.g. for `#[cfg(feature = "abi3-py37")]`) + if feature_set and "abi3" in feature_set and FREE_THREADED_BUILD: + # free-threaded builds don't support abi3 yet + continue + + _run_cargo_test(session, features=feature_set, extra_flags=flags) + + if ( + feature_set + and "abi3" in feature_set + and "full" in feature_set + and sys.version_info >= (3, 7) + ): + # run abi3-py37 tests to check abi3 forward compatibility + _run_cargo_test( + session, + features=feature_set.replace("abi3", "abi3-py37"), + extra_flags=flags, + ) @nox.session(name="test-py", venv_backend="none") @@ -46,6 +145,8 @@ def test_py(session: nox.Session) -> None: _run(session, "nox", "-f", "pytests/noxfile.py", external=True) for example in glob("examples/*/noxfile.py"): _run(session, "nox", "-f", example, external=True) + for example in glob("pyo3-ffi/examples/*/noxfile.py"): + _run(session, "nox", "-f", example, external=True) @nox.session(venv_backend="none") @@ -53,6 +154,26 @@ def coverage(session: nox.Session) -> None: session.env.update(_get_coverage_env()) _run_cargo(session, "llvm-cov", "clean", "--workspace") test(session) + generate_coverage_report(session) + + +@nox.session(name="set-coverage-env", venv_backend="none") +def set_coverage_env(session: nox.Session) -> None: + """For use in GitHub Actions to set coverage environment variables.""" + with open(os.environ["GITHUB_ENV"], "a") as env_file: + for k, v in _get_coverage_env().items(): + print(f"{k}={v}", file=env_file) + + +@nox.session(name="generate-coverage-report", venv_backend="none") +def generate_coverage_report(session: nox.Session) -> None: + cov_format = "codecov" + output_file = "coverage.json" + + if "lcov" in session.posargs: + cov_format = "lcov" + output_file = "lcov.info" + _run_cargo( session, "llvm-cov", @@ -62,9 +183,9 @@ def coverage(session: nox.Session) -> None: "--package=pyo3-macros", "--package=pyo3-ffi", "report", - "--codecov", + f"--{cov_format}", "--output-path", - "coverage.json", + output_file, ) @@ -81,9 +202,19 @@ def ruff(session: nox.Session): _run(session, "ruff", "check", ".") +@nox.session(name="rumdl") +def rumdl(session: nox.Session): + """Run rumdl to check markdown formatting in the guide. + + Can also run with uv directly, e.g. `uvx rumdl check guide`. + """ + session.install("rumdl") + _run(session, "rumdl", "check", "guide", *session.posargs) + + @nox.session(name="clippy", venv_backend="none") def clippy(session: nox.Session) -> bool: - if not _clippy(session) and _clippy_additional_workspaces(session): + if not (_clippy(session) and _clippy_additional_workspaces(session)): session.error("one or more jobs failed") @@ -95,7 +226,8 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: _run_cargo( session, "clippy", - *feature_set, + "--no-default-features", + *((f"--features={feature_set}",) if feature_set else ()), "--all-targets", "--workspace", "--", @@ -172,7 +304,8 @@ def _check(env: Dict[str, str]) -> None: _run_cargo( session, "check", - *feature_set, + "--no-default-features", + *((f"--features={feature_set}",) if feature_set else ()), "--all-targets", "--workspace", env=env, @@ -193,6 +326,7 @@ def publish(session: nox.Session) -> None: _run_cargo_publish(session, package="pyo3-macros") _run_cargo_publish(session, package="pyo3-ffi") _run_cargo_publish(session, package="pyo3") + _run_cargo_publish(session, package="pyo3-introspection") @nox.session(venv_backend="none") @@ -298,11 +432,13 @@ def test_emscripten(session: nox.Session): f"-C link-arg=-lpython{info.pymajorminor}", "-C link-arg=-lexpat", "-C link-arg=-lmpdec", + "-C link-arg=-lsqlite3", "-C link-arg=-lz", "-C link-arg=-lbz2", "-C link-arg=-sALLOW_MEMORY_GROWTH=1", ] ) + session.env["RUSTDOCFLAGS"] = session.env["RUSTFLAGS"] session.env["CARGO_BUILD_TARGET"] = target session.env["PYO3_CROSS_LIB_DIR"] = pythonlibdir _run(session, "rustup", "target", "add", target, "--toolchain", "stable") @@ -310,25 +446,91 @@ def test_emscripten(session: nox.Session): session, "bash", "-c", - f"source {info.builddir/'emsdk/emsdk_env.sh'} && cargo test", + f"source {info.builddir / 'emsdk/emsdk_env.sh'} && cargo test", + ) + + +@nox.session(name="test-cross-compilation-windows") +def test_cross_compilation_windows(session: nox.Session): + session.install("cargo-xwin") + + env = os.environ.copy() + env["XWIN_ARCH"] = "x86_64" + + # abi3 + _run_cargo( + session, + "build", + "--manifest-path", + "examples/maturin-starter/Cargo.toml", + "--features", + "abi3", + "--target", + "x86_64-pc-windows-gnu", + env=env, + ) + _run_cargo( + session, + "xwin", + "build", + "--cross-compiler", + "clang", + "--manifest-path", + "examples/maturin-starter/Cargo.toml", + "--features", + "abi3", + "--target", + "x86_64-pc-windows-msvc", + env=env, + ) + + # non-abi3 + env["PYO3_CROSS_PYTHON_VERSION"] = "3.13" + _run_cargo( + session, + "build", + "--manifest-path", + "examples/maturin-starter/Cargo.toml", + "--features", + "generate-import-lib", + "--target", + "x86_64-pc-windows-gnu", + env=env, + ) + _run_cargo( + session, + "xwin", + "build", + "--cross-compiler", + "clang", + "--manifest-path", + "examples/maturin-starter/Cargo.toml", + "--features", + "generate-import-lib", + "--target", + "x86_64-pc-windows-msvc", + env=env, ) @nox.session(venv_backend="none") -def docs(session: nox.Session) -> None: +def docs(session: nox.Session, nightly: bool = False, internal: bool = False) -> None: rustdoc_flags = ["-Dwarnings"] toolchain_flags = [] cargo_flags = [] + nightly = nightly or ("nightly" in session.posargs) + internal = internal or ("internal" in session.posargs) + if "open" in session.posargs: cargo_flags.append("--open") - if "nightly" in session.posargs: + if nightly: rustdoc_flags.append("--cfg docsrs") toolchain_flags.append("+nightly") cargo_flags.extend(["-Z", "unstable-options", "-Z", "rustdoc-scrape-examples"]) - if "nightly" in session.posargs and "internal" in session.posargs: + if internal: rustdoc_flags.append("--Z unstable-options") rustdoc_flags.append("--document-hidden-items") rustdoc_flags.extend(("--html-after-content", ".netlify/internal_banner.html")) @@ -339,13 +541,16 @@ def docs(session: nox.Session) -> None: rustdoc_flags.append(session.env.get("RUSTDOCFLAGS", "")) session.env["RUSTDOCFLAGS"] = " ".join(rustdoc_flags) + features = "full" + + shutil.rmtree(PYO3_DOCS_TARGET, ignore_errors=True) _run_cargo( session, *toolchain_flags, "doc", "--lib", "--no-default-features", - "--features=full", + f"--features={features}", "--no-deps", "--workspace", *cargo_flags, @@ -354,7 +559,204 @@ def docs(session: nox.Session) -> None: @nox.session(name="build-guide", venv_backend="none") def build_guide(session: nox.Session): - _run(session, "mdbook", "build", "-d", "../target/guide", "guide", *session.posargs) + shutil.rmtree(PYO3_GUIDE_TARGET, ignore_errors=True) + _run( + session, + "mdbook", + "build", + "-d", + str(PYO3_GUIDE_TARGET), + "guide", + *session.posargs, + external=True, + ) + for license in ("LICENSE-APACHE", "LICENSE-MIT"): + target_file = PYO3_GUIDE_TARGET / license + target_file.unlink(missing_ok=True) + shutil.copy(PYO3_DIR / license, target_file) + + +@nox.session(name="build-netlify-site") +def build_netlify_site(session: nox.Session): + # Remove netlify_build directory if it exists + netlify_build = Path("netlify_build") + if netlify_build.exists(): + shutil.rmtree(netlify_build) + + url = "https://github.com/PyO3/pyo3/archive/gh-pages.tar.gz" + response = requests.get(url, stream=True) + response.raise_for_status() + with tarfile.open(fileobj=io.BytesIO(response.content), mode="r:gz") as tar: + tar.extractall() + shutil.move("pyo3-gh-pages", "netlify_build") + + preview = "--preview" in session.posargs + if preview: + session.posargs.remove("--preview") + + _build_netlify_redirects(preview) + + session.install("towncrier") + # Save a copy of the changelog to restore later + changelog = (PYO3_DIR / "CHANGELOG.md").read_text() + + # Build the changelog + session.run( + "towncrier", "build", "--keep", "--version", "Unreleased", "--date", "TBC" + ) + + # Build the guide + build_guide(session) + PYO3_GUIDE_TARGET.rename("netlify_build/main") + + # Restore the original changelog + (PYO3_DIR / "CHANGELOG.md").write_text(changelog) + session.run("git", "restore", "--staged", "CHANGELOG.md", external=True) + + # Build the main branch docs + docs(session) + PYO3_DOCS_TARGET.rename("netlify_build/main/doc") + + Path("netlify_build/main/doc/index.html").write_text( + "" + ) + + # Build the internal docs + docs(session, nightly=True, internal=True) + PYO3_DOCS_TARGET.rename("netlify_build/internal") + + +def _build_netlify_redirects(preview: bool) -> None: + current_version = os.environ.get("PYO3_VERSION") + + with ExitStack() as stack: + redirects_file = stack.enter_context(open("netlify_build/_redirects", "w")) + headers_file = stack.enter_context(open("netlify_build/_headers", "w")) + for d in glob("netlify_build/v*"): + version = d.removeprefix("netlify_build/v") + full_directory = d + "/" + redirects_file.write( + f"/v{version}/doc/* https://docs.rs/pyo3/{version}/:splat\n" + ) + if version != current_version: + # for old versions, mark the files in the latest version as the canonical URL + for file in glob(f"{d}/**", recursive=True): + file_path = file.removeprefix(full_directory) + # remove index.html and/or .html suffix to match the page URL on the + # final netlfiy site + url_path = file_path + if file_path == "index.html": + url_path = "" + + url_path = url_path.removesuffix(".html") + + # if the file exists in the latest version, add a canonical + # URL as a header + for url in ( + f"/v{version}/{url_path}", + *( + (f"/v{version}/{file_path}",) + if file_path != url_path + else () + ), + ): + headers_file.write(url + "\n") + if os.path.exists( + f"netlify_build/v{current_version}/{file_path}" + ): + headers_file.write( + f' Link: ; rel="canonical"\n' + ) + else: + # this file doesn't exist in the latest guide, don't + # index it + headers_file.write(" X-Robots-Tag: noindex\n") + + # Add latest redirect + if current_version is not None: + redirects_file.write(f"/latest/* /v{current_version}/:splat 302\n") + + # some backwards compatbiility redirects + redirects_file.write( + """\ +/latest/building_and_distribution/* /latest/building-and-distribution/:splat 302 +/latest/building_and_distribution/multiple_python_versions/* /latest/building-and-distribution/multiple-python-versions:splat 302 +/latest/function/error_handling/* /latest/function/error-handling/:splat 302 +/latest/getting_started/* /latest/getting-started/:splat 302 +/latest/python_from_rust/* /latest/python-from-rust/:splat 302 +/latest/python_typing_hints/* /latest/python-typing-hints/:splat 302 +/latest/trait_bounds/* /latest/trait-bounds/:splat 302 +""" + ) + + # Add landing page redirect + if preview: + redirects_file.write("/ /main/ 302\n") + else: + redirects_file.write(f"/ /v{current_version}/ 302\n") + + +@nox.session(name="check-guide") +def check_guide(session: nox.Session): + # reuse other sessions, but with default args + posargs = [*session.posargs] + del session.posargs[:] + build_guide(session) + docs(session) + session.posargs.extend(posargs) + + if toml is None: + session.error("requires Python 3.11 or `toml` to be installed") + pyo3_version = toml.loads((PYO3_DIR / "Cargo.toml").read_text())["package"][ + "version" + ] + + remaps = { + f"file://{PYO3_GUIDE_SRC}/([^/]*/)*?%7B%7B#PYO3_DOCS_URL}}}}": f"file://{PYO3_DOCS_TARGET}", + f"https://pyo3.rs/v%7Bpyo3_version%7D": f"file://{PYO3_GUIDE_TARGET}", + "https://pyo3.rs/main/": f"file://{PYO3_GUIDE_TARGET}/", + "https://pyo3.rs/latest/": f"file://{PYO3_GUIDE_TARGET}/", + "%7B%7B#PYO3_DOCS_VERSION}}": "latest", + # bypass fragments for edge cases + # blob links + "(https://github.com/[^/]+/[^/]+/blob/[^#]+)#[a-zA-Z0-9._-]*": "$1", + # issue comments + "(https://github.com/[^/]+/[^/]+/issues/[0-9]+)#issuecomment-[0-9]*": "$1", + # rust docs + "(https://docs.rs/[^#]+)#[a-zA-Z0-9._-]*": "$1", + } + remap_args = [] + for key, value in remaps.items(): + remap_args.extend(("--remap", f"{key} {value}")) + + # check all links in the guide + _run( + session, + "lychee", + "--include-fragments", + str(PYO3_GUIDE_SRC), + *remap_args, + "--accept=200,429", + *session.posargs, + external=True, + ) + # check external links in the docs + # (intra-doc links are checked by rustdoc) + _run( + session, + "lychee", + str(PYO3_DOCS_TARGET), + *remap_args, + f"--exclude=file://{PYO3_DOCS_TARGET}", + # exclude some old http links from copyright notices, known to fail + "--exclude=http://www.adobe.com/", + "--exclude=http://www.nhncorp.com/", + "--accept=200,429", + # reduce the concurrency to avoid rate-limit from `pyo3.rs` + "--max-concurrency=32", + *session.posargs, + external=True, + ) @nox.session(name="format-guide", venv_backend="none") @@ -429,15 +831,17 @@ def address_sanitizer(session: nox.Session): _IGNORE_CHANGELOG_PR_CATEGORIES = ( "release", "docs", + "ci", ) @nox.session(name="check-changelog") def check_changelog(session: nox.Session): - event_path = os.environ.get("GITHUB_EVENT_PATH") - if event_path is None: + if not _is_github_actions(): session.error("Can only check changelog on github actions") + event_path = os.environ["GITHUB_EVENT_PATH"] + with open(event_path) as event_file: event = json.load(event_file) @@ -475,51 +879,29 @@ def check_changelog(session: nox.Session): print(fragment.name) -@nox.session(name="set-minimal-package-versions", venv_backend="none") -def set_minimal_package_versions(session: nox.Session): +@nox.session(name="set-msrv-package-versions", venv_backend="none") +def set_msrv_package_versions(session: nox.Session): from collections import defaultdict - try: - import tomllib as toml - except ImportError: - import toml - projects = ( - None, - "examples/decorator", - "examples/maturin-starter", - "examples/setuptools-rust-starter", - "examples/word-count", + PYO3_DIR, + *(Path(p).parent for p in glob("examples/*/Cargo.toml")), + *(Path(p).parent for p in glob("pyo3-ffi/examples/*/Cargo.toml")), ) - min_pkg_versions = { - "rust_decimal": "1.26.1", - "csv": "1.1.6", - "indexmap": "1.6.2", - "hashbrown": "0.9.1", - "log": "0.4.17", - "once_cell": "1.17.2", - "rayon": "1.6.1", - "rayon-core": "1.10.2", - "regex": "1.7.3", - "proptest": "1.0.0", - "chrono": "0.4.25", - "byteorder": "1.4.3", - "crossbeam-channel": "0.5.8", - "crossbeam-deque": "0.8.3", - "crossbeam-epoch": "0.9.15", - "crossbeam-utils": "0.8.16", - } + min_pkg_versions = {} # run cargo update first to ensure that everything is at highest # possible version, so that this matches what CI will resolve to. for project in projects: - if project is None: - _run_cargo(session, "update") - else: - _run_cargo(session, "update", f"--manifest-path={project}/Cargo.toml") + _run_cargo( + session, + "+stable", + "update", + f"--manifest-path={project}/Cargo.toml", + env=os.environ | {"CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS": "fallback"}, + ) - for project in projects: - lock_file = Path(project or "") / "Cargo.lock" + lock_file = project / "Cargo.lock" def load_pkg_versions(): cargo_lock = toml.loads(lock_file.read_text()) @@ -545,19 +927,15 @@ def load_pkg_versions(): # and re-read `Cargo.lock` pkg_versions = load_pkg_versions() - # As a smoke test, cargo metadata solves all dependencies, so - # will break if any crates rely on cargo features not - # supported on MSRV - for project in projects: - if project is None: - _run_cargo(session, "metadata", silent=True) - else: - _run_cargo( - session, - "metadata", - f"--manifest-path={project}/Cargo.toml", - silent=True, - ) + # As a smoke test, cargo metadata solves all dependencies, so + # will break if any crates rely on cargo features not + # supported on MSRV + _run_cargo( + session, + "metadata", + f"--manifest-path={project}/Cargo.toml", + silent=True, + ) @nox.session(name="ffi-check") @@ -576,26 +954,187 @@ def test_version_limits(session: nox.Session): config_file.set("CPython", "3.6") _run_cargo(session, "check", env=env, expect_error=True) - assert "3.13" not in PY_VERSIONS - config_file.set("CPython", "3.13") + assert "3.15" not in PY_VERSIONS + config_file.set("CPython", "3.15") _run_cargo(session, "check", env=env, expect_error=True) - # 3.13 CPython should build with forward compatibility + # 3.15 CPython should build if abi3 is explicitly requested + _run_cargo(session, "check", "--features=pyo3/abi3", env=env) + + # 3.15 CPython should build with forward compatibility env["PYO3_USE_ABI3_FORWARD_COMPATIBILITY"] = "1" _run_cargo(session, "check", env=env) - assert "3.6" not in PYPY_VERSIONS - config_file.set("PyPy", "3.6") + assert "3.10" not in PYPY_VERSIONS + config_file.set("PyPy", "3.10") _run_cargo(session, "check", env=env, expect_error=True) - assert "3.11" not in PYPY_VERSIONS - config_file.set("PyPy", "3.11") - _run_cargo(session, "check", env=env, expect_error=True) + # attempt to build with latest version and check that abi3 version + # configured matches the feature + max_minor_version = max(int(v.split(".")[1]) for v in PY_VERSIONS if "t" not in v) + with tempfile.TemporaryFile() as stderr: + env = os.environ.copy() + env["PYO3_PRINT_CONFIG"] = "1" # get diagnostics from the build + env["PYO3_NO_PYTHON"] = "1" # isolate the build from local Python + _run_cargo( + session, + "check", + f"--features=pyo3/abi3-py3{max_minor_version}", + env=env, + stderr=stderr, + expect_error=True, + ) + stderr.seek(0) + stderr = stderr.read().decode() + # NB if this assertion fails with something like + # "An abi3-py3* feature must be specified when compiling without a Python + # interpreter." + # + # then `ABI3_MAX_MINOR` in `pyo3-build-config/src/impl_.rs` is probably outdated. + assert f"version=3.{max_minor_version}" in stderr, ( + f"Expected to see version=3.{max_minor_version}, got: \n\n{stderr}" + ) + + +@nox.session(name="check-feature-powerset", venv_backend="none") +def check_feature_powerset(session: nox.Session): + if toml is None: + session.error("requires Python 3.11 or `toml` to be installed") + + cargo_toml = toml.loads((PYO3_DIR / "Cargo.toml").read_text()) + + # free-threaded builds do not support ABI3 (yet) + EXPECTED_ABI3_FEATURES = { + f"abi3-py3{ver.split('.')[1]}" for ver in PY_VERSIONS if not ver.endswith("t") + } + + EXCLUDED_FROM_FULL = { + "nightly", + "extension-module", + "full", + "default", + "auto-initialize", + "generate-import-lib", + "multiple-pymethods", # Because it's not supported on wasm + } + + features = cargo_toml["features"] + + full_feature = set(features["full"]) + abi3_features = {feature for feature in features if feature.startswith("abi3")} + abi3_version_features = abi3_features - {"abi3"} + + unexpected_abi3_features = abi3_version_features - EXPECTED_ABI3_FEATURES + if unexpected_abi3_features: + session.error( + f"unexpected `abi3` features found in Cargo.toml: {unexpected_abi3_features}" + ) + + missing_abi3_features = EXPECTED_ABI3_FEATURES - abi3_version_features + if missing_abi3_features: + session.error(f"missing `abi3` features in Cargo.toml: {missing_abi3_features}") + + expected_full_feature = features.keys() - EXCLUDED_FROM_FULL - abi3_features + + uncovered_features = expected_full_feature - full_feature + if uncovered_features: + session.error( + f"some features missing from `full` meta feature: {uncovered_features}" + ) + + experimental_features = { + feature for feature in features if feature.startswith("experimental-") + } + full_without_experimental = full_feature - experimental_features + + if len(experimental_features) >= 2: + # justification: we always assume that feature within these groups are + # mutually exclusive to simplify CI + features_to_group = [ + full_without_experimental, + experimental_features, + ] + elif len(experimental_features) == 1: + # no need to make an experimental features group + features_to_group = [full_without_experimental] + else: + session.error("no experimental features exist; please simplify the noxfile") + + features_to_skip = [ + *(EXCLUDED_FROM_FULL), + *abi3_version_features, + ] + + # deny warnings + env = os.environ.copy() + rust_flags = env.get("RUSTFLAGS", "") + env["RUSTFLAGS"] = f"{rust_flags} -Dwarnings" + + subcommand = "hack" + if "minimal-versions" in session.posargs: + subcommand = "minimal-versions" + + comma_join = ",".join + _run_cargo( + session, + subcommand, + "--feature-powerset", + '--optional-deps=""', + f'--skip="{comma_join(features_to_skip)}"', + *(f"--group-features={comma_join(group)}" for group in features_to_group), + "check", + "--all-targets", + env=env, + ) + + +@nox.session(name="update-ui-tests", venv_backend="none") +def update_ui_tests(session: nox.Session): + env = os.environ.copy() + env["TRYBUILD"] = "overwrite" + command = ["test", "--test", "test_compile_error"] + _run_cargo(session, *command, env=env) + _run_cargo(session, *command, "--features=full", env=env) + _run_cargo(session, *command, "--features=abi3,full", env=env) + + +@nox.session(name="test-introspection") +def test_introspection(session: nox.Session): + session.install("maturin") + session.install("ruff") + options = [] + target = os.environ.get("CARGO_BUILD_TARGET") + if target is not None: + options += ("--target", target) + profile = os.environ.get("CARGO_BUILD_PROFILE") + if profile == "release": + options.append("--release") + session.run_always( + "maturin", + "develop", + "-m", + "./pytests/Cargo.toml", + "--features", + "experimental-inspect", + *options, + ) + # We look for the built library + lib_file = None + for file in Path(session.virtualenv.location).rglob("pyo3_pytests.*"): + if file.is_file(): + lib_file = str(file.resolve()) + _run_cargo_test( + session, + package="pyo3-introspection", + env={"PYO3_PYTEST_LIB_PATH": lib_file}, + ) def _build_docs_for_ffi_check(session: nox.Session) -> None: # pyo3-ffi-check needs to scrape docs of pyo3-ffi - _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") + env = os.environ.copy() + env["PYO3_PYTHON"] = sys.executable + _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps", env=env) @lru_cache() @@ -605,7 +1144,7 @@ def _get_rust_info() -> Tuple[str, ...]: return tuple(output.splitlines()) -def _get_rust_version() -> Tuple[int, int, int, List[str]]: +def get_rust_version() -> Tuple[int, int, int, List[str]]: for line in _get_rust_info(): if line.startswith(_RELEASE_LINE_START): version = line[len(_RELEASE_LINE_START) :].strip() @@ -614,6 +1153,13 @@ def _get_rust_version() -> Tuple[int, int, int, List[str]]: return (*map(int, version_number.split(".")), extra) +def is_rust_nightly() -> bool: + for line in _get_rust_info(): + if line.startswith(_RELEASE_LINE_START): + return line.strip().endswith("-nightly") + return False + + def _get_rust_default_target() -> str: for line in _get_rust_info(): if line.startswith(_HOST_LINE_START): @@ -621,31 +1167,20 @@ def _get_rust_default_target() -> str: @lru_cache() -def _get_feature_sets() -> Tuple[Tuple[str, ...], ...]: - """Returns feature sets to use for clippy job""" - rust_version = _get_rust_version() +def _get_feature_sets() -> Tuple[Optional[str], ...]: + """Returns feature sets to use for Rust jobs""" cargo_target = os.getenv("CARGO_BUILD_TARGET", "") - if rust_version[:2] >= (1, 62) and "wasm32-wasi" not in cargo_target: - # multiple-pymethods feature not supported before 1.62 or on WASI - return ( - ("--no-default-features",), - ( - "--no-default-features", - "--features=abi3", - ), - ("--features=full multiple-pymethods",), - ("--features=abi3 full multiple-pymethods",), - ) - else: - return ( - ("--no-default-features",), - ( - "--no-default-features", - "--features=abi3", - ), - ("--features=full",), - ("--features=abi3 full",), - ) + + features = "full" + + if "wasm32-wasip1" not in cargo_target: + # multiple-pymethods not supported on wasm + features += ",multiple-pymethods" + + if is_rust_nightly(): + features += ",nightly" + + return (None, "abi3", features, f"abi3,{features}") _RELEASE_LINE_START = "release: " @@ -673,12 +1208,24 @@ def _get_coverage_env() -> Dict[str, str]: def _run(session: nox.Session, *args: str, **kwargs: Any) -> None: """Wrapper for _run(session, which creates nice groups on GitHub Actions.""" - if "GITHUB_ACTIONS" in os.environ: + is_github_actions = _is_github_actions() + failed = False + if is_github_actions: # Insert ::group:: at the start of nox's command line output print("::group::", end="", flush=True, file=sys.stderr) - session.run(*args, **kwargs) - if "GITHUB_ACTIONS" in os.environ: - print("::endgroup::", file=sys.stderr) + try: + session.run(*args, **kwargs) + except nox.command.CommandFailed: + failed = True + raise + finally: + if is_github_actions: + print("::endgroup::", file=sys.stderr) + # Defer the error message until after the group to make them easier + # to find in the log + if failed: + command = " ".join(args) + print(f"::error::`{command}` failed", file=sys.stderr) def _run_cargo( @@ -696,19 +1243,27 @@ def _run_cargo_test( *, package: Optional[str] = None, features: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + extra_flags: Optional[List[str]] = None, ) -> None: command = ["cargo"] if "careful" in session.posargs: + # do explicit setup so failures in setup can be seen + _run_cargo(session, "careful", "setup") command.append("careful") + command.extend(("test", "--no-fail-fast")) + if "release" in session.posargs: command.append("--release") if package: command.append(f"--package={package}") if features: command.append(f"--features={features}") + if extra_flags: + command.extend(extra_flags) - _run(session, *command, external=True) + _run(session, *command, external=True, env=env or {}) def _run_cargo_publish(session: nox.Session, *, package: str) -> None: @@ -728,10 +1283,6 @@ def _run_cargo_set_package_version( _run(session, *command, external=True) -def _get_output(*args: str) -> str: - return subprocess.run(args, capture_output=True, text=True, check=True).stdout - - def _for_all_version_configs( session: nox.Session, job: Callable[[Dict[str, str]], None] ) -> None: @@ -755,14 +1306,22 @@ class _ConfigFile: def __init__(self, config_file) -> None: self._config_file = config_file - def set(self, implementation: str, version: str) -> None: + def set( + self, implementation: str, version: str, build_flags: Iterable[str] = () + ) -> None: """Set the contents of this config file to the given implementation and version.""" + if version.endswith("t"): + # Free threaded versions pass the support in config file through a flag + version = version[:-1] + build_flags = (*build_flags, "Py_GIL_DISABLED") + self._config_file.seek(0) self._config_file.truncate(0) self._config_file.write( f"""\ implementation={implementation} version={version} +build_flags={",".join(build_flags)} suppress_build_script_link_lines=true """ ) @@ -780,5 +1339,9 @@ def _config_file() -> Iterator[_ConfigFile]: yield _ConfigFile(config) +def _is_github_actions() -> bool: + return "GITHUB_ACTIONS" in os.environ + + _BENCHES = "--manifest-path=pyo3-benches/Cargo.toml" _FFI_CHECK = "--manifest-path=pyo3-ffi-check/Cargo.toml" diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml index e99ef09e19c..6927bfa237d 100644 --- a/pyo3-benches/Cargo.toml +++ b/pyo3-benches/Cargo.toml @@ -9,16 +9,24 @@ publish = false [dependencies] pyo3 = { path = "../", features = ["auto-initialize", "full"] } +[build-dependencies] +pyo3-build-config = { path = "../pyo3-build-config" } + [dev-dependencies] -codspeed-criterion-compat = "2.3" -criterion = "0.5.1" +codspeed-criterion-compat = "4.0" +criterion = "0.7.0" num-bigint = "0.4.3" rust_decimal = { version = "1.0.0", default-features = false } +hashbrown = "0.16" [[bench]] name = "bench_any" harness = false +[[bench]] +name = "bench_attach" +harness = false + [[bench]] name = "bench_call" harness = false @@ -44,7 +52,7 @@ name = "bench_frompyobject" harness = false [[bench]] -name = "bench_gil" +name = "bench_intopyobject" harness = false [[bench]] @@ -52,11 +60,11 @@ name = "bench_list" harness = false [[bench]] -name = "bench_pyclass" +name = "bench_py" harness = false [[bench]] -name = "bench_pyobject" +name = "bench_pyclass" harness = false [[bench]] diff --git a/pyo3-benches/benches/bench_any.rs b/pyo3-benches/benches/bench_any.rs index b77ab9567a6..827f4713990 100644 --- a/pyo3-benches/benches/bench_any.rs +++ b/pyo3-benches/benches/bench_any.rs @@ -52,9 +52,9 @@ fn find_object_type(obj: &Bound<'_, PyAny>) -> ObjectType { ObjectType::Str } else if obj.is_instance_of::() { ObjectType::Tuple - } else if obj.downcast::().is_ok() { + } else if obj.cast::().is_ok() { ObjectType::Sequence - } else if obj.downcast::().is_ok() { + } else if obj.cast::().is_ok() { ObjectType::Mapping } else { ObjectType::Unknown @@ -62,8 +62,8 @@ fn find_object_type(obj: &Bound<'_, PyAny>) -> ObjectType { } fn bench_identify_object_type(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let obj = py.eval_bound("object()", None, None).unwrap(); + Python::attach(|py| { + let obj = py.eval(c"object()", None, None).unwrap(); b.iter(|| find_object_type(&obj)); @@ -72,12 +72,12 @@ fn bench_identify_object_type(b: &mut Bencher<'_>) { } fn bench_collect_generic_iterator(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let collection = py.eval_bound("list(range(1 << 20))", None, None).unwrap(); + Python::attach(|py| { + let collection = py.eval(c"list(range(1 << 20))", None, None).unwrap(); b.iter(|| { collection - .iter() + .try_iter() .unwrap() .collect::>>() .unwrap() diff --git a/pyo3-benches/benches/bench_attach.rs b/pyo3-benches/benches/bench_attach.rs new file mode 100644 index 00000000000..745a7555b89 --- /dev/null +++ b/pyo3-benches/benches/bench_attach.rs @@ -0,0 +1,22 @@ +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; + +use pyo3::prelude::*; + +fn bench_clean_attach(b: &mut Bencher<'_>) { + // Acquiring first GIL will also create a "clean" GILPool, so this measures the Python overhead. + b.iter(|| Python::attach(|_| {})); +} + +fn bench_dirty_attach(b: &mut Bencher<'_>) { + let obj = Python::attach(|py| py.None()); + // Drop the returned clone of the object so that the reference pool has work to do. + b.iter(|| Python::attach(|py| obj.clone_ref(py))); +} + +fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("clean_attach", bench_clean_attach); + c.bench_function("dirty_attach", bench_dirty_attach); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/pyo3-benches/benches/bench_bigint.rs b/pyo3-benches/benches/bench_bigint.rs index 4a50b437d95..6227e95e496 100644 --- a/pyo3-benches/benches/bench_bigint.rs +++ b/pyo3-benches/benches/bench_bigint.rs @@ -1,73 +1,59 @@ -use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; +use num_bigint::BigInt; use pyo3::prelude::*; use pyo3::types::PyDict; -use num_bigint::BigInt; - fn extract_bigint_extract_fail(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + Python::attach(|py| { + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } fn extract_bigint_small(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let int = py.eval_bound("-42", None, None).unwrap(); + Python::attach(|py| { + let int = py.eval(c"-42", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } fn extract_bigint_big_negative(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let int = py.eval_bound("-10**300", None, None).unwrap(); + Python::attach(|py| { + let int = py.eval(c"-10**300", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } fn extract_bigint_big_positive(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let int = py.eval_bound("10**300", None, None).unwrap(); + Python::attach(|py| { + let int = py.eval(c"10**300", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } fn extract_bigint_huge_negative(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let int = py.eval_bound("-10**3000", None, None).unwrap(); + Python::attach(|py| { + let int = py.eval(c"-10**3000", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } fn extract_bigint_huge_positive(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let int = py.eval_bound("10**3000", None, None).unwrap(); + Python::attach(|py| { + let int = py.eval(c"10**3000", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } diff --git a/pyo3-benches/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs index 50772097961..909e5761149 100644 --- a/pyo3-benches/benches/bench_call.rs +++ b/pyo3-benches/benches/bench_call.rs @@ -1,29 +1,90 @@ +use std::hint::black_box; + use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; +use pyo3::ffi::c_str; use pyo3::prelude::*; +use pyo3::types::IntoPyDict; macro_rules! test_module { ($py:ident, $code:literal) => { - PyModule::from_code($py, $code, file!(), "test_module").expect("module creation failed") + PyModule::from_code($py, c_str!($code), c_str!(file!()), c"test_module") + .expect("module creation failed") }; } fn bench_call_0(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { let module = test_module!(py, "def foo(): pass"); - let foo_module = module.getattr("foo").unwrap(); + let foo_module = &module.getattr("foo").unwrap(); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module).call0().unwrap(); + } + }); + }) +} + +fn bench_call_1(b: &mut Bencher<'_>) { + Python::attach(|py| { + let module = test_module!(py, "def foo(a, b, c): pass"); + + let foo_module = &module.getattr("foo").unwrap(); + let args = ( + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), + ); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module).call1(args.clone()).unwrap(); + } + }); + }) +} + +fn bench_call(b: &mut Bencher<'_>) { + Python::attach(|py| { + let module = test_module!(py, "def foo(a, b, c, d, e): pass"); + + let foo_module = &module.getattr("foo").unwrap(); + let args = ( + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), + ); + let kwargs = [("d", 1), ("e", 42)].into_py_dict(py).unwrap(); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module) + .call(args.clone(), Some(&kwargs)) + .unwrap(); + } + }); + }) +} + +fn bench_call_one_arg(b: &mut Bencher<'_>) { + Python::attach(|py| { + let module = test_module!(py, "def foo(a): pass"); + + let foo_module = &module.getattr("foo").unwrap(); + let arg = 1i32.into_pyobject(py).unwrap(); b.iter(|| { for _ in 0..1000 { - foo_module.call0().unwrap(); + black_box(foo_module).call1((arg.clone(),)).unwrap(); } }); }) } fn bench_call_method_0(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { let module = test_module!( py, " @@ -33,11 +94,92 @@ class Foo: " ); - let foo_module = module.getattr("Foo").unwrap().call0().unwrap(); + let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module).call_method0("foo").unwrap(); + } + }); + }) +} + +fn bench_call_method_1(b: &mut Bencher<'_>) { + Python::attach(|py| { + let module = test_module!( + py, + " +class Foo: + def foo(self, a, b, c): + pass +" + ); + + let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); + let args = ( + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), + ); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module) + .call_method1("foo", args.clone()) + .unwrap(); + } + }); + }) +} + +fn bench_call_method(b: &mut Bencher<'_>) { + Python::attach(|py| { + let module = test_module!( + py, + " +class Foo: + def foo(self, a, b, c, d, e): + pass +" + ); + + let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); + let args = ( + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), + ); + let kwargs = [("d", 1), ("e", 42)].into_py_dict(py).unwrap(); + + b.iter(|| { + for _ in 0..1000 { + black_box(foo_module) + .call_method("foo", args.clone(), Some(&kwargs)) + .unwrap(); + } + }); + }) +} + +fn bench_call_method_one_arg(b: &mut Bencher<'_>) { + Python::attach(|py| { + let module = test_module!( + py, + " +class Foo: + def foo(self, a): + pass +" + ); + + let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); + let arg = 1i32.into_pyobject(py).unwrap(); b.iter(|| { for _ in 0..1000 { - foo_module.call_method0("foo").unwrap(); + black_box(foo_module) + .call_method1("foo", (arg.clone(),)) + .unwrap(); } }); }) @@ -45,7 +187,13 @@ class Foo: fn criterion_benchmark(c: &mut Criterion) { c.bench_function("call_0", bench_call_0); + c.bench_function("call_1", bench_call_1); + c.bench_function("call", bench_call); + c.bench_function("call_one_arg", bench_call_one_arg); c.bench_function("call_method_0", bench_call_method_0); + c.bench_function("call_method_1", bench_call_method_1); + c.bench_function("call_method", bench_call_method); + c.bench_function("call_method_one_arg", bench_call_method_one_arg); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/benches/bench_comparisons.rs b/pyo3-benches/benches/bench_comparisons.rs index ffd4c1a452f..581e9738b75 100644 --- a/pyo3-benches/benches/bench_comparisons.rs +++ b/pyo3-benches/benches/bench_comparisons.rs @@ -44,18 +44,18 @@ impl OrderedRichcmp { } fn bench_ordered_dunder_methods(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let obj1 = Py::new(py, OrderedDunderMethods(0)).unwrap().into_ref(py); - let obj2 = Py::new(py, OrderedDunderMethods(1)).unwrap().into_ref(py); + Python::attach(|py| { + let obj1 = &Bound::new(py, OrderedDunderMethods(0)).unwrap().into_any(); + let obj2 = &Bound::new(py, OrderedDunderMethods(1)).unwrap().into_any(); b.iter(|| obj2.gt(obj1).unwrap()); }); } fn bench_ordered_richcmp(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let obj1 = Py::new(py, OrderedRichcmp(0)).unwrap().into_ref(py); - let obj2 = Py::new(py, OrderedRichcmp(1)).unwrap().into_ref(py); + Python::attach(|py| { + let obj1 = &Bound::new(py, OrderedRichcmp(0)).unwrap().into_any(); + let obj2 = &Bound::new(py, OrderedRichcmp(1)).unwrap().into_any(); b.iter(|| obj2.gt(obj1).unwrap()); }); diff --git a/pyo3-benches/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs index 6db6704bf8e..6a288253a82 100644 --- a/pyo3-benches/benches/bench_decimal.rs +++ b/pyo3-benches/benches/bench_decimal.rs @@ -1,14 +1,16 @@ -use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; +use rust_decimal::Decimal; use pyo3::prelude::*; use pyo3::types::PyDict; -use rust_decimal::Decimal; fn decimal_via_extract(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let locals = PyDict::new_bound(py); - py.run_bound( - r#" + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + cr#" import decimal py_dec = decimal.Decimal("0.0") "#, @@ -18,9 +20,7 @@ py_dec = decimal.Decimal("0.0") .unwrap(); let py_dec = locals.get_item("py_dec").unwrap().unwrap(); - b.iter(|| { - let _: Decimal = black_box(&py_dec).extract().unwrap(); - }); + b.iter(|| black_box(&py_dec).extract::().unwrap()); }) } diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs index 072dd9408ce..8ea38b7f67e 100644 --- a/pyo3-benches/benches/bench_dict.rs +++ b/pyo3-benches/benches/bench_dict.rs @@ -1,17 +1,21 @@ +use std::collections::{BTreeMap, HashMap}; +use std::hint::black_box; + use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::types::IntoPyDict; use pyo3::{prelude::*, types::PyMapping}; -use std::collections::{BTreeMap, HashMap}; -use std::hint::black_box; fn iter_dict(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap(); let mut sum = 0; b.iter(|| { - for (k, _v) in dict.iter() { + for (k, _v) in &dict { let i: u64 = k.extract().unwrap(); sum += i; } @@ -20,16 +24,24 @@ fn iter_dict(b: &mut Bencher<'_>) { } fn dict_new(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - b.iter_with_large_drop(|| (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py)); + b.iter_with_large_drop(|| { + (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap() + }); }); } fn dict_get_item(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -45,35 +57,49 @@ fn dict_get_item(b: &mut Bencher<'_>) { } fn extract_hashmap(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); - b.iter(|| HashMap::::extract_bound(&dict)); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap() + .into_any(); + b.iter(|| HashMap::::extract(dict.as_borrowed())); }); } fn extract_btreemap(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); - b.iter(|| BTreeMap::::extract_bound(&dict)); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap() + .into_any(); + b.iter(|| BTreeMap::::extract(dict.as_borrowed())); }); } -#[cfg(feature = "hashbrown")] fn extract_hashbrown_map(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); - b.iter(|| hashbrown::HashMap::::extract_bound(&dict)); + let dict = (0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap() + .into_any(); + b.iter(|| hashbrown::HashMap::::extract(dict.as_borrowed())); }); } fn mapping_from_dict(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let dict = &(0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); - b.iter(|| black_box(dict).downcast::().unwrap()); + let dict = &(0..LEN as u64) + .map(|i| (i, i * 2)) + .into_py_dict(py) + .unwrap(); + b.iter(|| black_box(dict).cast::().unwrap()); }); } @@ -83,11 +109,8 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("dict_get_item", dict_get_item); c.bench_function("extract_hashmap", extract_hashmap); c.bench_function("extract_btreemap", extract_btreemap); - - #[cfg(feature = "hashbrown")] - c.bench_function("extract_hashbrown_map", extract_hashbrown_map); - c.bench_function("mapping_from_dict", mapping_from_dict); + c.bench_function("extract_hashbrown_map", extract_hashbrown_map); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/benches/bench_err.rs b/pyo3-benches/benches/bench_err.rs index 998ed6975b0..543df090d48 100644 --- a/pyo3-benches/benches/bench_err.rs +++ b/pyo3-benches/benches/bench_err.rs @@ -3,7 +3,7 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criter use pyo3::{exceptions::PyValueError, prelude::*}; fn err_new_restore_and_fetch(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { b.iter(|| { PyValueError::new_err("some exception message").restore(py); PyErr::fetch(py) diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs index 1c783c3b706..1364185bcdd 100644 --- a/pyo3-benches/benches/bench_extract.rs +++ b/pyo3-benches/benches/bench_extract.rs @@ -1,4 +1,6 @@ -use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ prelude::*, @@ -6,88 +8,82 @@ use pyo3::{ }; fn extract_str_extract_success(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let s = &PyString::new_bound(py, "Hello, World!"); + Python::attach(|py| { + let s = PyString::new(py, "Hello, World!").into_any(); - bench.iter(|| black_box(s).extract::<&str>().unwrap()); + bench.iter(|| black_box(&s).extract::<&str>().unwrap()); }); } fn extract_str_extract_fail(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + Python::attach(|py| { + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).extract::<&str>() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } -#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] -fn extract_str_downcast_success(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let s = &PyString::new_bound(py, "Hello, World!"); +#[cfg(Py_3_10)] +fn extract_str_cast_success(bench: &mut Bencher<'_>) { + Python::attach(|py| { + let s = PyString::new(py, "Hello, World!").into_any(); bench.iter(|| { - let py_str = black_box(s).downcast::().unwrap(); + let py_str = black_box(&s).cast::().unwrap(); py_str.to_str().unwrap() }); }); } -fn extract_str_downcast_fail(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); +fn extract_str_cast_fail(bench: &mut Bencher<'_>) { + Python::attach(|py| { + let d = PyDict::new(py).into_any(); - bench.iter(|| match black_box(&d).downcast::() { + bench.iter(|| match black_box(&d).cast::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } fn extract_int_extract_success(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let int_obj: PyObject = 123.into_py(py); - let int = int_obj.as_ref(py); + Python::attach(|py| { + let int = 123i32.into_pyobject(py).unwrap(); - bench.iter(|| { - let v = black_box(int).extract::().unwrap(); - black_box(v); - }); + bench.iter(|| black_box(&int).extract::().unwrap()); }); } fn extract_int_extract_fail(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + Python::attach(|py| { + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } -fn extract_int_downcast_success(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let int_obj: PyObject = 123.into_py(py); - let int = int_obj.as_ref(py); +fn extract_int_cast_success(bench: &mut Bencher<'_>) { + Python::attach(|py| { + let int = 123i32.into_pyobject(py).unwrap(); bench.iter(|| { - let py_int = black_box(int).downcast::().unwrap(); - let v = py_int.extract::().unwrap(); - black_box(v); + let py_int = black_box(&int).cast::().unwrap(); + py_int.extract::().unwrap() }); }); } -fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); +fn extract_int_cast_fail(bench: &mut Bencher<'_>) { + Python::attach(|py| { + let d = PyDict::new(py).into_any(); - bench.iter(|| match black_box(&d).downcast::() { + bench.iter(|| match black_box(&d).cast::() { Ok(v) => panic!("should err {}", v), Err(e) => black_box(e), }); @@ -95,48 +91,42 @@ fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { } fn extract_float_extract_success(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let float_obj: PyObject = 23.42.into_py(py); - let float = float_obj.as_ref(py); + Python::attach(|py| { + let float = 23.42f64.into_pyobject(py).unwrap(); - bench.iter(|| { - let v = black_box(float).extract::().unwrap(); - black_box(v); - }); + bench.iter(|| black_box(&float).extract::().unwrap()); }); } fn extract_float_extract_fail(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); + Python::attach(|py| { + let d = PyDict::new(py).into_any(); bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } -fn extract_float_downcast_success(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let float_obj: PyObject = 23.42.into_py(py); - let float = float_obj.as_ref(py); +fn extract_float_cast_success(bench: &mut Bencher<'_>) { + Python::attach(|py| { + let float = 23.42f64.into_pyobject(py).unwrap(); bench.iter(|| { - let py_int = black_box(float).downcast::().unwrap(); - let v = py_int.extract::().unwrap(); - black_box(v); + let py_float = black_box(&float).cast::().unwrap(); + py_float.value() }); }); } -fn extract_float_downcast_fail(bench: &mut Bencher<'_>) { - Python::with_gil(|py| { - let d = PyDict::new_bound(py).into_any(); +fn extract_float_cast_fail(bench: &mut Bencher<'_>) { + Python::attach(|py| { + let d = PyDict::new(py).into_any(); - bench.iter(|| match black_box(&d).downcast::() { + bench.iter(|| match black_box(&d).cast::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } @@ -145,22 +135,22 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("extract_str_extract_success", extract_str_extract_success); c.bench_function("extract_str_extract_fail", extract_str_extract_fail); #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] - c.bench_function("extract_str_downcast_success", extract_str_downcast_success); - c.bench_function("extract_str_downcast_fail", extract_str_downcast_fail); + c.bench_function("extract_str_cast_success", extract_str_cast_success); + c.bench_function("extract_str_cast_fail", extract_str_cast_fail); c.bench_function("extract_int_extract_success", extract_int_extract_success); c.bench_function("extract_int_extract_fail", extract_int_extract_fail); - c.bench_function("extract_int_downcast_success", extract_int_downcast_success); - c.bench_function("extract_int_downcast_fail", extract_int_downcast_fail); + c.bench_function("extract_int_cast_success", extract_int_cast_success); + c.bench_function("extract_int_cast_fail", extract_int_cast_fail); c.bench_function( "extract_float_extract_success", extract_float_extract_success, ); c.bench_function("extract_float_extract_fail", extract_float_extract_fail); c.bench_function( - "extract_float_downcast_success", - extract_float_downcast_success, + "extract_float_cast_success", + extract_float_cast_success, ); - c.bench_function("extract_float_downcast_fail", extract_float_downcast_fail); + c.bench_function("extract_float_cast_fail", extract_float_cast_fail); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs index 8114ee5a802..43e128b62ec 100644 --- a/pyo3-benches/benches/bench_frompyobject.rs +++ b/pyo3-benches/benches/bench_frompyobject.rs @@ -1,11 +1,14 @@ -use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ prelude::*, - types::{PyFloat, PyList, PyString}, + types::{PyByteArray, PyBytes, PyList, PyString}, }; #[derive(FromPyObject)] +#[allow(dead_code)] enum ManyTypes { Int(i32), Bytes(Vec), @@ -13,42 +16,42 @@ enum ManyTypes { } fn enum_from_pyobject(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "hello world"); + Python::attach(|py| { + let any = PyString::new(py, "hello world").into_any(); - b.iter(|| any.extract::().unwrap()); + b.iter(|| black_box(&any).extract::().unwrap()); }) } -fn list_via_downcast(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyList::empty_bound(py); +fn list_via_cast(b: &mut Bencher<'_>) { + Python::attach(|py| { + let any = PyList::empty(py).into_any(); - b.iter(|| black_box(any).downcast::().unwrap()); + b.iter(|| black_box(&any).cast::().unwrap()); }) } fn list_via_extract(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyList::empty_bound(py); + Python::attach(|py| { + let any = PyList::empty(py).into_any(); - b.iter(|| black_box(any).extract::>().unwrap()); + b.iter(|| black_box(&any).extract::>().unwrap()); }) } -fn not_a_list_via_downcast(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); +fn not_a_list_via_cast(b: &mut Bencher<'_>) { + Python::attach(|py| { + let any = PyString::new(py, "foobar").into_any(); - b.iter(|| black_box(any).downcast::().unwrap_err()); + b.iter(|| black_box(&any).cast::().unwrap_err()); }) } fn not_a_list_via_extract(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); + Python::attach(|py| { + let any = PyString::new(py, "foobar").into_any(); - b.iter(|| black_box(any).extract::>().unwrap_err()); + b.iter(|| black_box(&any).extract::>().unwrap_err()); }) } @@ -59,10 +62,10 @@ enum ListOrNotList<'a> { } fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); + Python::attach(|py| { + let any = PyString::new(py, "foobar").into_any(); - b.iter(|| match black_box(any).extract::>() { + b.iter(|| match black_box(&any).extract::>() { Ok(ListOrNotList::List(_list)) => panic!(), Ok(ListOrNotList::NotList(any)) => any, Err(_) => panic!(), @@ -70,21 +73,86 @@ fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { }) } -fn f64_from_pyobject(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let obj = &PyFloat::new_bound(py, 1.234); - b.iter(|| black_box(obj).extract::().unwrap()); +fn bench_vec_from_py_bytes(b: &mut Bencher<'_>, data: &[u8]) { + Python::attach(|py| { + let any = PyBytes::new(py, data).into_any(); + + b.iter(|| black_box(&any).extract::>().unwrap()); + }) +} + +fn vec_bytes_from_py_bytes_small(b: &mut Bencher<'_>) { + bench_vec_from_py_bytes(b, &[]); +} + +fn vec_bytes_from_py_bytes_medium(b: &mut Bencher<'_>) { + let data = (0..u8::MAX).collect::>(); + bench_vec_from_py_bytes(b, &data); +} + +fn vec_bytes_from_py_bytes_large(b: &mut Bencher<'_>) { + let data = vec![10u8; 100_000]; + bench_vec_from_py_bytes(b, &data); +} + +fn bench_vec_from_py_bytearray(b: &mut Bencher<'_>, data: &[u8]) { + Python::attach(|py| { + let any = PyByteArray::new(py, data).into_any(); + + b.iter(|| black_box(&any).extract::>().unwrap()); }) } +fn vec_bytes_from_py_bytearray_small(b: &mut Bencher<'_>) { + bench_vec_from_py_bytearray(b, &[]); +} + +fn vec_bytes_from_py_bytearray_medium(b: &mut Bencher<'_>) { + let data = (0..u8::MAX).collect::>(); + bench_vec_from_py_bytearray(b, &data); +} + +fn vec_bytes_from_py_bytearray_large(b: &mut Bencher<'_>) { + let data = vec![10u8; 100_000]; + bench_vec_from_py_bytearray(b, &data); +} + fn criterion_benchmark(c: &mut Criterion) { c.bench_function("enum_from_pyobject", enum_from_pyobject); - c.bench_function("list_via_downcast", list_via_downcast); + + c.bench_function("list_via_cast", list_via_cast); + c.bench_function("list_via_extract", list_via_extract); - c.bench_function("not_a_list_via_downcast", not_a_list_via_downcast); + + c.bench_function("not_a_list_via_cast", not_a_list_via_cast); c.bench_function("not_a_list_via_extract", not_a_list_via_extract); c.bench_function("not_a_list_via_extract_enum", not_a_list_via_extract_enum); - c.bench_function("f64_from_pyobject", f64_from_pyobject); + + c.bench_function( + "vec_bytes_from_py_bytes_small", + vec_bytes_from_py_bytes_small, + ); + c.bench_function( + "vec_bytes_from_py_bytes_medium", + vec_bytes_from_py_bytes_medium, + ); + c.bench_function( + "vec_bytes_from_py_bytes_large", + vec_bytes_from_py_bytes_large, + ); + + c.bench_function( + "vec_bytes_from_py_bytearray_small", + vec_bytes_from_py_bytearray_small, + ); + c.bench_function( + "vec_bytes_from_py_bytearray_medium", + vec_bytes_from_py_bytearray_medium, + ); + c.bench_function( + "vec_bytes_from_py_bytearray_large", + vec_bytes_from_py_bytearray_large, + ); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs deleted file mode 100644 index 59b9ff9686f..00000000000 --- a/pyo3-benches/benches/bench_gil.rs +++ /dev/null @@ -1,28 +0,0 @@ -use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; - -use pyo3::prelude::*; - -fn bench_clean_acquire_gil(b: &mut Bencher<'_>) { - // Acquiring first GIL will also create a "clean" GILPool, so this measures the Python overhead. - b.iter(|| Python::with_gil(|_| {})); -} - -fn bench_dirty_acquire_gil(b: &mut Bencher<'_>) { - let obj = Python::with_gil(|py| py.None()); - b.iter_batched( - || { - // Clone and drop an object so that the GILPool has work to do. - let _ = obj.clone(); - }, - |_| Python::with_gil(|_| {}), - BatchSize::NumBatches(1), - ); -} - -fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("clean_acquire_gil", bench_clean_acquire_gil); - c.bench_function("dirty_acquire_gil", bench_dirty_acquire_gil); -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs index d8dd1b8fd30..a135fe13278 100644 --- a/pyo3-benches/benches/bench_intern.rs +++ b/pyo3-benches/benches/bench_intern.rs @@ -1,3 +1,5 @@ +use std::hint::black_box; + use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; @@ -5,18 +7,18 @@ use pyo3::prelude::*; use pyo3::intern; fn getattr_direct(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let sys = py.import("sys").unwrap(); + Python::attach(|py| { + let sys = &py.import("sys").unwrap(); - b.iter(|| sys.getattr("version").unwrap()); + b.iter(|| black_box(sys).getattr("version").unwrap()); }); } fn getattr_intern(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let sys = py.import("sys").unwrap(); + Python::attach(|py| { + let sys = &py.import("sys").unwrap(); - b.iter(|| sys.getattr(intern!(py, "version")).unwrap()); + b.iter(|| black_box(sys).getattr(intern!(py, "version")).unwrap()); }); } diff --git a/pyo3-benches/benches/bench_intopyobject.rs b/pyo3-benches/benches/bench_intopyobject.rs new file mode 100644 index 00000000000..1ec88501bb2 --- /dev/null +++ b/pyo3-benches/benches/bench_intopyobject.rs @@ -0,0 +1,76 @@ +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; + +use pyo3::conversion::IntoPyObject; +use pyo3::prelude::*; +use pyo3::types::PyBytes; + +fn bench_bytes_new(b: &mut Bencher<'_>, data: &[u8]) { + Python::attach(|py| { + b.iter_with_large_drop(|| PyBytes::new(py, black_box(data))); + }); +} + +fn bytes_new_small(b: &mut Bencher<'_>) { + bench_bytes_new(b, &[]); +} + +fn bytes_new_medium(b: &mut Bencher<'_>) { + let data = (0..u8::MAX).collect::>(); + bench_bytes_new(b, &data); +} + +fn bytes_new_large(b: &mut Bencher<'_>) { + let data = vec![10u8; 100_000]; + bench_bytes_new(b, &data); +} + +fn bench_bytes_into_pyobject(b: &mut Bencher<'_>, data: &[u8]) { + Python::attach(|py| { + b.iter_with_large_drop(|| black_box(data).into_pyobject(py)); + }); +} + +fn byte_slice_into_pyobject_small(b: &mut Bencher<'_>) { + bench_bytes_into_pyobject(b, &[]); +} + +fn byte_slice_into_pyobject_medium(b: &mut Bencher<'_>) { + let data = (0..u8::MAX).collect::>(); + bench_bytes_into_pyobject(b, &data); +} + +fn byte_slice_into_pyobject_large(b: &mut Bencher<'_>) { + let data = vec![10u8; 100_000]; + bench_bytes_into_pyobject(b, &data); +} + +fn vec_into_pyobject(b: &mut Bencher<'_>) { + Python::attach(|py| { + let bytes = (0..u8::MAX).collect::>(); + b.iter_with_large_drop(|| black_box(&bytes).clone().into_pyobject(py)); + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("bytes_new_small", bytes_new_small); + c.bench_function("bytes_new_medium", bytes_new_medium); + c.bench_function("bytes_new_large", bytes_new_large); + c.bench_function( + "byte_slice_into_pyobject_small", + byte_slice_into_pyobject_small, + ); + c.bench_function( + "byte_slice_into_pyobject_medium", + byte_slice_into_pyobject_medium, + ); + c.bench_function( + "byte_slice_into_pyobject_large", + byte_slice_into_pyobject_large, + ); + c.bench_function("vec_into_pyobject", vec_into_pyobject); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index e0f238c5599..5f83ff442f9 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -6,12 +6,12 @@ use pyo3::prelude::*; use pyo3::types::{PyList, PySequence}; fn iter_list(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let list = PyList::new_bound(py, 0..LEN); + let list = PyList::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { - for x in list.iter() { + for x in &list { let i: u64 = x.extract().unwrap(); sum += i; } @@ -20,16 +20,16 @@ fn iter_list(b: &mut Bencher<'_>) { } fn list_new(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - b.iter_with_large_drop(|| PyList::new_bound(py, 0..LEN)); + b.iter_with_large_drop(|| PyList::new(py, 0..LEN)); }); } fn list_get_item(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let list = PyList::new_bound(py, 0..LEN); + let list = PyList::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -39,11 +39,37 @@ fn list_get_item(b: &mut Bencher<'_>) { }); } -#[cfg(not(Py_LIMITED_API))] +fn list_nth(b: &mut Bencher<'_>) { + Python::attach(|py| { + const LEN: usize = 50; + let list = PyList::new(py, 0..LEN).unwrap(); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + sum += list.iter().nth(i).unwrap().extract::().unwrap(); + } + }); + }); +} + +fn list_nth_back(b: &mut Bencher<'_>) { + Python::attach(|py| { + const LEN: usize = 50; + let list = PyList::new(py, 0..LEN).unwrap(); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + sum += list.iter().nth_back(i).unwrap().extract::().unwrap(); + } + }); + }); +} + +#[cfg(not(any(Py_LIMITED_API, Py_GIL_DISABLED)))] fn list_get_item_unchecked(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let list = PyList::new_bound(py, 0..LEN); + let list = PyList::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -56,18 +82,20 @@ fn list_get_item_unchecked(b: &mut Bencher<'_>) { } fn sequence_from_list(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let list = &PyList::new_bound(py, 0..LEN); - b.iter(|| black_box(list).downcast::().unwrap()); + let list = &PyList::new(py, 0..LEN).unwrap(); + b.iter(|| black_box(list).cast::().unwrap()); }); } fn criterion_benchmark(c: &mut Criterion) { c.bench_function("iter_list", iter_list); c.bench_function("list_new", list_new); + c.bench_function("list_nth", list_nth); + c.bench_function("list_nth_back", list_nth_back); c.bench_function("list_get_item", list_get_item); - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, Py_GIL_DISABLED)))] c.bench_function("list_get_item_unchecked", list_get_item_unchecked); c.bench_function("sequence_from_list", sequence_from_list); } diff --git a/pyo3-benches/benches/bench_py.rs b/pyo3-benches/benches/bench_py.rs new file mode 100644 index 00000000000..fa022a6186d --- /dev/null +++ b/pyo3-benches/benches/bench_py.rs @@ -0,0 +1,116 @@ +use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; + +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc::channel, + Arc, Barrier, +}; +use std::thread::spawn; +use std::time::{Duration, Instant}; + +use pyo3::prelude::*; + +fn drop_many_objects(b: &mut Bencher<'_>) { + Python::attach(|py| { + b.iter(|| { + for _ in 0..1000 { + drop(py.None()); + } + }); + }); +} + +fn drop_many_objects_without_gil(b: &mut Bencher<'_>) { + b.iter_batched( + || Python::attach(|py| (0..1000).map(|_| py.None()).collect::>>()), + |objs| { + drop(objs); + + Python::attach(|_py| ()); + }, + BatchSize::SmallInput, + ); +} + +fn drop_many_objects_multiple_threads(b: &mut Bencher<'_>) { + const THREADS: usize = 5; + + let barrier = Arc::new(Barrier::new(1 + THREADS)); + + let done = Arc::new(AtomicUsize::new(0)); + + let sender = (0..THREADS) + .map(|_| { + let (sender, receiver) = channel(); + + let barrier = barrier.clone(); + + let done = done.clone(); + + spawn(move || { + for objs in receiver { + barrier.wait(); + + drop(objs); + + done.fetch_add(1, Ordering::AcqRel); + } + }); + + sender + }) + .collect::>(); + + b.iter_custom(|iters| { + let mut duration = Duration::ZERO; + + let mut last_done = done.load(Ordering::Acquire); + + for _ in 0..iters { + for sender in &sender { + let objs = Python::attach(|py| { + (0..1000 / THREADS) + .map(|_| py.None()) + .collect::>>() + }); + + sender.send(objs).unwrap(); + } + + barrier.wait(); + + let start = Instant::now(); + + loop { + Python::attach(|_py| ()); + + let done = done.load(Ordering::Acquire); + if done - last_done == THREADS { + last_done = done; + break; + } + } + + Python::attach(|_py| ()); + + duration += start.elapsed(); + } + + duration + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("drop_many_objects", drop_many_objects); + c.bench_function( + "drop_many_objects_without_gil", + drop_many_objects_without_gil, + ); + c.bench_function( + "drop_many_objects_multiple_threads", + drop_many_objects_multiple_threads, + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/pyo3-benches/benches/bench_pyclass.rs b/pyo3-benches/benches/bench_pyclass.rs index b917a4acc08..0967a897649 100644 --- a/pyo3-benches/benches/bench_pyclass.rs +++ b/pyo3-benches/benches/bench_pyclass.rs @@ -27,12 +27,12 @@ impl MyClass { } pub fn first_time_init(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { b.iter(|| { // This is using an undocumented internal PyO3 API to measure pyclass performance; please // don't use this in your own code! let ty = LazyTypeObject::::new(); - ty.get_or_init(py); + ty.get_or_try_init(py).unwrap(); }); }); } diff --git a/pyo3-benches/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs deleted file mode 100644 index af25d61ce6a..00000000000 --- a/pyo3-benches/benches/bench_pyobject.rs +++ /dev/null @@ -1,20 +0,0 @@ -use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; - -use pyo3::prelude::*; - -fn drop_many_objects(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - b.iter(|| { - for _ in 0..1000 { - std::mem::drop(py.None()); - } - }); - }); -} - -fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("drop_many_objects", drop_many_objects); -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs index 49243a63fd4..0c9fedba687 100644 --- a/pyo3-benches/benches/bench_set.rs +++ b/pyo3-benches/benches/bench_set.rs @@ -1,26 +1,29 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; -use pyo3::prelude::*; use pyo3::types::PySet; -use std::collections::{BTreeSet, HashSet}; +use pyo3::{prelude::*, IntoPyObjectExt}; +use std::{ + collections::{BTreeSet, HashSet}, + hint::black_box, +}; fn set_new(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; // Create Python objects up-front, so that the benchmark doesn't need to include // the cost of allocating LEN Python integers - let elements: Vec = (0..LEN).map(|i| i.into_py(py)).collect(); - b.iter_with_large_drop(|| PySet::new_bound(py, &elements).unwrap()); + let elements: Vec> = (0..LEN).map(|i| i.into_py_any(py).unwrap()).collect(); + b.iter_with_large_drop(|| PySet::new(py, &elements).unwrap()); }); } fn iter_set(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); + let set = PySet::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { - for x in set.iter() { + for x in &set { let i: u64 = x.extract().unwrap(); sum += i; } @@ -29,27 +32,26 @@ fn iter_set(b: &mut Bencher<'_>) { } fn extract_hashset(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); - b.iter_with_large_drop(|| HashSet::::extract(set.as_gil_ref())); + let any = PySet::new(py, 0..LEN).unwrap().into_any(); + b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } fn extract_btreeset(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); - b.iter_with_large_drop(|| BTreeSet::::extract(set.as_gil_ref())); + let any = PySet::new(py, 0..LEN).unwrap().into_any(); + b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } -#[cfg(feature = "hashbrown")] fn extract_hashbrown_set(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); - b.iter_with_large_drop(|| hashbrown::HashSet::::extract(set.as_gil_ref())); + let any = PySet::new(py, 0..LEN).unwrap().into_any(); + b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } @@ -58,8 +60,6 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("iter_set", iter_set); c.bench_function("extract_hashset", extract_hashset); c.bench_function("extract_btreeset", extract_btreeset); - - #[cfg(feature = "hashbrown")] c.bench_function("extract_hashbrown_set", extract_hashbrown_set); } diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index 24f32fac364..77c829945d6 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -1,12 +1,14 @@ +use std::hint::black_box; + use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; use pyo3::types::{PyList, PySequence, PyTuple}; fn iter_tuple(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 100_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for x in tuple.iter_borrowed() { @@ -18,16 +20,16 @@ fn iter_tuple(b: &mut Bencher<'_>) { } fn tuple_new(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - b.iter_with_large_drop(|| PyTuple::new_bound(py, 0..LEN)); + b.iter_with_large_drop(|| PyTuple::new(py, 0..LEN).unwrap()); }); } fn tuple_get_item(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -39,9 +41,9 @@ fn tuple_get_item(b: &mut Bencher<'_>) { #[cfg(not(any(Py_LIMITED_API, PyPy)))] fn tuple_get_item_unchecked(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -54,9 +56,9 @@ fn tuple_get_item_unchecked(b: &mut Bencher<'_>) { } fn tuple_get_borrowed_item(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -72,9 +74,9 @@ fn tuple_get_borrowed_item(b: &mut Bencher<'_>) { #[cfg(not(any(Py_LIMITED_API, PyPy)))] fn tuple_get_borrowed_item_unchecked(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -90,32 +92,62 @@ fn tuple_get_borrowed_item_unchecked(b: &mut Bencher<'_>) { } fn sequence_from_tuple(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN).to_object(py); - b.iter(|| tuple.downcast::(py).unwrap()); + let tuple = PyTuple::new(py, 0..LEN).unwrap().into_any(); + b.iter(|| black_box(&tuple).cast::().unwrap()); }); } fn tuple_new_list(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); - b.iter_with_large_drop(|| PyList::new_bound(py, tuple.iter_borrowed())); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); + b.iter_with_large_drop(|| PyList::new(py, tuple.iter_borrowed())); }); } fn tuple_to_list(b: &mut Bencher<'_>) { - Python::with_gil(|py| { + Python::attach(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN); + let tuple = PyTuple::new(py, 0..LEN).unwrap(); b.iter_with_large_drop(|| tuple.to_list()); }); } -fn tuple_into_py(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - b.iter(|| -> PyObject { (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12).into_py(py) }); +fn tuple_into_pyobject(b: &mut Bencher<'_>) { + Python::attach(|py| { + b.iter(|| { + (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) + .into_pyobject(py) + .unwrap() + }); + }); +} + +fn tuple_nth(b: &mut Bencher<'_>) { + Python::attach(|py| { + const LEN: usize = 50; + let list = PyTuple::new(py, 0..LEN).unwrap(); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + sum += list.iter().nth(i).unwrap().extract::().unwrap(); + } + }); + }); +} + +fn tuple_nth_back(b: &mut Bencher<'_>) { + Python::attach(|py| { + const LEN: usize = 50; + let list = PyTuple::new(py, 0..LEN).unwrap(); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + sum += list.iter().nth_back(i).unwrap().extract::().unwrap(); + } + }); }); } @@ -123,6 +155,8 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("iter_tuple", iter_tuple); c.bench_function("tuple_new", tuple_new); c.bench_function("tuple_get_item", tuple_get_item); + c.bench_function("tuple_nth", tuple_nth); + c.bench_function("tuple_nth_back", tuple_nth_back); #[cfg(not(any(Py_LIMITED_API, PyPy)))] c.bench_function("tuple_get_item_unchecked", tuple_get_item_unchecked); c.bench_function("tuple_get_borrowed_item", tuple_get_borrowed_item); @@ -134,7 +168,7 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("sequence_from_tuple", sequence_from_tuple); c.bench_function("tuple_new_list", tuple_new_list); c.bench_function("tuple_to_list", tuple_to_list); - c.bench_function("tuple_into_py", tuple_into_py); + c.bench_function("tuple_into_pyobject", tuple_into_pyobject); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/build.rs b/pyo3-benches/build.rs new file mode 100644 index 00000000000..0475124bb4e --- /dev/null +++ b/pyo3-benches/build.rs @@ -0,0 +1,3 @@ +fn main() { + pyo3_build_config::use_pyo3_cfgs(); +} diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 702e99a4aac..cd7cd52b73f 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.21.0-dev" +version = "0.27.0" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -9,15 +9,15 @@ repository = "https://github.com/pyo3/pyo3" categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" edition = "2021" +rust-version.workspace = true [dependencies] -once_cell = "1" -python3-dll-a = { version = "0.2.6", optional = true } -target-lexicon = "0.12" +python3-dll-a = { version = "0.2.12", optional = true } +target-lexicon = "0.13" [build-dependencies] -python3-dll-a = { version = "0.2.6", optional = true } -target-lexicon = "0.12" +python3-dll-a = { version = "0.2.12", optional = true } +target-lexicon = "0.13" [features] default = [] @@ -29,6 +29,9 @@ resolve-config = [] # This feature is enabled by pyo3 when building an extension module. extension-module = [] +# Automatically generates `python3.dll` import libraries for Windows targets. +generate-import-lib = ["dep:python3-dll-a"] + # These features are enabled by pyo3 when building Stable ABI extension modules. abi3 = [] abi3-py37 = ["abi3-py38"] @@ -36,7 +39,9 @@ abi3-py38 = ["abi3-py39"] abi3-py39 = ["abi3-py310"] abi3-py310 = ["abi3-py311"] abi3-py311 = ["abi3-py312"] -abi3-py312 = ["abi3"] +abi3-py312 = ["abi3-py313"] +abi3-py313 = ["abi3-py314"] +abi3-py314 = ["abi3"] [package.metadata.docs.rs] features = ["resolve-config"] diff --git a/pyo3-build-config/build.rs b/pyo3-build-config/build.rs index 309a78c87da..a6e767edcf0 100644 --- a/pyo3-build-config/build.rs +++ b/pyo3-build-config/build.rs @@ -12,7 +12,7 @@ mod errors; use std::{env, path::Path}; use errors::{Context, Result}; -use impl_::{env_var, make_interpreter_config, InterpreterConfig}; +use impl_::{make_interpreter_config, InterpreterConfig}; fn configure(interpreter_config: Option, name: &str) -> Result { let target = Path::new(&env::var_os("OUT_DIR").unwrap()).join(name); @@ -29,28 +29,12 @@ fn configure(interpreter_config: Option, name: &str) -> Resul } } -/// If PYO3_CONFIG_FILE is set, copy it into the crate. -fn config_file() -> Result> { - if let Some(path) = env_var("PYO3_CONFIG_FILE") { - let path = Path::new(&path); - println!("cargo:rerun-if-changed={}", path.display()); - // Absolute path is necessary because this build script is run with a cwd different to the - // original `cargo build` instruction. - ensure!( - path.is_absolute(), - "PYO3_CONFIG_FILE must be an absolute path" - ); - - let interpreter_config = InterpreterConfig::from_path(path) - .context("failed to parse contents of PYO3_CONFIG_FILE")?; - Ok(Some(interpreter_config)) - } else { - Ok(None) - } -} - fn generate_build_configs() -> Result<()> { - let configured = configure(config_file()?, "pyo3-build-config-file.txt")?; + // If PYO3_CONFIG_FILE is set, copy it into the crate. + let configured = configure( + InterpreterConfig::from_pyo3_config_file_env().transpose()?, + "pyo3-build-config-file.txt", + )?; if configured { // Don't bother trying to find an interpreter on the host system diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index 87c59a998b4..b11d02fd581 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -68,7 +68,7 @@ impl std::fmt::Display for ErrorReport<'_> { writeln!(f, "\ncaused by:")?; let mut index = 0; while let Some(some_source) = source { - writeln!(f, " - {}: {}", index, some_source)?; + writeln!(f, " - {index}: {some_source}")?; source = some_source.source(); index += 1; } diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index fd467b72e0a..cca9393d873 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -2,13 +2,14 @@ //! and its build script. // Optional python3.dll import library generator for Windows -#[cfg(feature = "python3-dll-a")] +#[cfg(feature = "generate-import-lib")] #[path = "import_lib.rs"] mod import_lib; +#[cfg(test)] +use std::cell::RefCell; use std::{ collections::{HashMap, HashSet}, - convert::AsRef, env, ffi::{OsStr, OsString}, fmt::Display, @@ -16,25 +17,35 @@ use std::{ io::{BufRead, BufReader, Read, Write}, path::{Path, PathBuf}, process::{Command, Stdio}, - str, - str::FromStr, + str::{self, FromStr}, }; pub use target_lexicon::Triple; -use target_lexicon::{Environment, OperatingSystem}; +use target_lexicon::{Architecture, Environment, OperatingSystem, Vendor}; use crate::{ bail, ensure, errors::{Context, Error, Result}, - format_warn, warn, + warn, }; /// Minimum Python version PyO3 supports. -const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; +pub(crate) const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; + +/// GraalPy may implement the same CPython version over multiple releases. +const MINIMUM_SUPPORTED_VERSION_GRAALPY: PythonVersion = PythonVersion { + major: 25, + minor: 0, +}; /// Maximum Python version that can be used as minimum required Python version with abi3. -const ABI3_MAX_MINOR: u8 = 12; +pub(crate) const ABI3_MAX_MINOR: u8 = 14; + +#[cfg(test)] +thread_local! { + static READ_ENV_VARS: RefCell> = const { RefCell::new(Vec::new()) }; +} /// Gets an environment variable owned by cargo. /// @@ -47,7 +58,13 @@ pub fn cargo_env_var(var: &str) -> Option { /// the variable changes. pub fn env_var(var: &str) -> Option { if cfg!(feature = "resolve-config") { - println!("cargo:rerun-if-env-changed={}", var); + println!("cargo:rerun-if-env-changed={var}"); + } + #[cfg(test)] + { + READ_ENV_VARS.with(|env_vars| { + env_vars.borrow_mut().push(var.to_owned()); + }); } env::var_os(var) } @@ -150,6 +167,8 @@ pub struct InterpreterConfig { /// /// Serialized to multiple `extra_build_script_line` values. pub extra_build_script_lines: Vec, + /// macOS Python3.framework requires special rpath handling + pub python_framework_prefix: Option, } impl InterpreterConfig { @@ -160,26 +179,28 @@ impl InterpreterConfig { let mut out = vec![]; - // pyo3-build-config was released when Python 3.6 was supported, so minimum flag to emit is - // Py_3_6 (to avoid silently breaking users who depend on this cfg). - for i in 6..=self.version.minor { - out.push(format!("cargo:rustc-cfg=Py_3_{}", i)); + for i in MINIMUM_SUPPORTED_VERSION.minor..=self.version.minor { + out.push(format!("cargo:rustc-cfg=Py_3_{i}")); } - if self.implementation.is_pypy() { - out.push("cargo:rustc-cfg=PyPy".to_owned()); - if self.abi3 { - out.push(format_warn!( - "PyPy does not yet support abi3 so the build artifacts will be version-specific. \ - See https://foss.heptapod.net/pypy/pypy/-/issues/3397 for more information." - )); - } - } else if self.abi3 { + match self.implementation { + PythonImplementation::CPython => {} + PythonImplementation::PyPy => out.push("cargo:rustc-cfg=PyPy".to_owned()), + PythonImplementation::GraalPy => out.push("cargo:rustc-cfg=GraalPy".to_owned()), + } + + // If Py_GIL_DISABLED is set, do not build with limited API support + if self.abi3 && !self.is_free_threaded() { out.push("cargo:rustc-cfg=Py_LIMITED_API".to_owned()); } for flag in &self.build_flags.0 { - out.push(format!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag)); + match flag { + BuildFlag::Py_GIL_DISABLED => { + out.push("cargo:rustc-cfg=Py_GIL_DISABLED".to_owned()) + } + flag => out.push(format!("cargo:rustc-cfg=py_sys_config=\"{flag}\"")), + } } out @@ -198,6 +219,12 @@ import sys from sysconfig import get_config_var, get_platform PYPY = platform.python_implementation() == "PyPy" +GRAALPY = platform.python_implementation() == "GraalVM" + +if GRAALPY: + graalpy_ver = map(int, __graalpython__.get_graalvm_version().split('.')); + print("graalpy_major", next(graalpy_ver)) + print("graalpy_minor", next(graalpy_ver)) # sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue # so that the version mismatch can be reported in a nicer way later. @@ -220,6 +247,7 @@ WINDOWS = platform.system() == "Windows" # macOS framework packages use shared linking FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) +FRAMEWORK_PREFIX = get_config_var("PYTHONFRAMEWORKPREFIX") # unix-style shared library enabled SHARED = bool(get_config_var("Py_ENABLE_SHARED")) @@ -227,7 +255,8 @@ SHARED = bool(get_config_var("Py_ENABLE_SHARED")) print("implementation", platform.python_implementation()) print("version_major", sys.version_info[0]) print("version_minor", sys.version_info[1]) -print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print("shared", PYPY or GRAALPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print("python_framework_prefix", FRAMEWORK_PREFIX) print_if_set("ld_version", get_config_var("LDVERSION")) print_if_set("libdir", get_config_var("LIBDIR")) print_if_set("base_prefix", base_prefix) @@ -235,6 +264,7 @@ print("executable", sys.executable) print("calcsize_pointer", struct.calcsize("P")) print("mingw", get_platform().startswith("mingw")) print("ext_suffix", get_config_var("EXT_SUFFIX")) +print("gil_disabled", get_config_var("Py_GIL_DISABLED")) "#; let output = run_python_script(interpreter.as_ref(), SCRIPT)?; let map: HashMap = parse_script_output(&output); @@ -245,7 +275,25 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) interpreter.as_ref().display() ); + if let Some(value) = map.get("graalpy_major") { + let graalpy_version = PythonVersion { + major: value + .parse() + .context("failed to parse GraalPy major version")?, + minor: map["graalpy_minor"] + .parse() + .context("failed to parse GraalPy minor version")?, + }; + ensure!( + graalpy_version >= MINIMUM_SUPPORTED_VERSION_GRAALPY, + "At least GraalPy version {} needed, got {}", + MINIMUM_SUPPORTED_VERSION_GRAALPY, + graalpy_version + ); + }; + let shared = map["shared"].as_str() == "True"; + let python_framework_prefix = map.get("python_framework_prefix").cloned(); let version = PythonVersion { major: map["version_major"] @@ -260,6 +308,13 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let implementation = map["implementation"].parse()?; + let gil_disabled = match map["gil_disabled"].as_str() { + "1" => true, + "0" => false, + "None" => false, + _ => panic!("Unknown Py_GIL_DISABLED value"), + }; + let lib_name = if cfg!(windows) { default_lib_name_windows( version, @@ -270,18 +325,20 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) // on Windows from sysconfig - e.g. ext_suffix may be // `_d.cp312-win_amd64.pyd` for 3.12 debug build map["ext_suffix"].starts_with("_d."), - ) + gil_disabled, + )? } else { default_lib_name_unix( version, implementation, map.get("ld_version").map(String::as_str), - ) + gil_disabled, + )? }; let lib_dir = if cfg!(windows) { map.get("base_prefix") - .map(|base_prefix| format!("{}\\libs", base_prefix)) + .map(|base_prefix| format!("{base_prefix}\\libs")) } else { map.get("libdir").cloned() }; @@ -307,6 +364,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) build_flags: BuildFlags::from_interpreter(interpreter)?, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -344,12 +402,20 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) Some(s) => !s.is_empty(), _ => false, }; + let python_framework_prefix = sysconfigdata + .get_value("PYTHONFRAMEWORKPREFIX") + .map(str::to_string); let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string); + let gil_disabled = match sysconfigdata.get_value("Py_GIL_DISABLED") { + Some(value) => value == "1", + None => false, + }; let lib_name = Some(default_lib_name_unix( version, implementation, sysconfigdata.get_value("LDVERSION"), - )); + gil_disabled, + )?); let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P") .map(|bytes_width: u32| bytes_width * 8) .ok(); @@ -367,6 +433,36 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, + }) + } + + /// Import an externally-provided config file. + /// + /// The `abi3` features, if set, may apply an `abi3` constraint to the Python version. + #[allow(dead_code)] // only used in build.rs + pub(super) fn from_pyo3_config_file_env() -> Option> { + env_var("PYO3_CONFIG_FILE").map(|path| { + let path = Path::new(&path); + println!("cargo:rerun-if-changed={}", path.display()); + // Absolute path is necessary because this build script is run with a cwd different to the + // original `cargo build` instruction. + ensure!( + path.is_absolute(), + "PYO3_CONFIG_FILE must be an absolute path" + ); + + let mut config = InterpreterConfig::from_path(path) + .context("failed to parse contents of PYO3_CONFIG_FILE")?; + // If the abi3 feature is enabled, the minimum Python version is constrained by the abi3 + // feature. + // + // TODO: abi3 is a property of the build mode, not the interpreter. Should this be + // removed from `InterpreterConfig`? + config.abi3 |= is_abi3(); + config.fixup_for_abi3_version(get_abi3_version())?; + + Ok(config) }) } @@ -411,9 +507,10 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let mut lib_dir = None; let mut executable = None; let mut pointer_width = None; - let mut build_flags = None; + let mut build_flags: Option = None; let mut suppress_build_script_link_lines = None; let mut extra_build_script_lines = vec![]; + let mut python_framework_prefix = None; for (i, line) in lines.enumerate() { let line = line.context("failed to read line from config")?; @@ -442,6 +539,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) "extra_build_script_line" => { extra_build_script_lines.push(value.to_string()); } + "python_framework_prefix" => parse_value!(python_framework_prefix, value), unknown => warn!("unknown config key `{}`", unknown), } } @@ -449,14 +547,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let version = version.ok_or("missing value for version")?; let implementation = implementation.unwrap_or(PythonImplementation::CPython); let abi3 = abi3.unwrap_or(false); - // Fixup lib_name if it's not set - let lib_name = lib_name.or_else(|| { - if let Ok(Ok(target)) = env::var("TARGET").map(|target| target.parse::()) { - default_lib_name_for_target(version, implementation, abi3, &target) - } else { - None - } - }); + let build_flags = build_flags.unwrap_or_default(); Ok(InterpreterConfig { implementation, @@ -467,26 +558,61 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) lib_dir, executable, pointer_width, - build_flags: build_flags.unwrap_or_default(), + build_flags, suppress_build_script_link_lines: suppress_build_script_link_lines.unwrap_or(false), extra_build_script_lines, + python_framework_prefix, }) } - #[cfg(feature = "python3-dll-a")] + /// Helper function to apply a default lib_name if none is set in `PYO3_CONFIG_FILE`. + /// + /// This requires knowledge of the final target, so cannot be done when the config file is + /// inlined into `pyo3-build-config` at build time and instead needs to be done when + /// resolving the build config for linking. + #[cfg(any(test, feature = "resolve-config"))] + pub(crate) fn apply_default_lib_name_to_config_file(&mut self, target: &Triple) { + if self.lib_name.is_none() { + self.lib_name = Some(default_lib_name_for_target( + self.version, + self.implementation, + self.abi3, + self.is_free_threaded(), + target, + )); + } + } + + #[cfg(feature = "generate-import-lib")] #[allow(clippy::unnecessary_wraps)] pub fn generate_import_libs(&mut self) -> Result<()> { // Auto generate python3.dll import libraries for Windows targets. if self.lib_dir.is_none() { let target = target_triple_from_env(); - let py_version = if self.abi3 { None } else { Some(self.version) }; - self.lib_dir = - import_lib::generate_import_lib(&target, self.implementation, py_version)?; + let py_version = if self.implementation == PythonImplementation::CPython + && self.abi3 + && !self.is_free_threaded() + { + None + } else { + Some(self.version) + }; + let abiflags = if self.is_free_threaded() { + Some("t") + } else { + None + }; + self.lib_dir = import_lib::generate_import_lib( + &target, + self.implementation, + py_version, + abiflags, + )?; } Ok(()) } - #[cfg(not(feature = "python3-dll-a"))] + #[cfg(not(feature = "generate-import-lib"))] #[allow(clippy::unnecessary_wraps)] pub fn generate_import_libs(&mut self) -> Result<()> { Ok(()) @@ -546,9 +672,10 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) write_option_line!(executable)?; write_option_line!(pointer_width)?; write_line!(build_flags)?; + write_option_line!(python_framework_prefix)?; write_line!(suppress_build_script_link_lines)?; for line in &self.extra_build_script_lines { - writeln!(writer, "extra_build_script_line={}", line) + writeln!(writer, "extra_build_script_line={line}") .context("failed to write extra_build_script_line")?; } Ok(()) @@ -586,10 +713,18 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) ) } - /// Lowers the configured version to the abi3 version, if set. + pub fn is_free_threaded(&self) -> bool { + self.build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED) + } + + /// Updates configured ABI to build for to the requested abi3 version + /// This is a no-op for platforms where abi3 is not supported fn fixup_for_abi3_version(&mut self, abi3_version: Option) -> Result<()> { - // PyPy doesn't support abi3; don't adjust the version - if self.implementation.is_pypy() { + // PyPy, GraalPy, and the free-threaded build don't support abi3; don't adjust the version + if self.implementation.is_pypy() + || self.implementation.is_graalpy() + || self.is_free_threaded() + { return Ok(()); } @@ -604,6 +739,9 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) ); self.version = version; + } else if is_abi3() && self.version.minor > ABI3_MAX_MINOR { + warn!("Automatically falling back to abi3-py3{ABI3_MAX_MINOR} because current Python is higher than the maximum supported"); + self.version.minor = ABI3_MAX_MINOR; } Ok(()) @@ -617,6 +755,14 @@ pub struct PythonVersion { } impl PythonVersion { + pub const PY313: Self = PythonVersion { + major: 3, + minor: 13, + }; + const PY310: Self = PythonVersion { + major: 3, + minor: 10, + }; const PY37: Self = PythonVersion { major: 3, minor: 7 }; } @@ -648,6 +794,7 @@ impl FromStr for PythonVersion { pub enum PythonImplementation { CPython, PyPy, + GraalPy, } impl PythonImplementation { @@ -656,12 +803,19 @@ impl PythonImplementation { self == PythonImplementation::PyPy } + #[doc(hidden)] + pub fn is_graalpy(self) -> bool { + self == PythonImplementation::GraalPy + } + #[doc(hidden)] pub fn from_soabi(soabi: &str) -> Result { if soabi.starts_with("pypy") { Ok(PythonImplementation::PyPy) } else if soabi.starts_with("cpython") { Ok(PythonImplementation::CPython) + } else if soabi.starts_with("graalpy") { + Ok(PythonImplementation::GraalPy) } else { bail!("unsupported Python interpreter"); } @@ -673,6 +827,7 @@ impl Display for PythonImplementation { match self { PythonImplementation::CPython => write!(f, "CPython"), PythonImplementation::PyPy => write!(f, "PyPy"), + PythonImplementation::GraalPy => write!(f, "GraalVM"), } } } @@ -683,6 +838,7 @@ impl FromStr for PythonImplementation { match s { "CPython" => Ok(PythonImplementation::CPython), "PyPy" => Ok(PythonImplementation::PyPy), + "GraalVM" => Ok(PythonImplementation::GraalPy), _ => bail!("unknown interpreter: {}", s), } } @@ -701,7 +857,7 @@ fn have_python_interpreter() -> bool { /// Must be called from a PyO3 crate build script. fn is_abi3() -> bool { cargo_env_var("CARGO_FEATURE_ABI3").is_some() - || env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1") + || env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").is_some_and(|os_str| os_str == "1") } /// Gets the minimum supported Python version from PyO3 `abi3-py*` features. @@ -709,29 +865,29 @@ fn is_abi3() -> bool { /// Must be called from a PyO3 crate build script. pub fn get_abi3_version() -> Option { let minor_version = (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) - .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()); + .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{i}")).is_some()); minor_version.map(|minor| PythonVersion { major: 3, minor }) } /// Checks if the `extension-module` feature is enabled for the PyO3 crate. /// +/// This can be triggered either by: +/// - The `extension-module` Cargo feature +/// - Setting the `PYO3_BUILD_EXTENSION_MODULE` environment variable +/// /// Must be called from a PyO3 crate build script. pub fn is_extension_module() -> bool { cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some() -} - -/// Checks if we need to link to `libpython` for the current build target. -/// -/// Must be called from a PyO3 crate build script. -pub fn is_linking_libpython() -> bool { - is_linking_libpython_for_target(&target_triple_from_env()) + || env_var("PYO3_BUILD_EXTENSION_MODULE").is_some() } /// Checks if we need to link to `libpython` for the target. /// /// Must be called from a PyO3 crate build script. -fn is_linking_libpython_for_target(target: &Triple) -> bool { +pub fn is_linking_libpython_for_target(target: &Triple) -> bool { target.operating_system == OperatingSystem::Windows + // See https://github.com/PyO3/pyo3/issues/4068#issuecomment-2051159852 + || target.operating_system == OperatingSystem::Aix || target.environment == Environment::Android || target.environment == Environment::Androideabi || !is_extension_module() @@ -742,7 +898,7 @@ fn is_linking_libpython_for_target(target: &Triple) -> bool { /// /// Must be called from a PyO3 crate build script. fn require_libdir_for_target(target: &Triple) -> bool { - let is_generating_libpython = cfg!(feature = "python3-dll-a") + let is_generating_libpython = cfg!(feature = "generate-import-lib") && target.operating_system == OperatingSystem::Windows && is_abi3(); @@ -761,11 +917,14 @@ pub struct CrossCompileConfig { /// The version of the Python library to link against. version: Option, - /// The target Python implementation hint (CPython or PyPy) + /// The target Python implementation hint (CPython, PyPy, GraalPy, ...) implementation: Option, /// The compile target triple (e.g. aarch64-unknown-linux-gnu) target: Triple, + + /// Python ABI flags, used to detect free-threaded Python builds. + abiflags: Option, } impl CrossCompileConfig { @@ -780,7 +939,7 @@ impl CrossCompileConfig { ) -> Result> { if env_vars.any() || Self::is_cross_compiling_from_to(host, target) { let lib_dir = env_vars.lib_dir_path()?; - let version = env_vars.parse_version()?; + let (version, abiflags) = env_vars.parse_version()?; let implementation = env_vars.parse_implementation()?; let target = target.clone(); @@ -789,6 +948,7 @@ impl CrossCompileConfig { version, implementation, target, + abiflags, })) } else { Ok(None) @@ -803,16 +963,20 @@ impl CrossCompileConfig { // e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host // x86_64-pc-windows-gnu on x86_64-pc-windows-msvc host let mut compatible = host.architecture == target.architecture - && host.vendor == target.vendor + && (host.vendor == target.vendor + // Don't treat `-pc-` to `-win7-` as cross-compiling + || (host.vendor == Vendor::Pc && target.vendor.as_str() == "win7")) && host.operating_system == target.operating_system; // Not cross-compiling to compile for 32-bit Python from windows 64-bit compatible |= target.operating_system == OperatingSystem::Windows - && host.operating_system == OperatingSystem::Windows; + && host.operating_system == OperatingSystem::Windows + && matches!(target.architecture, Architecture::X86_32(_)) + && host.architecture == Architecture::X86_64; // Not cross-compiling to compile for x86-64 Python from macOS arm64 and vice versa - compatible |= target.operating_system == OperatingSystem::Darwin - && host.operating_system == OperatingSystem::Darwin; + compatible |= matches!(target.operating_system, OperatingSystem::Darwin(_)) + && matches!(host.operating_system, OperatingSystem::Darwin(_)); !compatible } @@ -821,6 +985,7 @@ impl CrossCompileConfig { /// /// The conversion can not fail because `PYO3_CROSS_LIB_DIR` variable /// is ensured contain a valid UTF-8 string. + #[allow(dead_code)] fn lib_dir_string(&self) -> Option { self.lib_dir .as_ref() @@ -862,22 +1027,25 @@ impl CrossCompileEnvVars { } /// Parses `PYO3_CROSS_PYTHON_VERSION` environment variable value - /// into `PythonVersion`. - fn parse_version(&self) -> Result> { - let version = self - .pyo3_cross_python_version - .as_ref() - .map(|os_string| { + /// into `PythonVersion` and ABI flags. + fn parse_version(&self) -> Result<(Option, Option)> { + match self.pyo3_cross_python_version.as_ref() { + Some(os_string) => { let utf8_str = os_string .to_str() .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid a UTF-8 string")?; - utf8_str + let (utf8_str, abiflags) = if let Some(version) = utf8_str.strip_suffix('t') { + (version, Some("t".to_string())) + } else { + (utf8_str, None) + }; + let version = utf8_str .parse() - .context("failed to parse PYO3_CROSS_PYTHON_VERSION") - }) - .transpose()?; - - Ok(version) + .context("failed to parse PYO3_CROSS_PYTHON_VERSION")?; + Ok((Some(version), abiflags)) + } + None => Ok((None, None)), + } } /// Parses `PYO3_CROSS_PYTHON_IMPLEMENTATION` environment variable value @@ -921,12 +1089,12 @@ impl CrossCompileEnvVars { /// /// This function relies on PyO3 cross-compiling environment variables: /// -/// * `PYO3_CROSS`: If present, forces PyO3 to configure as a cross-compilation. -/// * `PYO3_CROSS_LIB_DIR`: If present, must be set to the directory containing +/// * `PYO3_CROSS`: If present, forces PyO3 to configure as a cross-compilation. +/// * `PYO3_CROSS_LIB_DIR`: If present, must be set to the directory containing /// the target's libpython DSO and the associated `_sysconfigdata*.py` file for /// Unix-like targets, or the Python DLL import libraries for the Windows target. -/// * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python -/// installation. This variable is only needed if PyO3 cannnot determine the version to target +/// * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python +/// installation. This variable is only needed if PyO3 cannot determine the version to target /// from `abi3-py3*` features, or if there are multiple versions of Python present in /// `PYO3_CROSS_LIB_DIR`. /// @@ -944,6 +1112,7 @@ pub fn cross_compiling_from_to( /// /// This must be called from PyO3's build script, because it relies on environment /// variables such as `CARGO_CFG_TARGET_OS` which aren't available at any other time. +#[allow(dead_code)] pub fn cross_compiling_from_cargo_env() -> Result> { let env_vars = CrossCompileEnvVars::from_env(); let host = Triple::host(); @@ -958,6 +1127,7 @@ pub enum BuildFlag { Py_DEBUG, Py_REF_DEBUG, Py_TRACE_REFS, + Py_GIL_DISABLED, COUNT_ALLOCS, Other(String), } @@ -965,8 +1135,8 @@ pub enum BuildFlag { impl Display for BuildFlag { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - BuildFlag::Other(flag) => write!(f, "{}", flag), - _ => write!(f, "{:?}", self), + BuildFlag::Other(flag) => write!(f, "{flag}"), + _ => write!(f, "{self:?}"), } } } @@ -978,17 +1148,19 @@ impl FromStr for BuildFlag { "Py_DEBUG" => Ok(BuildFlag::Py_DEBUG), "Py_REF_DEBUG" => Ok(BuildFlag::Py_REF_DEBUG), "Py_TRACE_REFS" => Ok(BuildFlag::Py_TRACE_REFS), + "Py_GIL_DISABLED" => Ok(BuildFlag::Py_GIL_DISABLED), "COUNT_ALLOCS" => Ok(BuildFlag::COUNT_ALLOCS), other => Ok(BuildFlag::Other(other.to_owned())), } } } -/// A list of python interpreter compile-time preprocessor defines that -/// we will pick up and pass to rustc via `--cfg=py_sys_config={varname}`; +/// A list of python interpreter compile-time preprocessor defines. +/// +/// PyO3 will pick these up and pass to rustc via `--cfg=py_sys_config={varname}`; /// this allows using them conditional cfg attributes in the .rs files, so /// -/// ```rust +/// ```rust,no_run /// #[cfg(py_sys_config="{varname}")] /// # struct Foo; /// ``` @@ -1001,10 +1173,11 @@ impl FromStr for BuildFlag { pub struct BuildFlags(pub HashSet); impl BuildFlags { - const ALL: [BuildFlag; 4] = [ + const ALL: [BuildFlag; 5] = [ BuildFlag::Py_DEBUG, BuildFlag::Py_REF_DEBUG, BuildFlag::Py_TRACE_REFS, + BuildFlag::Py_GIL_DISABLED, BuildFlag::COUNT_ALLOCS, ]; @@ -1016,11 +1189,7 @@ impl BuildFlags { Self( BuildFlags::ALL .iter() - .filter(|flag| { - config_map - .get_value(&flag.to_string()) - .map_or(false, |value| value == "1") - }) + .filter(|flag| config_map.get_value(flag.to_string()) == Some("1")) .cloned() .collect(), ) @@ -1031,10 +1200,15 @@ impl BuildFlags { /// the interpreter and printing variables of interest from /// sysconfig.get_config_vars. fn from_interpreter(interpreter: impl AsRef) -> Result { - // sysconfig is missing all the flags on windows, so we can't actually - // query the interpreter directly for its build flags. + // sysconfig is missing all the flags on windows for Python 3.12 and + // older, so we can't actually query the interpreter directly for its + // build flags on those versions. if cfg!(windows) { - return Ok(Self::new()); + let script = String::from("import sys;print(sys.version_info < (3, 13))"); + let stdout = run_python_script(interpreter.as_ref(), &script)?; + if stdout.trim_end() == "True" { + return Ok(Self::new()); + } } let mut script = String::from("import sysconfig\n"); @@ -1042,7 +1216,7 @@ impl BuildFlags { for k in &BuildFlags::ALL { use std::fmt::Write; - writeln!(&mut script, "print(config.get('{}', '0'))", k).unwrap(); + writeln!(&mut script, "print(config.get('{k}', '0'))").unwrap(); } let stdout = run_python_script(interpreter.as_ref(), &script)?; @@ -1080,7 +1254,7 @@ impl Display for BuildFlags { } else { write!(f, ",")?; } - write!(f, "{}", flag)?; + write!(f, "{flag}")?; } Ok(()) } @@ -1146,6 +1320,10 @@ pub fn parse_sysconfigdata(sysconfigdata_path: impl AsRef) -> Result bool { /// /// Returns `None` if the library directory is not available, and a runtime error /// when no or multiple sysconfigdata files are found. +#[allow(dead_code)] fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result> { - let mut sysconfig_paths = find_all_sysconfigdata(cross); + let mut sysconfig_paths = find_all_sysconfigdata(cross)?; if sysconfig_paths.is_empty() { if let Some(lib_dir) = cross.lib_dir.as_ref() { bail!("Could not find _sysconfigdata*.py in {}", lib_dir.display()); @@ -1231,11 +1410,16 @@ fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result> { /// /// Returns an empty vector when the target Python library directory /// is not set via `PYO3_CROSS_LIB_DIR`. -pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Vec { +pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Result> { let sysconfig_paths = if let Some(lib_dir) = cross.lib_dir.as_ref() { - search_lib_dir(lib_dir, cross) + search_lib_dir(lib_dir, cross).with_context(|| { + format!( + "failed to search the lib dir at 'PYO3_CROSS_LIB_DIR={}'", + lib_dir.display() + ) + })? } else { - return Vec::new(); + return Ok(Vec::new()); }; let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME"); @@ -1253,21 +1437,30 @@ pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Vec { sysconfig_paths.sort(); sysconfig_paths.dedup(); - sysconfig_paths + Ok(sysconfig_paths) } fn is_pypy_lib_dir(path: &str, v: &Option) -> bool { let pypy_version_pat = if let Some(v) = v { - format!("pypy{}", v) + format!("pypy{v}") } else { "pypy3.".into() }; path == "lib_pypy" || path.starts_with(&pypy_version_pat) } +fn is_graalpy_lib_dir(path: &str, v: &Option) -> bool { + let graalpy_version_pat = if let Some(v) = v { + format!("graalpy{v}") + } else { + "graalpy2".into() + }; + path == "lib_graalpython" || path.starts_with(&graalpy_version_pat) +} + fn is_cpython_lib_dir(path: &str, v: &Option) -> bool { let cpython_version_pat = if let Some(v) = v { - format!("python{}", v) + format!("python{v}") } else { "python3.".into() }; @@ -1275,17 +1468,22 @@ fn is_cpython_lib_dir(path: &str, v: &Option) -> bool { } /// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths -fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec { +fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Result> { let mut sysconfig_paths = vec![]; - for f in fs::read_dir(path).expect("Path does not exist") { + for f in fs::read_dir(path.as_ref()).with_context(|| { + format!( + "failed to list the entries in '{}'", + path.as_ref().display() + ) + })? { sysconfig_paths.extend(match &f { // Python 3.7+ sysconfigdata with platform specifics Ok(f) if starts_with(f, "_sysconfigdata_") && ends_with(f, "py") => vec![f.path()], - Ok(f) if f.metadata().map_or(false, |metadata| metadata.is_dir()) => { + Ok(f) if f.metadata().is_ok_and(|metadata| metadata.is_dir()) => { let file_name = f.file_name(); let file_name = file_name.to_string_lossy(); if file_name == "build" || file_name == "lib" { - search_lib_dir(f.path(), cross) + search_lib_dir(f.path(), cross)? } else if file_name.starts_with("lib.") { // check if right target os if !file_name.contains(&cross.target.operating_system.to_string()) { @@ -1295,11 +1493,12 @@ fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec, cross: &CrossCompileConfig) -> Vec, cross: &CrossCompileConfig) -> Vec Result> { if let Some(path) = find_sysconfigdata(cross_compile_config)? { let data = parse_sysconfigdata(path)?; - let config = InterpreterConfig::from_sysconfigdata(&data)?; + let mut config = InterpreterConfig::from_sysconfigdata(&data)?; + if let Some(cross_lib_dir) = cross_compile_config.lib_dir_string() { + config.lib_dir = Some(cross_lib_dir) + } Ok(Some(config)) } else { @@ -1357,7 +1560,7 @@ fn cross_compile_from_sysconfigdata( /// Windows, macOS and Linux. /// /// Must be called from a PyO3 crate build script. -#[allow(unused_mut)] +#[allow(unused_mut, dead_code)] fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result { let version = cross_compile_config .version @@ -1366,7 +1569,7 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result Result Result InterpreterConfig { - // FIXME: PyPy does not support the Stable ABI yet. +fn default_abi3_config(host: &Triple, version: PythonVersion) -> Result { + // FIXME: PyPy & GraalPy do not support the Stable ABI. let implementation = PythonImplementation::CPython; let abi3 = true; @@ -1430,12 +1646,13 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf abi3, false, false, - )) + false, + )?) } else { None }; - InterpreterConfig { + Ok(InterpreterConfig { implementation, version, shared: true, @@ -1447,7 +1664,8 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], - } + python_framework_prefix: None, + }) } /// Detects the cross compilation target interpreter configuration from all @@ -1457,6 +1675,7 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf /// when no target Python interpreter is found. /// /// Must be called from a PyO3 crate build script. +#[allow(dead_code)] fn load_cross_compile_config( cross_compile_config: CrossCompileConfig, ) -> Result { @@ -1475,42 +1694,26 @@ fn load_cross_compile_config( default_cross_compile(&cross_compile_config)? }; - if config.lib_name.is_some() && config.lib_dir.is_none() { - warn!( - "The output binary will link to libpython, \ - but PYO3_CROSS_LIB_DIR environment variable is not set. \ - Ensure that the target Python library directory is \ - in the rustc native library search path." - ); - } - Ok(config) } -// Link against python3.lib for the stable ABI on Windows. -// See https://www.python.org/dev/peps/pep-0384/#linkage -// -// This contains only the limited ABI symbols. +// These contains only the limited ABI symbols. const WINDOWS_ABI3_LIB_NAME: &str = "python3"; +const WINDOWS_ABI3_DEBUG_LIB_NAME: &str = "python3_d"; +/// Generates the default library name for the target platform. +#[allow(dead_code)] fn default_lib_name_for_target( version: PythonVersion, implementation: PythonImplementation, abi3: bool, + gil_disabled: bool, target: &Triple, -) -> Option { +) -> String { if target.operating_system == OperatingSystem::Windows { - Some(default_lib_name_windows( - version, - implementation, - abi3, - false, - false, - )) - } else if is_linking_libpython_for_target(target) { - Some(default_lib_name_unix(version, implementation, None)) + default_lib_name_windows(version, implementation, abi3, false, false, gil_disabled).unwrap() } else { - None + default_lib_name_unix(version, implementation, None, gil_disabled).unwrap() } } @@ -1520,18 +1723,36 @@ fn default_lib_name_windows( abi3: bool, mingw: bool, debug: bool, -) -> String { - if debug { + gil_disabled: bool, +) -> Result { + if debug && version < PythonVersion::PY310 { // CPython bug: linking against python3_d.dll raises error // https://github.com/python/cpython/issues/101614 - format!("python{}{}_d", version.major, version.minor) - } else if abi3 && !implementation.is_pypy() { - WINDOWS_ABI3_LIB_NAME.to_owned() + Ok(format!("python{}{}_d", version.major, version.minor)) + } else if abi3 && !(gil_disabled || implementation.is_pypy() || implementation.is_graalpy()) { + if debug { + Ok(WINDOWS_ABI3_DEBUG_LIB_NAME.to_owned()) + } else { + Ok(WINDOWS_ABI3_LIB_NAME.to_owned()) + } } else if mingw { + ensure!( + !gil_disabled, + "MinGW free-threaded builds are not currently tested or supported" + ); // https://packages.msys2.org/base/mingw-w64-python - format!("python{}.{}", version.major, version.minor) + Ok(format!("python{}.{}", version.major, version.minor)) + } else if gil_disabled { + ensure!(version >= PythonVersion::PY313, "Cannot compile C extensions for the free-threaded build on Python versions earlier than 3.13, found {}.{}", version.major, version.minor); + if debug { + Ok(format!("python{}{}t_d", version.major, version.minor)) + } else { + Ok(format!("python{}{}t", version.major, version.minor)) + } + } else if debug { + Ok(format!("python{}{}_d", version.major, version.minor)) } else { - format!("python{}{}", version.major, version.minor) + Ok(format!("python{}{}", version.major, version.minor)) } } @@ -1539,30 +1760,32 @@ fn default_lib_name_unix( version: PythonVersion, implementation: PythonImplementation, ld_version: Option<&str>, -) -> String { + gil_disabled: bool, +) -> Result { match implementation { PythonImplementation::CPython => match ld_version { - Some(ld_version) => format!("python{}", ld_version), + Some(ld_version) => Ok(format!("python{ld_version}")), None => { if version > PythonVersion::PY37 { // PEP 3149 ABI version tags are finally gone - format!("python{}.{}", version.major, version.minor) + if gil_disabled { + ensure!(version >= PythonVersion::PY313, "Cannot compile C extensions for the free-threaded build on Python versions earlier than 3.13, found {}.{}", version.major, version.minor); + Ok(format!("python{}.{}t", version.major, version.minor)) + } else { + Ok(format!("python{}.{}", version.major, version.minor)) + } } else { // Work around https://bugs.python.org/issue36707 - format!("python{}.{}m", version.major, version.minor) + Ok(format!("python{}.{}m", version.major, version.minor)) } } }, - PythonImplementation::PyPy => { - if version >= (PythonVersion { major: 3, minor: 9 }) { - match ld_version { - Some(ld_version) => format!("pypy{}-c", ld_version), - None => format!("pypy{}.{}-c", version.major, version.minor), - } - } else { - format!("pypy{}-c", version.major) - } - } + PythonImplementation::PyPy => match ld_version { + Some(ld_version) => Ok(format!("pypy{ld_version}-c")), + None => Ok(format!("pypy{}.{}-c", version.major, version.minor)), + }, + + PythonImplementation::GraalPy => Ok("python-native".to_string()), } } @@ -1663,7 +1886,9 @@ pub fn find_interpreter() -> Result { .find(|bin| { if let Ok(out) = Command::new(bin).arg("--version").output() { // begin with `Python 3.X.X :: additional info` - out.stdout.starts_with(b"Python 3") || out.stderr.starts_with(b"Python 3") + out.stdout.starts_with(b"Python 3") + || out.stderr.starts_with(b"Python 3") + || out.stdout.starts_with(b"GraalPy 3") } else { false } @@ -1689,6 +1914,7 @@ fn get_host_interpreter(abi3_version: Option) -> Result Result> { let interpreter_config = if let Some(cross_config) = cross_compiling_from_cargo_env()? { let mut interpreter_config = load_cross_compile_config(cross_config)?; @@ -1730,12 +1956,19 @@ pub fn make_interpreter_config() -> Result { ); }; - let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap()); + let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap())?; // Auto generate python3.dll import libraries for Windows targets. - #[cfg(feature = "python3-dll-a")] + #[cfg(feature = "generate-import-lib")] { - let py_version = if interpreter_config.abi3 { + let gil_disabled = interpreter_config + .build_flags + .0 + .contains(&BuildFlag::Py_GIL_DISABLED); + let py_version = if interpreter_config.implementation == PythonImplementation::CPython + && interpreter_config.abi3 + && !gil_disabled + { None } else { Some(interpreter_config.version) @@ -1744,6 +1977,7 @@ pub fn make_interpreter_config() -> Result { &host, interpreter_config.implementation, py_version, + None, )?; } @@ -1764,7 +1998,7 @@ fn escape(bytes: &[u8]) -> String { } fn unescape(escaped: &str) -> Vec { - assert!(escaped.len() % 2 == 0, "invalid hex encoding"); + assert_eq!(escaped.len() % 2, 0, "invalid hex encoding"); let mut bytes = Vec::with_capacity(escaped.len() / 2); @@ -1777,7 +2011,7 @@ fn unescape(escaped: &str) -> Vec { } } - bytes.push(unhex(chunk[0]) << 4 | unhex(chunk[1])); + bytes.push((unhex(chunk[0]) << 4) | unhex(chunk[1])); } bytes @@ -1785,7 +2019,6 @@ fn unescape(escaped: &str) -> Vec { #[cfg(test)] mod tests { - use std::iter::FromIterator; use target_lexicon::triple; use super::*; @@ -1804,6 +2037,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1832,6 +2066,7 @@ mod tests { }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1853,6 +2088,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1879,6 +2115,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -1901,6 +2138,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2003,6 +2241,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2032,6 +2271,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); @@ -2058,6 +2298,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2068,7 +2309,7 @@ mod tests { let min_version = "3.7".parse().unwrap(); assert_eq!( - default_abi3_config(&host, min_version), + default_abi3_config(&host, min_version).unwrap(), InterpreterConfig { implementation: PythonImplementation::CPython, version: PythonVersion { major: 3, minor: 7 }, @@ -2081,6 +2322,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2091,7 +2333,7 @@ mod tests { let min_version = "3.9".parse().unwrap(); assert_eq!( - default_abi3_config(&host, min_version), + default_abi3_config(&host, min_version).unwrap(), InterpreterConfig { implementation: PythonImplementation::CPython, version: PythonVersion { major: 3, minor: 9 }, @@ -2104,6 +2346,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2138,6 +2381,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2172,6 +2416,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2206,6 +2451,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2216,7 +2462,7 @@ mod tests { pyo3_cross: None, pyo3_cross_lib_dir: None, pyo3_cross_python_implementation: Some("PyPy".into()), - pyo3_cross_python_version: Some("3.10".into()), + pyo3_cross_python_version: Some("3.11".into()), }; let triple = triple!("x86_64-unknown-linux-gnu"); @@ -2231,17 +2477,18 @@ mod tests { implementation: PythonImplementation::PyPy, version: PythonVersion { major: 3, - minor: 10 + minor: 11 }, shared: true, abi3: false, - lib_name: Some("pypy3.10-c".into()), + lib_name: Some("pypy3.11-c".into()), lib_dir: None, executable: None, pointer_width: None, build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2251,75 +2498,184 @@ mod tests { use PythonImplementation::*; assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, false, false, false, - ), - "python37", + false, + ) + .unwrap(), + "python39", ); + assert!(super::default_lib_name_windows( + PythonVersion { major: 3, minor: 9 }, + CPython, + false, + false, + false, + true, + ) + .is_err()); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, true, false, false, - ), + false, + ) + .unwrap(), "python3", ); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, false, true, false, - ), - "python3.7", + false, + ) + .unwrap(), + "python3.9", ); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, true, true, false, - ), + false, + ) + .unwrap(), "python3", ); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, PyPy, true, false, false, - ), - "python37", + false, + ) + .unwrap(), + "python39", ); assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, false, false, true, - ), - "python37_d", + false, + ) + .unwrap(), + "python39_d", ); - // abi3 debug builds on windows use version-specific lib + // abi3 debug builds on windows use version-specific lib on 3.9 and older // to workaround https://github.com/python/cpython/issues/101614 assert_eq!( super::default_lib_name_windows( - PythonVersion { major: 3, minor: 7 }, + PythonVersion { major: 3, minor: 9 }, CPython, true, false, true, - ), - "python37_d", + false, + ) + .unwrap(), + "python39_d", + ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 10 + }, + CPython, + true, + false, + true, + false, + ) + .unwrap(), + "python3_d", + ); + // Python versions older than 3.13 don't support gil_disabled + assert!(super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + false, + false, + false, + true, + ) + .is_err()); + // mingw and free-threading are incompatible (until someone adds support) + assert!(super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + false, + true, + false, + true, + ) + .is_err()); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + false, + false, + false, + true, + ) + .unwrap(), + "python313t", + ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + true, // abi3 true should not affect the free-threaded lib name + false, + false, + true, + ) + .unwrap(), + "python313t", + ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + false, + false, + true, + true, + ) + .unwrap(), + "python313t_d", ); } @@ -2328,16 +2684,34 @@ mod tests { use PythonImplementation::*; // Defaults to python3.7m for CPython 3.7 assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 7 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 7 }, + CPython, + None, + false + ) + .unwrap(), "python3.7m", ); // Defaults to pythonX.Y for CPython 3.8+ assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 8 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 8 }, + CPython, + None, + false + ) + .unwrap(), "python3.8", ); assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + CPython, + None, + false + ) + .unwrap(), "python3.9", ); // Can use ldversion to override for CPython @@ -2345,22 +2719,64 @@ mod tests { super::default_lib_name_unix( PythonVersion { major: 3, minor: 9 }, CPython, - Some("3.7md") - ), + Some("3.7md"), + false + ) + .unwrap(), "python3.7md", ); - // PyPy 3.7 ignores ldversion + // PyPy 3.11 includes ldversion assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 7 }, PyPy, Some("3.7md")), - "pypy3-c", + super::default_lib_name_unix( + PythonVersion { + major: 3, + minor: 11 + }, + PyPy, + None, + false + ) + .unwrap(), + "pypy3.11-c", + ); + + assert_eq!( + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + PyPy, + Some("3.11d"), + false + ) + .unwrap(), + "pypy3.11d-c", ); - // PyPy 3.9 includes ldversion + // free-threading adds a t suffix assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, Some("3.9d")), - "pypy3.9d-c", + super::default_lib_name_unix( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + None, + true + ) + .unwrap(), + "python3.13t", ); + // 3.12 and older are incompatible with gil_disabled + assert!(super::default_lib_name_unix( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + None, + true, + ) + .is_err()); } #[test] @@ -2374,7 +2790,7 @@ mod tests { assert_eq!( env_vars.parse_version().unwrap(), - Some(PythonVersion { major: 3, minor: 9 }) + (Some(PythonVersion { major: 3, minor: 9 }), None), ); let env_vars = CrossCompileEnvVars { @@ -2384,7 +2800,25 @@ mod tests { pyo3_cross_python_implementation: None, }; - assert_eq!(env_vars.parse_version().unwrap(), None); + assert_eq!(env_vars.parse_version().unwrap(), (None, None)); + + let env_vars = CrossCompileEnvVars { + pyo3_cross: None, + pyo3_cross_lib_dir: None, + pyo3_cross_python_version: Some("3.13t".into()), + pyo3_cross_python_implementation: None, + }; + + assert_eq!( + env_vars.parse_version().unwrap(), + ( + Some(PythonVersion { + major: 3, + minor: 13 + }), + Some("t".into()) + ), + ); let env_vars = CrossCompileEnvVars { pyo3_cross: None, @@ -2410,6 +2844,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; config @@ -2432,6 +2867,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert!(config @@ -2467,6 +2903,11 @@ mod tests { version: Some(interpreter_config.version), implementation: Some(interpreter_config.implementation), target: triple!("x86_64-unknown-linux-gnu"), + abiflags: if interpreter_config.is_free_threaded() { + Some("t".into()) + } else { + None + }, }; let sysconfigdata_path = match find_sysconfigdata(&cross) { @@ -2491,6 +2932,7 @@ mod tests { version: interpreter_config.version, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2564,6 +3006,23 @@ mod tests { ) .unwrap() .is_none()); + + assert!(cross_compiling_from_to( + &triple!("x86_64-pc-windows-msvc"), + &triple!("x86_64-win7-windows-msvc"), + ) + .unwrap() + .is_none()); + } + + #[test] + fn test_is_cross_compiling_from_to() { + assert!(cross_compiling_from_to( + &triple!("x86_64-pc-windows-msvc"), + &triple!("aarch64-pc-windows-msvc") + ) + .unwrap() + .is_some()); } #[test] @@ -2595,7 +3054,10 @@ mod tests { fn test_build_script_outputs_base() { let interpreter_config = InterpreterConfig { implementation: PythonImplementation::CPython, - version: PythonVersion { major: 3, minor: 8 }, + version: PythonVersion { + major: 3, + minor: 11, + }, shared: true, abi3: false, lib_name: Some("python3".into()), @@ -2605,13 +3067,16 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), + "cargo:rustc-cfg=Py_3_10".to_owned(), + "cargo:rustc-cfg=Py_3_11".to_owned(), ] ); @@ -2622,9 +3087,11 @@ mod tests { assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), + "cargo:rustc-cfg=Py_3_10".to_owned(), + "cargo:rustc-cfg=Py_3_11".to_owned(), "cargo:rustc-cfg=PyPy".to_owned(), ] ); @@ -2634,7 +3101,7 @@ mod tests { fn test_build_script_outputs_abi3() { let interpreter_config = InterpreterConfig { implementation: PythonImplementation::CPython, - version: PythonVersion { major: 3, minor: 7 }, + version: PythonVersion { major: 3, minor: 9 }, shared: true, abi3: true, lib_name: Some("python3".into()), @@ -2644,13 +3111,15 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), + "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), "cargo:rustc-cfg=Py_LIMITED_API".to_owned(), ] ); @@ -2662,13 +3131,48 @@ mod tests { assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), + "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), "cargo:rustc-cfg=PyPy".to_owned(), - "cargo:warning=PyPy does not yet support abi3 so the build artifacts \ - will be version-specific. See https://foss.heptapod.net/pypy/pypy/-/issues/3397 \ - for more information." - .to_owned(), + "cargo:rustc-cfg=Py_LIMITED_API".to_owned(), + ] + ); + } + + #[test] + fn test_build_script_outputs_gil_disabled() { + let mut build_flags = BuildFlags::default(); + build_flags.0.insert(BuildFlag::Py_GIL_DISABLED); + let interpreter_config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { + major: 3, + minor: 13, + }, + shared: true, + abi3: false, + lib_name: Some("python3".into()), + lib_dir: None, + executable: None, + pointer_width: None, + build_flags, + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: None, + }; + + assert_eq!( + interpreter_config.build_script_outputs(), + [ + "cargo:rustc-cfg=Py_3_7".to_owned(), + "cargo:rustc-cfg=Py_3_8".to_owned(), + "cargo:rustc-cfg=Py_3_9".to_owned(), + "cargo:rustc-cfg=Py_3_10".to_owned(), + "cargo:rustc-cfg=Py_3_11".to_owned(), + "cargo:rustc-cfg=Py_3_12".to_owned(), + "cargo:rustc-cfg=Py_3_13".to_owned(), + "cargo:rustc-cfg=Py_GIL_DISABLED".to_owned(), ] ); } @@ -2689,15 +3193,123 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), [ - "cargo:rustc-cfg=Py_3_6".to_owned(), "cargo:rustc-cfg=Py_3_7".to_owned(), "cargo:rustc-cfg=py_sys_config=\"Py_DEBUG\"".to_owned(), ] ); } + + #[test] + fn test_find_sysconfigdata_in_invalid_lib_dir() { + let e = find_all_sysconfigdata(&CrossCompileConfig { + lib_dir: Some(PathBuf::from("/abc/123/not/a/real/path")), + version: None, + implementation: None, + target: triple!("x86_64-unknown-linux-gnu"), + abiflags: None, + }) + .unwrap_err(); + + // actual error message is platform-dependent, so just check the context we add + assert!(e.report().to_string().starts_with( + "failed to search the lib dir at 'PYO3_CROSS_LIB_DIR=/abc/123/not/a/real/path'\n\ + caused by:\n \ + - 0: failed to list the entries in '/abc/123/not/a/real/path'\n \ + - 1: \ + " + )); + } + + #[test] + fn test_from_pyo3_config_file_env_rebuild() { + READ_ENV_VARS.with(|vars| vars.borrow_mut().clear()); + let _ = InterpreterConfig::from_pyo3_config_file_env(); + // it's possible that other env vars were also read, hence just checking for contains + READ_ENV_VARS.with(|vars| assert!(vars.borrow().contains(&"PYO3_CONFIG_FILE".to_string()))); + } + + #[test] + fn test_apply_default_lib_name_to_config_file() { + let mut config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { major: 3, minor: 9 }, + shared: true, + abi3: false, + lib_name: None, + lib_dir: None, + executable: None, + pointer_width: None, + build_flags: BuildFlags::default(), + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: None, + }; + + let unix = Triple::from_str("x86_64-unknown-linux-gnu").unwrap(); + let win_x64 = Triple::from_str("x86_64-pc-windows-msvc").unwrap(); + let win_arm64 = Triple::from_str("aarch64-pc-windows-msvc").unwrap(); + + config.apply_default_lib_name_to_config_file(&unix); + assert_eq!(config.lib_name, Some("python3.9".into())); + + config.lib_name = None; + config.apply_default_lib_name_to_config_file(&win_x64); + assert_eq!(config.lib_name, Some("python39".into())); + + config.lib_name = None; + config.apply_default_lib_name_to_config_file(&win_arm64); + assert_eq!(config.lib_name, Some("python39".into())); + + // PyPy + config.implementation = PythonImplementation::PyPy; + config.version = PythonVersion { + major: 3, + minor: 11, + }; + config.lib_name = None; + config.apply_default_lib_name_to_config_file(&unix); + assert_eq!(config.lib_name, Some("pypy3.11-c".into())); + + config.implementation = PythonImplementation::CPython; + + // Free-threaded + config.build_flags.0.insert(BuildFlag::Py_GIL_DISABLED); + config.version = PythonVersion { + major: 3, + minor: 13, + }; + config.lib_name = None; + config.apply_default_lib_name_to_config_file(&unix); + assert_eq!(config.lib_name, Some("python3.13t".into())); + + config.lib_name = None; + config.apply_default_lib_name_to_config_file(&win_x64); + assert_eq!(config.lib_name, Some("python313t".into())); + + config.lib_name = None; + config.apply_default_lib_name_to_config_file(&win_arm64); + assert_eq!(config.lib_name, Some("python313t".into())); + + config.build_flags.0.remove(&BuildFlag::Py_GIL_DISABLED); + + // abi3 + config.abi3 = true; + config.lib_name = None; + config.apply_default_lib_name_to_config_file(&unix); + assert_eq!(config.lib_name, Some("python3.13".into())); + + config.lib_name = None; + config.apply_default_lib_name_to_config_file(&win_x64); + assert_eq!(config.lib_name, Some("python3".into())); + + config.lib_name = None; + config.apply_default_lib_name_to_config_file(&win_arm64); + assert_eq!(config.lib_name, Some("python3".into())); + } } diff --git a/pyo3-build-config/src/import_lib.rs b/pyo3-build-config/src/import_lib.rs index dc21638c3db..ee934441f77 100644 --- a/pyo3-build-config/src/import_lib.rs +++ b/pyo3-build-config/src/import_lib.rs @@ -7,7 +7,7 @@ use python3_dll_a::ImportLibraryGenerator; use target_lexicon::{Architecture, OperatingSystem, Triple}; use super::{PythonImplementation, PythonVersion}; -use crate::errors::{Context, Result}; +use crate::errors::{Context, Error, Result}; /// Generates the `python3.dll` or `pythonXY.dll` import library for Windows targets. /// @@ -19,6 +19,7 @@ pub(super) fn generate_import_lib( target: &Triple, py_impl: PythonImplementation, py_version: Option, + abiflags: Option<&str>, ) -> Result> { if target.operating_system != OperatingSystem::Windows { return Ok(None); @@ -42,11 +43,15 @@ pub(super) fn generate_import_lib( let implementation = match py_impl { PythonImplementation::CPython => python3_dll_a::PythonImplementation::CPython, PythonImplementation::PyPy => python3_dll_a::PythonImplementation::PyPy, + PythonImplementation::GraalPy => { + return Err(Error::from("No support for GraalPy on Windows")) + } }; ImportLibraryGenerator::new(&arch, &env) .version(py_version.map(|v| (v.major, v.minor))) .implementation(implementation) + .abiflags(abiflags) .generate(&out_lib_dir) .context("failed to generate python3.dll import library")?; diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 576dd37024b..267d3eb6b8c 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -16,15 +16,13 @@ use std::{ path::{Path, PathBuf}, }; -use std::{env, process::Command, str::FromStr}; - -#[cfg(feature = "resolve-config")] -use once_cell::sync::OnceCell; +use std::{env, process::Command, str::FromStr, sync::OnceLock}; pub use impl_::{ cross_compiling_from_to, find_all_sysconfigdata, parse_sysconfigdata, BuildFlag, BuildFlags, CrossCompileConfig, InterpreterConfig, PythonImplementation, PythonVersion, Triple, }; + use target_lexicon::OperatingSystem; /// Adds all the [`#[cfg]` flags](index.html) to the current compilation. @@ -38,12 +36,16 @@ use target_lexicon::OperatingSystem; /// | `#[cfg(Py_3_7)]`, `#[cfg(Py_3_8)]`, `#[cfg(Py_3_9)]`, `#[cfg(Py_3_10)]` | These attributes mark code only for a given Python version and up. For example, `#[cfg(Py_3_7)]` marks code which can run on Python 3.7 **and newer**. | /// | `#[cfg(Py_LIMITED_API)]` | This marks code which is run when compiling with PyO3's `abi3` feature enabled. | /// | `#[cfg(PyPy)]` | This marks code which is run when compiling for PyPy. | +/// | `#[cfg(GraalPy)]` | This marks code which is run when compiling for GraalPy. | /// -/// For examples of how to use these attributes, [see PyO3's guide](https://pyo3.rs/latest/building_and_distribution/multiple_python_versions.html). +/// For examples of how to use these attributes, +#[doc = concat!("[see PyO3's guide](https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/building-and-distribution/multiple_python_versions.html)")] +/// . #[cfg(feature = "resolve-config")] pub fn use_pyo3_cfgs() { + print_expected_cfgs(); for cargo_command in get().build_script_outputs() { - println!("{}", cargo_command) + println!("{cargo_command}") } } @@ -62,7 +64,7 @@ pub fn add_extension_module_link_args() { } fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Write) { - if triple.operating_system == OperatingSystem::Darwin { + if matches!(triple.operating_system, OperatingSystem::Darwin(_)) { writeln!(writer, "cargo:rustc-cdylib-link-arg=-undefined").unwrap(); writeln!(writer, "cargo:rustc-cdylib-link-arg=dynamic_lookup").unwrap(); } else if triple == &Triple::from_str("wasm32-unknown-emscripten").unwrap() { @@ -71,12 +73,45 @@ fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Wr } } +/// Adds linker arguments suitable for linking against the Python framework on macOS. +/// +/// This should be called from a build script. +/// +/// The following link flags are added: +/// - macOS: `-Wl,-rpath,` +/// +/// All other platforms currently are no-ops. +#[cfg(feature = "resolve-config")] +pub fn add_python_framework_link_args() { + let target = impl_::target_triple_from_env(); + _add_python_framework_link_args( + get(), + &target, + impl_::is_linking_libpython_for_target(&target), + std::io::stdout(), + ) +} + +#[cfg(feature = "resolve-config")] +fn _add_python_framework_link_args( + interpreter_config: &InterpreterConfig, + triple: &Triple, + link_libpython: bool, + mut writer: impl std::io::Write, +) { + if matches!(triple.operating_system, OperatingSystem::Darwin(_)) && link_libpython { + if let Some(framework_prefix) = interpreter_config.python_framework_prefix.as_ref() { + writeln!(writer, "cargo:rustc-link-arg=-Wl,-rpath,{framework_prefix}").unwrap(); + } + } +} + /// Loads the configuration determined from the build environment. /// -/// Because this will never change in a given compilation run, this is cached in a `once_cell`. +/// Because this will never change in a given compilation run, this is cached in a `OnceLock`. #[cfg(feature = "resolve-config")] pub fn get() -> &'static InterpreterConfig { - static CONFIG: OnceCell = OnceCell::new(); + static CONFIG: OnceLock = OnceLock::new(); CONFIG.get_or_init(|| { // Check if we are in a build script and cross compiling to a different target. let cross_compile_config_path = resolve_cross_compile_config_path(); @@ -85,10 +120,12 @@ pub fn get() -> &'static InterpreterConfig { .map(|path| path.exists()) .unwrap_or(false); + // CONFIG_FILE is generated in build.rs, so its content can vary + #[allow(unknown_lints, clippy::const_is_empty)] if let Some(interpreter_config) = InterpreterConfig::from_cargo_dep_env() { interpreter_config - } else if !CONFIG_FILE.is_empty() { - InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE)) + } else if let Some(interpreter_config) = config_from_pyo3_config_file_env() { + Ok(interpreter_config) } else if cross_compiling { InterpreterConfig::from_path(cross_compile_config_path.as_ref().unwrap()) } else { @@ -98,13 +135,25 @@ pub fn get() -> &'static InterpreterConfig { }) } -/// Build configuration provided by `PYO3_CONFIG_FILE`. May be empty if env var not set. -#[doc(hidden)] +/// Build configuration provided by `PYO3_CONFIG_FILE`, inlined into the `pyo3-build-config` binary. #[cfg(feature = "resolve-config")] -const CONFIG_FILE: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config-file.txt")); +fn config_from_pyo3_config_file_env() -> Option { + #[doc(hidden)] + const CONFIG_FILE: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config-file.txt")); + + // CONFIG_FILE is generated in build.rs, so its content can vary + #[allow(clippy::const_is_empty)] + if !CONFIG_FILE.is_empty() { + let config = InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE)) + .expect("contents of CONFIG_FILE should always be valid (generated by pyo3-build-config's build.rs)"); + Some(config) + } else { + None + } +} /// Build configuration discovered by `pyo3-build-config` build script. Not aware of -/// cross-compilation settings. +/// cross-compilation settings. Not generated if `PYO3_CONFIG_FILE` is set. #[doc(hidden)] #[cfg(feature = "resolve-config")] const HOST_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config.txt")); @@ -125,33 +174,53 @@ fn resolve_cross_compile_config_path() -> Option { }) } +/// Helper to print a feature cfg with a minimum rust version required. +fn print_feature_cfg(minor_version_required: u32, cfg: &str) { + let minor_version = rustc_minor_version().unwrap_or(0); + + if minor_version >= minor_version_required { + println!("cargo:rustc-cfg={cfg}"); + } + + // rustc 1.80.0 stabilized `rustc-check-cfg` feature, don't emit before + if minor_version >= 80 { + println!("cargo:rustc-check-cfg=cfg({cfg})"); + } +} + /// Use certain features if we detect the compiler being used supports them. /// /// Features may be removed or added as MSRV gets bumped or new features become available, /// so this function is unstable. #[doc(hidden)] pub fn print_feature_cfgs() { - fn rustc_minor_version() -> Option { - let rustc = env::var_os("RUSTC")?; - let output = Command::new(rustc).arg("--version").output().ok()?; - let version = core::str::from_utf8(&output.stdout).ok()?; - let mut pieces = version.split('.'); - if pieces.next() != Some("rustc 1") { - return None; - } - pieces.next()?.parse().ok() - } - - let rustc_minor_version = rustc_minor_version().unwrap_or(0); + print_feature_cfg(85, "fn_ptr_eq"); + print_feature_cfg(86, "from_bytes_with_nul_error"); +} - // Enable use of const initializer for thread_local! on Rust 1.59 and greater - if rustc_minor_version >= 59 { - println!("cargo:rustc-cfg=thread_local_const_init"); +/// Registers `pyo3`s config names as reachable cfg expressions +/// +/// - +/// - +#[doc(hidden)] +pub fn print_expected_cfgs() { + if rustc_minor_version().is_some_and(|version| version < 80) { + // rustc 1.80.0 stabilized `rustc-check-cfg` feature, don't emit before + return; } - // invalid_from_utf8 lint was added in Rust 1.74 - if rustc_minor_version >= 74 { - println!("cargo:rustc-cfg=invalid_from_utf8_lint"); + println!("cargo:rustc-check-cfg=cfg(Py_LIMITED_API)"); + println!("cargo:rustc-check-cfg=cfg(Py_GIL_DISABLED)"); + println!("cargo:rustc-check-cfg=cfg(PyPy)"); + println!("cargo:rustc-check-cfg=cfg(GraalPy)"); + println!("cargo:rustc-check-cfg=cfg(py_sys_config, values(\"Py_DEBUG\", \"Py_REF_DEBUG\", \"Py_TRACE_REFS\", \"COUNT_ALLOCS\"))"); + println!("cargo:rustc-check-cfg=cfg(pyo3_disable_reference_pool)"); + println!("cargo:rustc-check-cfg=cfg(pyo3_leak_on_drop_without_reference_pool)"); + + // allow `Py_3_*` cfgs from the minimum supported version up to the + // maximum minor version (+1 for development for the next) + for i in impl_::MINIMUM_SUPPORTED_VERSION.minor..=impl_::ABI3_MAX_MINOR + 1 { + println!("cargo:rustc-check-cfg=cfg(Py_3_{i})"); } } @@ -159,36 +228,56 @@ pub fn print_feature_cfgs() { /// /// Please don't use these - they could change at any time. #[doc(hidden)] +#[cfg(feature = "resolve-config")] pub mod pyo3_build_script_impl { - #[cfg(feature = "resolve-config")] use crate::errors::{Context, Result}; - #[cfg(feature = "resolve-config")] use super::*; pub mod errors { pub use crate::errors::*; } pub use crate::impl_::{ - cargo_env_var, env_var, is_linking_libpython, make_cross_compile_config, InterpreterConfig, - PythonVersion, + cargo_env_var, env_var, is_linking_libpython_for_target, make_cross_compile_config, + target_triple_from_env, InterpreterConfig, PythonVersion, }; + pub enum BuildConfigSource { + /// Config was provided by `PYO3_CONFIG_FILE`. + ConfigFile, + /// Config was found by an interpreter on the host system. + Host, + /// Config was configured by cross-compilation settings. + CrossCompile, + } - /// Gets the configuration for use from PyO3's build script. + pub struct BuildConfig { + pub interpreter_config: InterpreterConfig, + pub source: BuildConfigSource, + } + + /// Gets the configuration for use from `pyo3-ffi`'s build script. /// - /// Differs from .get() above only in the cross-compile case, where PyO3's build script is - /// required to generate a new config (as it's the first build script which has access to the - /// correct value for CARGO_CFG_TARGET_OS). - #[cfg(feature = "resolve-config")] - pub fn resolve_interpreter_config() -> Result { - if !CONFIG_FILE.is_empty() { - let mut interperter_config = InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE))?; - interperter_config.generate_import_libs()?; - Ok(interperter_config) + /// Differs from `.get()` in three ways: + /// 1. The cargo_dep_env config is not yet available (exported by `pyo3-ffi`'s build script). + /// 1. If `PYO3_CONFIG_FILE` is set, lib name is fixed up and the windows import libs might be generated. + /// 2. The cross-compile config file is generated if necessary. + /// + /// Steps 2 and 3 are necessary because `pyo3-ffi`'s build script is the first code run which knows + /// the correct target triple. + pub fn resolve_build_config(target: &Triple) -> Result { + // CONFIG_FILE is generated in build.rs, so it's content can vary + #[allow(unknown_lints, clippy::const_is_empty)] + if let Some(mut interpreter_config) = config_from_pyo3_config_file_env() { + interpreter_config.apply_default_lib_name_to_config_file(target); + interpreter_config.generate_import_libs()?; + Ok(BuildConfig { + interpreter_config, + source: BuildConfigSource::ConfigFile, + }) } else if let Some(interpreter_config) = make_cross_compile_config()? { // This is a cross compile and need to write the config file. let path = resolve_cross_compile_config_path() - .expect("resolve_interpreter_config() must be called from a build script"); + .expect("resolve_build_config() must be called from a build script"); let parent_dir = path.parent().ok_or_else(|| { format!( "failed to resolve parent directory of config file {}", @@ -204,13 +293,71 @@ pub mod pyo3_build_script_impl { interpreter_config.to_writer(&mut std::fs::File::create(&path).with_context( || format!("failed to create config file at {}", path.display()), )?)?; - Ok(interpreter_config) + Ok(BuildConfig { + interpreter_config, + source: BuildConfigSource::CrossCompile, + }) } else { - InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG)) + let interpreter_config = InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG))?; + Ok(BuildConfig { + interpreter_config, + source: BuildConfigSource::Host, + }) + } + } + + /// Helper to generate an error message when the configured Python version is newer + /// than PyO3's current supported version. + pub struct MaximumVersionExceeded { + message: String, + } + + impl MaximumVersionExceeded { + pub fn new( + interpreter_config: &InterpreterConfig, + supported_version: PythonVersion, + ) -> Self { + let implementation = match interpreter_config.implementation { + PythonImplementation::CPython => "Python", + PythonImplementation::PyPy => "PyPy", + PythonImplementation::GraalPy => "GraalPy", + }; + let version = &interpreter_config.version; + let message = format!( + "the configured {implementation} version ({version}) is newer than PyO3's maximum supported version ({supported_version})\n\ + = help: this package is being built with PyO3 version {current_version}\n\ + = help: check https://crates.io/crates/pyo3 for the latest PyO3 version available\n\ + = help: updating this package to the latest version of PyO3 may provide compatibility with this {implementation} version", + current_version = env!("CARGO_PKG_VERSION") + ); + Self { message } + } + + pub fn add_help(&mut self, help: &str) { + self.message.push_str("\n= help: "); + self.message.push_str(help); + } + + pub fn finish(self) -> String { + self.message } } } +fn rustc_minor_version() -> Option { + static RUSTC_MINOR_VERSION: OnceLock> = OnceLock::new(); + *RUSTC_MINOR_VERSION.get_or_init(|| { + let rustc = env::var_os("RUSTC")?; + let output = Command::new(rustc).arg("--version").output().ok()?; + let version = core::str::from_utf8(&output.stdout).ok()?; + let mut pieces = version.split('.'); + if pieces.next() != Some("rustc 1") { + return None; + } + pieces.next()?.parse().ok() + }) +} + #[cfg(test)] mod tests { use super::*; @@ -247,4 +394,88 @@ mod tests { cargo:rustc-cdylib-link-arg=-sWASM_BIGINT\n" ); } + + #[cfg(feature = "resolve-config")] + #[test] + fn python_framework_link_args() { + let mut buf = Vec::new(); + + let interpreter_config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { + major: 3, + minor: 13, + }, + shared: true, + abi3: false, + lib_name: None, + lib_dir: None, + executable: None, + pointer_width: None, + build_flags: BuildFlags::default(), + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: Some( + "/Applications/Xcode.app/Contents/Developer/Library/Frameworks".to_string(), + ), + }; + // Does nothing on non-mac + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-pc-windows-msvc").unwrap(), + true, + &mut buf, + ); + assert_eq!(buf, Vec::new()); + + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-apple-darwin").unwrap(), + true, + &mut buf, + ); + assert_eq!( + std::str::from_utf8(&buf).unwrap(), + "cargo:rustc-link-arg=-Wl,-rpath,/Applications/Xcode.app/Contents/Developer/Library/Frameworks\n" + ); + } + + #[test] + #[cfg(feature = "resolve-config")] + fn test_maximum_version_exceeded_formatting() { + let interpreter_config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { + major: 3, + minor: 13, + }, + shared: true, + abi3: false, + lib_name: None, + lib_dir: None, + executable: None, + pointer_width: None, + build_flags: BuildFlags::default(), + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: None, + }; + let mut error = pyo3_build_script_impl::MaximumVersionExceeded::new( + &interpreter_config, + PythonVersion { + major: 3, + minor: 12, + }, + ); + error.add_help("this is a help message"); + let error = error.finish(); + let expected = concat!("\ + the configured Python version (3.13) is newer than PyO3's maximum supported version (3.12)\n\ + = help: this package is being built with PyO3 version ", env!("CARGO_PKG_VERSION"), "\n\ + = help: check https://crates.io/crates/pyo3 for the latest PyO3 version available\n\ + = help: updating this package to the latest version of PyO3 may provide compatibility with this Python version\n\ + = help: this is a help message" + ); + assert_eq!(error, expected); + } } diff --git a/pyo3-ffi-check/Cargo.toml b/pyo3-ffi-check/Cargo.toml index 776add0c910..874da3d2bef 100644 --- a/pyo3-ffi-check/Cargo.toml +++ b/pyo3-ffi-check/Cargo.toml @@ -13,7 +13,7 @@ path = "../pyo3-ffi" features = ["extension-module"] # A lazy way of skipping linking in most cases (as we don't use any runtime symbols) [build-dependencies] -bindgen = "0.66.1" +bindgen = "0.69.4" pyo3-build-config = { path = "../pyo3-build-config" } [workspace] diff --git a/pyo3-ffi-check/build.rs b/pyo3-ffi-check/build.rs index ca4a17b6a61..e7cfbe40df3 100644 --- a/pyo3-ffi-check/build.rs +++ b/pyo3-ffi-check/build.rs @@ -1,6 +1,24 @@ use std::env; use std::path::PathBuf; +#[derive(Debug)] +struct ParseCallbacks; + +impl bindgen::callbacks::ParseCallbacks for ParseCallbacks { + // these are anonymous fields and structs in CPython that we needed to + // invent names for. Bindgen seems to generate stable names, so we remap the + // automatically generated names to the names we invented in the FFI + fn item_name(&self, _original_item_name: &str) -> Option { + if _original_item_name == "_object__bindgen_ty_1__bindgen_ty_1" { + Some("PyObjectObFlagsAndRefcnt".into()) + } else if _original_item_name == "_object__bindgen_ty_1" { + Some("PyObjectObRefcnt".into()) + } else { + None + } + } +} + fn main() { let config = pyo3_build_config::get(); let python_include_dir = config @@ -8,13 +26,28 @@ fn main() { "import sysconfig; print(sysconfig.get_config_var('INCLUDEPY'), end='');", ) .expect("failed to get lib dir"); + let gil_disabled_on_windows = config + .run_python_script( + "import sysconfig; import platform; print(sysconfig.get_config_var('Py_GIL_DISABLED') == 1 and platform.system() == 'Windows');", + ) + .expect("failed to get Py_GIL_DISABLED").trim_end() == "True"; + + let clang_args = if gil_disabled_on_windows { + vec![ + format!("-I{python_include_dir}"), + "-DPy_GIL_DISABLED".to_string(), + ] + } else { + vec![format!("-I{python_include_dir}")] + }; println!("cargo:rerun-if-changed=wrapper.h"); let bindings = bindgen::Builder::default() .header("wrapper.h") - .clang_arg(format!("-I{python_include_dir}")) - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .clang_args(clang_args) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .parse_callbacks(Box::new(ParseCallbacks)) // blocklist some values which apparently have conflicting definitions on unix .blocklist_item("FP_NORMAL") .blocklist_item("FP_SUBNORMAL") diff --git a/pyo3-ffi-check/macro/Cargo.toml b/pyo3-ffi-check/macro/Cargo.toml index 46395fc8a8c..dfefcd2c1e1 100644 --- a/pyo3-ffi-check/macro/Cargo.toml +++ b/pyo3-ffi-check/macro/Cargo.toml @@ -10,6 +10,6 @@ proc-macro = true [dependencies] glob = "0.3" quote = "1" -proc-macro2 = "1" +proc-macro2 = "1.0.60" scraper = "0.17" pyo3-build-config = { path = "../../pyo3-build-config" } diff --git a/pyo3-ffi-check/macro/src/lib.rs b/pyo3-ffi-check/macro/src/lib.rs index e3d442c3703..41092b9020e 100644 --- a/pyo3-ffi-check/macro/src/lib.rs +++ b/pyo3-ffi-check/macro/src/lib.rs @@ -49,6 +49,7 @@ pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStrea .unwrap() .strip_suffix(".html") .unwrap(); + let struct_ident = Ident::new(struct_name, Span::call_site()); output.extend(quote!(#macro_name!(#struct_ident);)); } diff --git a/pyo3-ffi-check/src/main.rs b/pyo3-ffi-check/src/main.rs index 99713524702..0407a2ffa39 100644 --- a/pyo3-ffi-check/src/main.rs +++ b/pyo3-ffi-check/src/main.rs @@ -48,7 +48,8 @@ fn main() { macro_rules! check_field { ($struct_name:ident, $field:ident, $bindgen_field:ident) => {{ - #[allow(clippy::used_underscore_binding)] + // some struct fields are deprecated but still present in the ABI + #[allow(clippy::used_underscore_binding, deprecated)] let pyo3_ffi_offset = memoffset::offset_of!(pyo3_ffi::$struct_name, $field); #[allow(clippy::used_underscore_binding)] let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $bindgen_field); diff --git a/pyo3-ffi/ACKNOWLEDGEMENTS b/pyo3-ffi/ACKNOWLEDGEMENTS index 4502d7774e0..8b20727dece 100644 --- a/pyo3-ffi/ACKNOWLEDGEMENTS +++ b/pyo3-ffi/ACKNOWLEDGEMENTS @@ -3,4 +3,4 @@ for binary compatibility, with additional metadata to support PyPy. For original implementations please see: - https://github.com/python/cpython - - https://foss.heptapod.net/pypy/pypy + - https://github.com/pypy/pypy diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 8021dc72b69..03de6584643 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.21.0-dev" +version = "0.27.0" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -10,6 +10,7 @@ categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" edition = "2021" links = "python" +rust-version.workspace = true [dependencies] libc = "0.2.62" @@ -32,13 +33,26 @@ abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"] abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"] abi3-py310 = ["abi3-py311", "pyo3-build-config/abi3-py310"] abi3-py311 = ["abi3-py312", "pyo3-build-config/abi3-py311"] -abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312"] +abi3-py312 = ["abi3-py313", "pyo3-build-config/abi3-py312"] +abi3-py313 = ["abi3-py314", "pyo3-build-config/abi3-py313"] +abi3-py314 = ["abi3", "pyo3-build-config/abi3-py314"] # Automatically generates `python3.dll` import libraries for Windows targets. -generate-import-lib = ["pyo3-build-config/python3-dll-a"] +generate-import-lib = ["pyo3-build-config/generate-import-lib"] + +[dev-dependencies] +paste = "1" [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.27.0", features = ["resolve-config"] } [lints] workspace = true + +[package.metadata.cpython] +min-version = "3.7" +max-version = "3.14" # inclusive + +[package.metadata.pypy] +min-version = "3.11" +max-version = "3.11" # inclusive diff --git a/pyo3-ffi/README.md b/pyo3-ffi/README.md index 030c59d449c..41b9546ac50 100644 --- a/pyo3-ffi/README.md +++ b/pyo3-ffi/README.md @@ -12,9 +12,12 @@ Manual][capi] for up-to-date documentation. # Minimum supported Rust and Python versions -PyO3 supports the following software versions: - - Python 3.7 and up (CPython and PyPy) - - Rust 1.56 and up +Requires Rust 1.63 or greater. + +`pyo3-ffi` supports the following Python distributions: + - CPython 3.7 or greater + - PyPy 7.3 (Python 3.9+) + - GraalPy 24.0 or greater (Python 3.10+) # Example: Building Python Native modules @@ -38,26 +41,43 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies.pyo3-ffi] -version = "*" +version = "0.27.0" features = ["extension-module"] + +[build-dependencies] +# This is only necessary if you need to configure your build based on +# the Python version or the compile-time configuration for the interpreter. +pyo3_build_config = "0.27.0" +``` + +If you need to use conditional compilation based on Python version or how +Python was compiled, you need to add `pyo3-build-config` as a +`build-dependency` in your `Cargo.toml` as in the example above and either +create a new `build.rs` file or modify an existing one so that +`pyo3_build_config::use_pyo3_cfgs()` gets called at build time: + +**`build.rs`** + +```rust,ignore +fn main() { + pyo3_build_config::use_pyo3_cfgs() +} ``` **`src/lib.rs`** -```rust -use std::os::raw::c_char; +```rust,no_run +use std::ffi::{c_char, c_long}; use std::ptr; use pyo3_ffi::*; static mut MODULE_DEF: PyModuleDef = PyModuleDef { m_base: PyModuleDef_HEAD_INIT, - m_name: "string_sum\0".as_ptr().cast::(), - m_doc: "A Python module written in Rust.\0" - .as_ptr() - .cast::(), + m_name: c"string_sum".as_ptr(), + m_doc: c"A Python module written in Rust.".as_ptr(), m_size: 0, - m_methods: unsafe { METHODS.as_mut_ptr().cast() }, - m_slots: std::ptr::null_mut(), + m_methods: std::ptr::addr_of_mut!(METHODS).cast(), + m_slots: unsafe { SLOTS as *const [PyModuleDef_Slot] as *mut PyModuleDef_Slot }, m_traverse: None, m_clear: None, m_free: None, @@ -65,74 +85,116 @@ static mut MODULE_DEF: PyModuleDef = PyModuleDef { static mut METHODS: [PyMethodDef; 2] = [ PyMethodDef { - ml_name: "sum_as_string\0".as_ptr().cast::(), + ml_name: c"sum_as_string".as_ptr(), ml_meth: PyMethodDefPointer { - _PyCFunctionFast: sum_as_string, + PyCFunctionFast: sum_as_string, }, ml_flags: METH_FASTCALL, - ml_doc: "returns the sum of two integers as a string\0" - .as_ptr() - .cast::(), + ml_doc: c"returns the sum of two integers as a string".as_ptr(), }, // A zeroed PyMethodDef to mark the end of the array. - PyMethodDef::zeroed() + PyMethodDef::zeroed(), +]; + +static mut SLOTS: &[PyModuleDef_Slot] = &[ + // NB: only include this slot if the module does not store any global state in `static` variables + // or other data which could cross between subinterpreters + #[cfg(Py_3_12)] + PyModuleDef_Slot { + slot: Py_mod_multiple_interpreters, + value: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED, + }, + // NB: only include this slot if the module does not depend on the GIL for thread safety + #[cfg(Py_GIL_DISABLED)] + PyModuleDef_Slot { + slot: Py_mod_gil, + value: Py_MOD_GIL_NOT_USED, + }, + PyModuleDef_Slot { + slot: 0, + value: ptr::null_mut(), + }, ]; // The module initialization function, which must be named `PyInit_`. #[allow(non_snake_case)] #[no_mangle] pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { - PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)) + PyModuleDef_Init(ptr::addr_of_mut!(MODULE_DEF)) } -pub unsafe extern "C" fn sum_as_string( - _self: *mut PyObject, - args: *mut *mut PyObject, - nargs: Py_ssize_t, -) -> *mut PyObject { - if nargs != 2 { - PyErr_SetString( - PyExc_TypeError, - "sum_as_string() expected 2 positional arguments\0" - .as_ptr() - .cast::(), +/// A helper to parse function arguments +/// If we used PyO3's proc macros they'd handle all of this boilerplate for us :) +unsafe fn parse_arg_as_i32(obj: *mut PyObject, n_arg: usize) -> Option { + if PyLong_Check(obj) == 0 { + let msg = format!( + "sum_as_string expected an int for positional argument {}\0", + n_arg ); - return std::ptr::null_mut(); + PyErr_SetString(PyExc_TypeError, msg.as_ptr().cast::()); + return None; } - let arg1 = *args; - if PyLong_Check(arg1) == 0 { - PyErr_SetString( - PyExc_TypeError, - "sum_as_string() expected an int for positional argument 1\0" - .as_ptr() - .cast::(), - ); - return std::ptr::null_mut(); + // Let's keep the behaviour consistent on platforms where `c_long` is bigger than 32 bits. + // In particular, it is an i32 on Windows but i64 on most Linux systems + let mut overflow = 0; + let i_long: c_long = PyLong_AsLongAndOverflow(obj, &mut overflow); + + #[allow(irrefutable_let_patterns)] // some platforms have c_long equal to i32 + if overflow != 0 { + raise_overflowerror(obj); + None + } else if let Ok(i) = i_long.try_into() { + Some(i) + } else { + raise_overflowerror(obj); + None } +} - let arg1 = PyLong_AsLong(arg1); - if !PyErr_Occurred().is_null() { - return ptr::null_mut(); +unsafe fn raise_overflowerror(obj: *mut PyObject) { + let obj_repr = PyObject_Str(obj); + if !obj_repr.is_null() { + let mut size = 0; + let p = PyUnicode_AsUTF8AndSize(obj_repr, &mut size); + if !p.is_null() { + let s = std::str::from_utf8_unchecked(std::slice::from_raw_parts( + p.cast::(), + size as usize, + )); + let msg = format!("cannot fit {} in 32 bits\0", s); + + PyErr_SetString(PyExc_OverflowError, msg.as_ptr().cast::()); + } + Py_DECREF(obj_repr); } +} - let arg2 = *args.add(1); - if PyLong_Check(arg2) == 0 { +pub unsafe extern "C" fn sum_as_string( + _self: *mut PyObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, +) -> *mut PyObject { + if nargs != 2 { PyErr_SetString( PyExc_TypeError, - "sum_as_string() expected an int for positional argument 2\0" - .as_ptr() - .cast::(), + c"sum_as_string expected 2 positional arguments".as_ptr(), ); return std::ptr::null_mut(); } - let arg2 = PyLong_AsLong(arg2); - if !PyErr_Occurred().is_null() { - return ptr::null_mut(); - } + let (first, second) = (*args, *args.add(1)); + + let first = match parse_arg_as_i32(first, 1) { + Some(x) => x, + None => return std::ptr::null_mut(), + }; + let second = match parse_arg_as_i32(second, 2) { + Some(x) => x, + None => return std::ptr::null_mut(), + }; - match arg1.checked_add(arg2) { + match first.checked_add(second) { Some(sum) => { let string = sum.to_string(); PyUnicode_FromStringAndSize(string.as_ptr().cast::(), string.len() as isize) @@ -140,7 +202,7 @@ pub unsafe extern "C" fn sum_as_string( None => { PyErr_SetString( PyExc_OverflowError, - "arguments too large to add\0".as_ptr().cast::(), + c"arguments too large to add".as_ptr(), ); std::ptr::null_mut() } @@ -184,7 +246,7 @@ can be easily converted to rust as well. [`maturin`]: https://github.com/PyO3/maturin "Build and publish crates with pyo3, rust-cpython and cffi bindings as well as rust binaries as python packages" [`pyo3-build-config`]: https://docs.rs/pyo3-build-config [feature flags]: https://doc.rust-lang.org/cargo/reference/features.html "Features - The Cargo Book" -[manual_builds]: https://pyo3.rs/latest/building_and_distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide" +[manual_builds]: https://pyo3.rs/latest/building-and-distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide" [setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions" [PEP 384]: https://www.python.org/dev/peps/pep-0384 "PEP 384 -- Defining a Stable ABI" [Features chapter of the guide]: https://pyo3.rs/latest/features.html#features-reference "Features Reference - PyO3 user guide" diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 286767d8f25..3e2354f2d38 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -1,10 +1,11 @@ use pyo3_build_config::{ bail, ensure, print_feature_cfgs, pyo3_build_script_impl::{ - cargo_env_var, env_var, errors::Result, is_linking_libpython, resolve_interpreter_config, - InterpreterConfig, PythonVersion, + cargo_env_var, env_var, errors::Result, is_linking_libpython_for_target, + resolve_build_config, target_triple_from_env, BuildConfig, BuildConfigSource, + InterpreterConfig, MaximumVersionExceeded, PythonVersion, }, - PythonImplementation, + warn, PythonImplementation, }; /// Minimum Python version PyO3 supports. @@ -17,16 +18,24 @@ const SUPPORTED_VERSIONS_CPYTHON: SupportedVersions = SupportedVersions { min: PythonVersion { major: 3, minor: 7 }, max: PythonVersion { major: 3, - minor: 12, + minor: 14, }, }; const SUPPORTED_VERSIONS_PYPY: SupportedVersions = SupportedVersions { - min: PythonVersion { major: 3, minor: 7 }, - max: PythonVersion { + min: PythonVersion { + major: 3, + minor: 11, + }, + max: SUPPORTED_VERSIONS_CPYTHON.max, +}; + +const SUPPORTED_VERSIONS_GRAALPY: SupportedVersions = SupportedVersions { + min: PythonVersion { major: 3, minor: 10, }, + max: SUPPORTED_VERSIONS_CPYTHON.max, }; fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { @@ -45,15 +54,21 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { interpreter_config.version, versions.min, ); - ensure!( - interpreter_config.version <= versions.max || env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1"), - "the configured Python interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ - = help: please check if an updated version of PyO3 is available. Current version: {}\n\ - = help: set PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 to suppress this check and build anyway using the stable ABI", - interpreter_config.version, - versions.max, - std::env::var("CARGO_PKG_VERSION").unwrap(), - ); + if interpreter_config.version > versions.max { + let mut error = MaximumVersionExceeded::new(interpreter_config, versions.max); + if interpreter_config.is_free_threaded() { + error.add_help( + "the free-threaded build of CPython does not support the limited API so this check cannot be suppressed.", + ); + return Err(error.finish().into()); + } + + if env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").is_none_or(|os_str| os_str != "1") + { + error.add_help("set PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 to suppress this check and build anyway using the stable ABI"); + return Err(error.finish().into()); + } + } } PythonImplementation::PyPy => { let versions = SUPPORTED_VERSIONS_PYPY; @@ -64,14 +79,43 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { versions.min, ); // PyO3 does not support abi3, so we cannot offer forward compatibility + if interpreter_config.version > versions.max { + let error = MaximumVersionExceeded::new(interpreter_config, versions.max); + return Err(error.finish().into()); + } + } + PythonImplementation::GraalPy => { + let versions = SUPPORTED_VERSIONS_GRAALPY; ensure!( - interpreter_config.version <= versions.max, - "the configured PyPy interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ - = help: please check if an updated version of PyO3 is available. Current version: {}", + interpreter_config.version >= versions.min, + "the configured GraalPy interpreter version ({}) is lower than PyO3's minimum supported version ({})", interpreter_config.version, - versions.max, - std::env::var("CARGO_PKG_VERSION").unwrap() + versions.min, ); + // GraalPy does not support abi3, so we cannot offer forward compatibility + if interpreter_config.version > versions.max { + let error = MaximumVersionExceeded::new(interpreter_config, versions.max); + return Err(error.finish().into()); + } + } + } + + if interpreter_config.abi3 { + match interpreter_config.implementation { + PythonImplementation::CPython => { + if interpreter_config.is_free_threaded() { + warn!( + "The free-threaded build of CPython does not yet support abi3 so the build artifacts will be version-specific." + ) + } + } + PythonImplementation::PyPy => warn!( + "PyPy does not yet support abi3 so the build artifacts will be version-specific. \ + See https://github.com/pypy/pypy/issues/3397 for more information." + ), + PythonImplementation::GraalPy => warn!( + "GraalPy does not support abi3 so the build artifacts will be version-specific." + ), } } @@ -100,7 +144,8 @@ fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result Ok(()) } -fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { +fn emit_link_config(build_config: &BuildConfig) -> Result<()> { + let interpreter_config = &build_config.interpreter_config; let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap(); println!( @@ -121,7 +166,14 @@ fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { ); if let Some(lib_dir) = &interpreter_config.lib_dir { - println!("cargo:rustc-link-search=native={}", lib_dir); + println!("cargo:rustc-link-search=native={lib_dir}"); + } else if matches!(build_config.source, BuildConfigSource::CrossCompile) { + warn!( + "The output binary will link to libpython, \ + but PYO3_CROSS_LIB_DIR environment variable is not set. \ + Ensure that the target Python library directory is \ + in the rustc native library search path." + ); } Ok(()) @@ -135,32 +187,35 @@ fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { /// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler /// version to enable features which aren't supported on MSRV. fn configure_pyo3() -> Result<()> { - let interpreter_config = resolve_interpreter_config()?; + let target = target_triple_from_env(); + let build_config = resolve_build_config(&target)?; + let interpreter_config = &build_config.interpreter_config; - if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") { - print_config_and_exit(&interpreter_config); + if env_var("PYO3_PRINT_CONFIG").is_some_and(|os_str| os_str == "1") { + print_config_and_exit(interpreter_config); } - ensure_python_version(&interpreter_config)?; - ensure_target_pointer_width(&interpreter_config)?; + ensure_python_version(interpreter_config)?; + ensure_target_pointer_width(interpreter_config)?; // Serialize the whole interpreter config into DEP_PYTHON_PYO3_CONFIG env var. interpreter_config.to_cargo_dep_env()?; - if is_linking_libpython() && !interpreter_config.suppress_build_script_link_lines { - emit_link_config(&interpreter_config)?; + if is_linking_libpython_for_target(&target) + && !interpreter_config.suppress_build_script_link_lines + { + emit_link_config(&build_config)?; } for cfg in interpreter_config.build_script_outputs() { - println!("{}", cfg) + println!("{cfg}") } // Extra lines come last, to support last write wins. for line in &interpreter_config.extra_build_script_lines { - println!("{}", line); + println!("{line}"); } - // Emit cfgs like `thread_local_const_init` print_feature_cfgs(); Ok(()) @@ -176,6 +231,7 @@ fn print_config_and_exit(config: &InterpreterConfig) { } fn main() { + pyo3_build_config::print_expected_cfgs(); if let Err(e) = configure_pyo3() { eprintln!("error: {}", e.report()); std::process::exit(1) diff --git a/pyo3-ffi/examples/README.md b/pyo3-ffi/examples/README.md new file mode 100644 index 00000000000..f02ae4ba6b4 --- /dev/null +++ b/pyo3-ffi/examples/README.md @@ -0,0 +1,21 @@ +# `pyo3-ffi` Examples + +These example crates are a collection of toy extension modules built with +`pyo3-ffi`. They are all tested using `nox` in PyO3's CI. + +Below is a brief description of each of these: + +| Example | Description | +| `word-count` | Illustrates how to use pyo3-ffi to write a static rust extension | +| `sequential` | Illustrates how to use pyo3-ffi to write subinterpreter-safe modules using multi-phase module initialization | + +## Creating new projects from these examples + +To copy an example, use [`cargo-generate`](https://crates.io/crates/cargo-generate). Follow the commands below, replacing `` with the example to start from: + +```bash +$ cargo install cargo-generate +$ cargo generate --git https://github.com/PyO3/pyo3 examples/ +``` + +(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.) diff --git a/examples/sequential/.template/Cargo.toml b/pyo3-ffi/examples/sequential/.template/Cargo.toml similarity index 100% rename from examples/sequential/.template/Cargo.toml rename to pyo3-ffi/examples/sequential/.template/Cargo.toml diff --git a/examples/sequential/.template/pre-script.rhai b/pyo3-ffi/examples/sequential/.template/pre-script.rhai similarity index 100% rename from examples/sequential/.template/pre-script.rhai rename to pyo3-ffi/examples/sequential/.template/pre-script.rhai diff --git a/examples/sequential/.template/pyproject.toml b/pyo3-ffi/examples/sequential/.template/pyproject.toml similarity index 100% rename from examples/sequential/.template/pyproject.toml rename to pyo3-ffi/examples/sequential/.template/pyproject.toml diff --git a/pyo3-ffi/examples/sequential/Cargo.toml b/pyo3-ffi/examples/sequential/Cargo.toml new file mode 100644 index 00000000000..21e87c09496 --- /dev/null +++ b/pyo3-ffi/examples/sequential/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sequential" +version = "0.1.0" +edition = "2021" +rust-version = "1.83" + +[lib] +name = "sequential" +crate-type = ["cdylib", "lib"] + +[dependencies] +pyo3-ffi = { path = "../../", features = ["extension-module"] } + +[build-dependencies] +pyo3-build-config = { path = "../../../pyo3-build-config" } + +[workspace] diff --git a/examples/sequential/MANIFEST.in b/pyo3-ffi/examples/sequential/MANIFEST.in similarity index 100% rename from examples/sequential/MANIFEST.in rename to pyo3-ffi/examples/sequential/MANIFEST.in diff --git a/examples/sequential/README.md b/pyo3-ffi/examples/sequential/README.md similarity index 91% rename from examples/sequential/README.md rename to pyo3-ffi/examples/sequential/README.md index e3c0078241d..6eb718d9d98 100644 --- a/examples/sequential/README.md +++ b/pyo3-ffi/examples/sequential/README.md @@ -1,6 +1,6 @@ # sequential -A project built using only `pyo3_ffi`, without any of PyO3's safe api. It can be executed by subinterpreters that have their own GIL. +A project built using only `pyo3_ffi`, without any of PyO3's safe api. It supports both subinterpreters and free-threaded Python. ## Building and Testing diff --git a/pyo3-ffi/examples/sequential/build.rs b/pyo3-ffi/examples/sequential/build.rs new file mode 100644 index 00000000000..0475124bb4e --- /dev/null +++ b/pyo3-ffi/examples/sequential/build.rs @@ -0,0 +1,3 @@ +fn main() { + pyo3_build_config::use_pyo3_cfgs(); +} diff --git a/examples/sequential/cargo-generate.toml b/pyo3-ffi/examples/sequential/cargo-generate.toml similarity index 100% rename from examples/sequential/cargo-generate.toml rename to pyo3-ffi/examples/sequential/cargo-generate.toml diff --git a/examples/sequential/noxfile.py b/pyo3-ffi/examples/sequential/noxfile.py similarity index 100% rename from examples/sequential/noxfile.py rename to pyo3-ffi/examples/sequential/noxfile.py diff --git a/examples/sequential/pyproject.toml b/pyo3-ffi/examples/sequential/pyproject.toml similarity index 100% rename from examples/sequential/pyproject.toml rename to pyo3-ffi/examples/sequential/pyproject.toml diff --git a/examples/sequential/src/id.rs b/pyo3-ffi/examples/sequential/src/id.rs similarity index 77% rename from examples/sequential/src/id.rs rename to pyo3-ffi/examples/sequential/src/id.rs index d80e84b4eab..21c667242de 100644 --- a/examples/sequential/src/id.rs +++ b/pyo3-ffi/examples/sequential/src/id.rs @@ -1,6 +1,7 @@ use core::sync::atomic::{AtomicU64, Ordering}; use core::{mem, ptr}; -use std::os::raw::{c_char, c_int, c_uint, c_ulonglong, c_void}; +use std::ffi::CString; +use std::ffi::{c_char, c_int, c_uint, c_ulonglong, c_void}; use pyo3_ffi::*; @@ -27,10 +28,10 @@ unsafe extern "C" fn id_new( kwds: *mut PyObject, ) -> *mut PyObject { if PyTuple_Size(args) != 0 || !kwds.is_null() { - PyErr_SetString( - PyExc_TypeError, - "Id() takes no arguments\0".as_ptr().cast::(), - ); + // We use pyo3-ffi's `c_str!` macro to create null-terminated literals because + // Rust's string literals are not null-terminated + // On Rust 1.77 or newer you can use `c"text"` instead. + PyErr_SetString(PyExc_TypeError, c"Id() takes no arguments".as_ptr()); return ptr::null_mut(); } @@ -81,8 +82,12 @@ unsafe extern "C" fn id_richcompare( pyo3_ffi::Py_GT => slf > other, pyo3_ffi::Py_GE => slf >= other, unrecognized => { - let msg = format!("unrecognized richcompare opcode {}\0", unrecognized); - PyErr_SetString(PyExc_SystemError, msg.as_ptr().cast::()); + let msg = CString::new(&*format!( + "unrecognized richcompare opcode {}", + unrecognized + )) + .unwrap(); + PyErr_SetString(PyExc_SystemError, msg.as_ptr()); return ptr::null_mut(); } }; @@ -94,15 +99,14 @@ unsafe extern "C" fn id_richcompare( } } -static mut SLOTS: &[PyType_Slot] = &[ +static mut SLOTS: [PyType_Slot; 6] = [ PyType_Slot { slot: Py_tp_new, pfunc: id_new as *mut c_void, }, PyType_Slot { slot: Py_tp_doc, - pfunc: "An id that is increased every time an instance is created\0".as_ptr() - as *mut c_void, + pfunc: c"An id that is increased every time an instance is created".as_ptr() as *mut c_void, }, PyType_Slot { slot: Py_tp_repr, @@ -123,9 +127,9 @@ static mut SLOTS: &[PyType_Slot] = &[ ]; pub static mut ID_SPEC: PyType_Spec = PyType_Spec { - name: "sequential.Id\0".as_ptr().cast::(), + name: c"sequential.Id".as_ptr(), basicsize: mem::size_of::() as c_int, itemsize: 0, flags: (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE) as c_uint, - slots: unsafe { SLOTS as *const [PyType_Slot] as *mut PyType_Slot }, + slots: ptr::addr_of_mut!(SLOTS).cast(), }; diff --git a/examples/sequential/src/lib.rs b/pyo3-ffi/examples/sequential/src/lib.rs similarity index 100% rename from examples/sequential/src/lib.rs rename to pyo3-ffi/examples/sequential/src/lib.rs diff --git a/examples/sequential/src/module.rs b/pyo3-ffi/examples/sequential/src/module.rs similarity index 73% rename from examples/sequential/src/module.rs rename to pyo3-ffi/examples/sequential/src/module.rs index 5552baf3368..c8d506c5568 100644 --- a/examples/sequential/src/module.rs +++ b/pyo3-ffi/examples/sequential/src/module.rs @@ -1,30 +1,35 @@ use core::{mem, ptr}; use pyo3_ffi::*; -use std::os::raw::{c_char, c_int, c_void}; +use std::ffi::{c_int, c_void}; pub static mut MODULE_DEF: PyModuleDef = PyModuleDef { m_base: PyModuleDef_HEAD_INIT, - m_name: "sequential\0".as_ptr().cast::(), - m_doc: "A library for generating sequential ids, written in Rust.\0" - .as_ptr() - .cast::(), + m_name: c"sequential".as_ptr(), + m_doc: c"A library for generating sequential ids, written in Rust.".as_ptr(), m_size: mem::size_of::() as Py_ssize_t, m_methods: std::ptr::null_mut(), - m_slots: unsafe { SEQUENTIAL_SLOTS as *const [PyModuleDef_Slot] as *mut PyModuleDef_Slot }, + m_slots: std::ptr::addr_of_mut!(SEQUENTIAL_SLOTS).cast(), m_traverse: Some(sequential_traverse), m_clear: Some(sequential_clear), m_free: Some(sequential_free), }; -static mut SEQUENTIAL_SLOTS: &[PyModuleDef_Slot] = &[ +const SEQUENTIAL_SLOTS_LEN: usize = 3 + if cfg!(Py_GIL_DISABLED) { 1 } else { 0 }; +static mut SEQUENTIAL_SLOTS: [PyModuleDef_Slot; SEQUENTIAL_SLOTS_LEN] = [ PyModuleDef_Slot { slot: Py_mod_exec, value: sequential_exec as *mut c_void, }, + #[cfg(Py_3_12)] PyModuleDef_Slot { slot: Py_mod_multiple_interpreters, value: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED, }, + #[cfg(Py_GIL_DISABLED)] + PyModuleDef_Slot { + slot: Py_mod_gil, + value: Py_MOD_GIL_NOT_USED, + }, PyModuleDef_Slot { slot: 0, value: ptr::null_mut(), @@ -40,15 +45,12 @@ unsafe extern "C" fn sequential_exec(module: *mut PyObject) -> c_int { ptr::null_mut(), ); if id_type.is_null() { - PyErr_SetString( - PyExc_SystemError, - "cannot locate type object\0".as_ptr().cast::(), - ); + PyErr_SetString(PyExc_SystemError, c"cannot locate type object".as_ptr()); return -1; } (*state).id_type = id_type.cast::(); - PyModule_AddObjectRef(module, "Id\0".as_ptr().cast::(), id_type) + PyModule_AddObjectRef(module, c"Id".as_ptr(), id_type) } unsafe extern "C" fn sequential_traverse( diff --git a/examples/sequential/tests/test.rs b/pyo3-ffi/examples/sequential/tests/test.rs similarity index 88% rename from examples/sequential/tests/test.rs rename to pyo3-ffi/examples/sequential/tests/test.rs index 6076edd4974..f4b68092e1d 100644 --- a/examples/sequential/tests/test.rs +++ b/pyo3-ffi/examples/sequential/tests/test.rs @@ -5,11 +5,11 @@ use std::thread; use pyo3_ffi::*; use sequential::PyInit_sequential; -static COMMAND: &'static str = " +static COMMAND: &'static CStr= c" from sequential import Id s = sum(int(Id()) for _ in range(12)) -\0"; +"; // Newtype to be able to pass it to another thread. struct State(*mut PyThreadState); @@ -19,10 +19,7 @@ unsafe impl Send for State {} #[test] fn lets_go_fast() -> Result<(), String> { unsafe { - let ret = PyImport_AppendInittab( - "sequential\0".as_ptr().cast::(), - Some(PyInit_sequential), - ); + let ret = PyImport_AppendInittab(c"sequential".as_ptr(), Some(PyInit_sequential)); if ret == -1 { return Err("could not add module to inittab".into()); } @@ -122,11 +119,7 @@ unsafe fn fetch() -> String { fn run_code() -> Result { unsafe { - let code_obj = Py_CompileString( - COMMAND.as_ptr().cast::(), - "program\0".as_ptr().cast::(), - Py_file_input, - ); + let code_obj = Py_CompileString(COMMAND.as_ptr(), c"program".as_ptr(), Py_file_input); if code_obj.is_null() { return Err(fetch()); } @@ -138,7 +131,7 @@ fn run_code() -> Result { } else { Py_DECREF(res_ptr); } - let sum = PyDict_GetItemString(globals, "s\0".as_ptr().cast::()); /* borrowed reference */ + let sum = PyDict_GetItemString(globals, c"s".as_ptr()); /* borrowed reference */ if sum.is_null() { Py_DECREF(globals); return Err("globals did not have `s`".into()); diff --git a/examples/sequential/tests/test_.py b/pyo3-ffi/examples/sequential/tests/test_.py similarity index 100% rename from examples/sequential/tests/test_.py rename to pyo3-ffi/examples/sequential/tests/test_.py diff --git a/examples/string-sum/.template/Cargo.toml b/pyo3-ffi/examples/string-sum/.template/Cargo.toml similarity index 100% rename from examples/string-sum/.template/Cargo.toml rename to pyo3-ffi/examples/string-sum/.template/Cargo.toml diff --git a/examples/string-sum/.template/pre-script.rhai b/pyo3-ffi/examples/string-sum/.template/pre-script.rhai similarity index 100% rename from examples/string-sum/.template/pre-script.rhai rename to pyo3-ffi/examples/string-sum/.template/pre-script.rhai diff --git a/examples/string-sum/.template/pyproject.toml b/pyo3-ffi/examples/string-sum/.template/pyproject.toml similarity index 100% rename from examples/string-sum/.template/pyproject.toml rename to pyo3-ffi/examples/string-sum/.template/pyproject.toml diff --git a/pyo3-ffi/examples/string-sum/Cargo.toml b/pyo3-ffi/examples/string-sum/Cargo.toml new file mode 100644 index 00000000000..173521eaab5 --- /dev/null +++ b/pyo3-ffi/examples/string-sum/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "string_sum" +version = "0.1.0" +edition = "2021" +rust-version = "1.83" + +[lib] +name = "string_sum" +crate-type = ["cdylib"] + +[dependencies] +pyo3-ffi = { path = "../../", features = ["extension-module"] } + +[build-dependencies] +pyo3-build-config = { path = "../../../pyo3-build-config" } + +[workspace] diff --git a/examples/string-sum/MANIFEST.in b/pyo3-ffi/examples/string-sum/MANIFEST.in similarity index 100% rename from examples/string-sum/MANIFEST.in rename to pyo3-ffi/examples/string-sum/MANIFEST.in diff --git a/examples/string-sum/README.md b/pyo3-ffi/examples/string-sum/README.md similarity index 100% rename from examples/string-sum/README.md rename to pyo3-ffi/examples/string-sum/README.md diff --git a/pyo3-ffi/examples/string-sum/build.rs b/pyo3-ffi/examples/string-sum/build.rs new file mode 100644 index 00000000000..0475124bb4e --- /dev/null +++ b/pyo3-ffi/examples/string-sum/build.rs @@ -0,0 +1,3 @@ +fn main() { + pyo3_build_config::use_pyo3_cfgs(); +} diff --git a/examples/string-sum/cargo-generate.toml b/pyo3-ffi/examples/string-sum/cargo-generate.toml similarity index 100% rename from examples/string-sum/cargo-generate.toml rename to pyo3-ffi/examples/string-sum/cargo-generate.toml diff --git a/examples/string-sum/noxfile.py b/pyo3-ffi/examples/string-sum/noxfile.py similarity index 100% rename from examples/string-sum/noxfile.py rename to pyo3-ffi/examples/string-sum/noxfile.py diff --git a/examples/string-sum/pyproject.toml b/pyo3-ffi/examples/string-sum/pyproject.toml similarity index 100% rename from examples/string-sum/pyproject.toml rename to pyo3-ffi/examples/string-sum/pyproject.toml diff --git a/examples/string-sum/src/lib.rs b/pyo3-ffi/examples/string-sum/src/lib.rs similarity index 71% rename from examples/string-sum/src/lib.rs rename to pyo3-ffi/examples/string-sum/src/lib.rs index 91072418038..a0e53765481 100644 --- a/examples/string-sum/src/lib.rs +++ b/pyo3-ffi/examples/string-sum/src/lib.rs @@ -1,42 +1,55 @@ -use std::os::raw::{c_char, c_long}; +use std::ffi::{c_char, c_long}; use std::ptr; use pyo3_ffi::*; static mut MODULE_DEF: PyModuleDef = PyModuleDef { m_base: PyModuleDef_HEAD_INIT, - m_name: "string_sum\0".as_ptr().cast::(), - m_doc: "A Python module written in Rust.\0" - .as_ptr() - .cast::(), + m_name: c"string_sum".as_ptr(), + m_doc: c"A Python module written in Rust.".as_ptr(), m_size: 0, - m_methods: unsafe { METHODS as *const [PyMethodDef] as *mut PyMethodDef }, - m_slots: std::ptr::null_mut(), + m_methods: std::ptr::addr_of_mut!(METHODS).cast(), + m_slots: unsafe { SLOTS as *const [PyModuleDef_Slot] as *mut PyModuleDef_Slot }, m_traverse: None, m_clear: None, m_free: None, }; -static mut METHODS: &[PyMethodDef] = &[ +static mut METHODS: [PyMethodDef; 2] = [ PyMethodDef { - ml_name: "sum_as_string\0".as_ptr().cast::(), + ml_name: c"sum_as_string".as_ptr(), ml_meth: PyMethodDefPointer { - _PyCFunctionFast: sum_as_string, + PyCFunctionFast: sum_as_string, }, ml_flags: METH_FASTCALL, - ml_doc: "returns the sum of two integers as a string\0" - .as_ptr() - .cast::(), + ml_doc: c"returns the sum of two integers as a string".as_ptr(), }, // A zeroed PyMethodDef to mark the end of the array. PyMethodDef::zeroed(), ]; +static mut SLOTS: &[PyModuleDef_Slot] = &[ + #[cfg(Py_3_12)] + PyModuleDef_Slot { + slot: Py_mod_multiple_interpreters, + value: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED, + }, + #[cfg(Py_GIL_DISABLED)] + PyModuleDef_Slot { + slot: Py_mod_gil, + value: Py_MOD_GIL_NOT_USED, + }, + PyModuleDef_Slot { + slot: 0, + value: ptr::null_mut(), + }, +]; + // The module initialization function, which must be named `PyInit_`. #[allow(non_snake_case)] #[no_mangle] pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { - PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)) + PyModuleDef_Init(ptr::addr_of_mut!(MODULE_DEF)) } /// A helper to parse function arguments @@ -56,6 +69,7 @@ unsafe fn parse_arg_as_i32(obj: *mut PyObject, n_arg: usize) -> Option { let mut overflow = 0; let i_long: c_long = PyLong_AsLongAndOverflow(obj, &mut overflow); + #[allow(irrefutable_let_patterns)] // some platforms have c_long equal to i32 if overflow != 0 { raise_overflowerror(obj); None @@ -93,9 +107,7 @@ pub unsafe extern "C" fn sum_as_string( if nargs != 2 { PyErr_SetString( PyExc_TypeError, - "sum_as_string expected 2 positional arguments\0" - .as_ptr() - .cast::(), + c"sum_as_string expected 2 positional arguments".as_ptr(), ); return std::ptr::null_mut(); } @@ -117,10 +129,7 @@ pub unsafe extern "C" fn sum_as_string( PyUnicode_FromStringAndSize(string.as_ptr().cast::(), string.len() as isize) } None => { - PyErr_SetString( - PyExc_OverflowError, - "arguments too large to add\0".as_ptr().cast::(), - ); + PyErr_SetString(PyExc_OverflowError, c"arguments too large to add".as_ptr()); std::ptr::null_mut() } } diff --git a/examples/string-sum/tests/test_.py b/pyo3-ffi/examples/string-sum/tests/test_.py similarity index 100% rename from examples/string-sum/tests/test_.py rename to pyo3-ffi/examples/string-sum/tests/test_.py diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs index 0b3b7dbb3c2..712a0739b43 100644 --- a/pyo3-ffi/src/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -1,23 +1,25 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int}; -use std::ptr; - -extern "C" { - #[cfg(PyPy)] - #[link_name = "PyPyObject_DelAttrString"] - pub fn PyObject_DelAttrString(o: *mut PyObject, attr_name: *const c_char) -> c_int; -} +#[cfg(any(Py_3_12, all(Py_3_8, not(Py_LIMITED_API))))] +use libc::size_t; +use std::ffi::{c_char, c_int}; #[inline] -#[cfg(not(PyPy))] +#[cfg(all( + not(Py_3_13), // CPython exposed as a function in 3.13, in object.h + not(all(PyPy, not(Py_3_11))) // PyPy exposed as a function until PyPy 3.10, macro in 3.11+ +))] pub unsafe fn PyObject_DelAttrString(o: *mut PyObject, attr_name: *const c_char) -> c_int { - PyObject_SetAttrString(o, attr_name, ptr::null_mut()) + PyObject_SetAttrString(o, attr_name, std::ptr::null_mut()) } #[inline] +#[cfg(all( + not(Py_3_13), // CPython exposed as a function in 3.13, in object.h + not(all(PyPy, not(Py_3_11))) // PyPy exposed as a function until PyPy 3.10, macro in 3.11+ +))] pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_int { - PyObject_SetAttr(o, attr_name, ptr::null_mut()) + PyObject_SetAttr(o, attr_name, std::ptr::null_mut()) } extern "C" { @@ -76,6 +78,28 @@ extern "C" { method: *mut PyObject, ... ) -> *mut PyObject; +} +#[cfg(any(Py_3_12, all(Py_3_8, not(Py_LIMITED_API))))] +pub const PY_VECTORCALL_ARGUMENTS_OFFSET: size_t = + 1 << (8 * std::mem::size_of::() as size_t - 1); + +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyObject_Vectorcall")] + #[cfg(any(Py_3_12, all(Py_3_11, not(Py_LIMITED_API))))] + pub fn PyObject_Vectorcall( + callable: *mut PyObject, + args: *const *mut PyObject, + nargsf: size_t, + kwnames: *mut PyObject, + ) -> *mut PyObject; + + #[cfg(any(Py_3_12, all(Py_3_9, not(any(Py_LIMITED_API, PyPy)))))] + pub fn PyObject_VectorcallMethod( + name: *mut PyObject, + args: *const *mut PyObject, + nargsf: size_t, + kwnames: *mut PyObject, + ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_Type")] pub fn PyObject_Type(o: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_Size")] @@ -113,10 +137,7 @@ extern "C" { #[cfg(not(any(Py_3_8, PyPy)))] #[inline] pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int { - crate::PyObject_HasAttrString( - crate::Py_TYPE(o).cast(), - "__next__\0".as_ptr() as *const c_char, - ) + crate::PyObject_HasAttrString(crate::Py_TYPE(o).cast(), c"__next__".as_ptr()) } extern "C" { @@ -128,7 +149,11 @@ extern "C" { pub fn PyIter_Next(arg1: *mut PyObject) -> *mut PyObject; #[cfg(all(not(PyPy), Py_3_10))] #[cfg_attr(PyPy, link_name = "PyPyIter_Send")] - pub fn PyIter_Send(iter: *mut PyObject, arg: *mut PyObject, presult: *mut *mut PyObject); + pub fn PyIter_Send( + iter: *mut PyObject, + arg: *mut PyObject, + presult: *mut *mut PyObject, + ) -> PySendResult; #[cfg_attr(PyPy, link_name = "PyPyNumber_Check")] pub fn PyNumber_Check(o: *mut PyObject) -> c_int; diff --git a/pyo3-ffi/src/boolobject.rs b/pyo3-ffi/src/boolobject.rs index 78972ff0835..4d2ce70a600 100644 --- a/pyo3-ffi/src/boolobject.rs +++ b/pyo3-ffi/src/boolobject.rs @@ -1,14 +1,9 @@ +#[cfg(not(GraalPy))] use crate::longobject::PyLongObject; use crate::object::*; -use std::os::raw::{c_int, c_long}; +use std::ffi::{c_int, c_long}; use std::ptr::addr_of_mut; -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - #[cfg_attr(PyPy, link_name = "PyPyBool_Type")] - pub static mut PyBool_Type: PyTypeObject; -} - #[inline] pub unsafe fn PyBool_Check(op: *mut PyObject) -> c_int { (Py_TYPE(op) == addr_of_mut!(PyBool_Type)) as c_int @@ -16,20 +11,33 @@ pub unsafe fn PyBool_Check(op: *mut PyObject) -> c_int { #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "_PyPy_FalseStruct")] static mut _Py_FalseStruct: PyLongObject; + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "_PyPy_TrueStruct")] static mut _Py_TrueStruct: PyLongObject; + + #[cfg(GraalPy)] + static mut _Py_FalseStructReference: *mut PyObject; + #[cfg(GraalPy)] + static mut _Py_TrueStructReference: *mut PyObject; } #[inline] pub unsafe fn Py_False() -> *mut PyObject { - addr_of_mut!(_Py_FalseStruct) as *mut PyObject + #[cfg(not(GraalPy))] + return addr_of_mut!(_Py_FalseStruct) as *mut PyObject; + #[cfg(GraalPy)] + return _Py_FalseStructReference; } #[inline] pub unsafe fn Py_True() -> *mut PyObject { - addr_of_mut!(_Py_TrueStruct) as *mut PyObject + #[cfg(not(GraalPy))] + return addr_of_mut!(_Py_TrueStruct) as *mut PyObject; + #[cfg(GraalPy)] + return _Py_TrueStructReference; } #[inline] diff --git a/pyo3-ffi/src/bytearrayobject.rs b/pyo3-ffi/src/bytearrayobject.rs index a37deb410f7..6729f6167c6 100644 --- a/pyo3-ffi/src/bytearrayobject.rs +++ b/pyo3-ffi/src/bytearrayobject.rs @@ -1,11 +1,10 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; use std::ptr::addr_of_mut; -#[cfg(not(any(PyPy, Py_LIMITED_API)))] +#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyByteArrayObject { pub ob_base: PyVarObject, pub ob_alloc: Py_ssize_t, @@ -17,8 +16,8 @@ pub struct PyByteArrayObject { pub ob_exports: c_int, } -#[cfg(any(PyPy, Py_LIMITED_API))] -opaque_struct!(PyByteArrayObject); +#[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] +opaque_struct!(pub PyByteArrayObject); #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/pyo3-ffi/src/bytesobject.rs b/pyo3-ffi/src/bytesobject.rs index 0fd327b6fca..08351d18daa 100644 --- a/pyo3-ffi/src/bytesobject.rs +++ b/pyo3-ffi/src/bytesobject.rs @@ -1,6 +1,6 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] diff --git a/pyo3-ffi/src/ceval.rs b/pyo3-ffi/src/ceval.rs index 7aae25f8c3e..5746d574346 100644 --- a/pyo3-ffi/src/ceval.rs +++ b/pyo3-ffi/src/ceval.rs @@ -1,6 +1,6 @@ use crate::object::PyObject; use crate::pystate::PyThreadState; -use std::os::raw::{c_char, c_int, c_void}; +use std::ffi::{c_char, c_int, c_void}; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyEval_EvalCode")] @@ -24,6 +24,7 @@ extern "C" { closure: *mut PyObject, ) -> *mut PyObject; + #[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] #[cfg_attr(PyPy, link_name = "PyPyEval_CallObjectWithKeywords")] pub fn PyEval_CallObjectWithKeywords( @@ -33,6 +34,7 @@ extern "C" { ) -> *mut PyObject; } +#[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] #[inline] pub unsafe fn PyEval_CallObject(func: *mut PyObject, arg: *mut PyObject) -> *mut PyObject { @@ -41,9 +43,11 @@ pub unsafe fn PyEval_CallObject(func: *mut PyObject, arg: *mut PyObject) -> *mut } extern "C" { + #[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] #[cfg_attr(PyPy, link_name = "PyPyEval_CallFunction")] pub fn PyEval_CallFunction(obj: *mut PyObject, format: *const c_char, ...) -> *mut PyObject; + #[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] #[cfg_attr(PyPy, link_name = "PyPyEval_CallMethod")] pub fn PyEval_CallMethod( @@ -95,9 +99,22 @@ extern "C" { } extern "C" { + #[cfg(not(Py_3_13))] #[cfg_attr(PyPy, link_name = "PyPyEval_ThreadsInitialized")] + #[cfg_attr( + Py_3_9, + deprecated( + note = "Deprecated in Python 3.9, this function always returns true in Python 3.7 or newer." + ) + )] pub fn PyEval_ThreadsInitialized() -> c_int; #[cfg_attr(PyPy, link_name = "PyPyEval_InitThreads")] + #[cfg_attr( + Py_3_9, + deprecated( + note = "Deprecated in Python 3.9, this function does nothing in Python 3.7 or newer." + ) + )] pub fn PyEval_InitThreads(); pub fn PyEval_AcquireLock(); pub fn PyEval_ReleaseLock(); diff --git a/pyo3-ffi/src/code.rs b/pyo3-ffi/src/code.rs index d28f68cded7..296b17f6aa4 100644 --- a/pyo3-ffi/src/code.rs +++ b/pyo3-ffi/src/code.rs @@ -1,4 +1,4 @@ // This header doesn't exist in CPython, but Include/cpython/code.h does. We add // this here so that PyCodeObject has a definition under the limited API. -opaque_struct!(PyCodeObject); +opaque_struct!(pub PyCodeObject); diff --git a/pyo3-ffi/src/codecs.rs b/pyo3-ffi/src/codecs.rs index 2fd214cbbfe..9e4c52c9348 100644 --- a/pyo3-ffi/src/codecs.rs +++ b/pyo3-ffi/src/codecs.rs @@ -1,5 +1,5 @@ use crate::object::PyObject; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; extern "C" { pub fn PyCodec_Register(search_function: *mut PyObject) -> c_int; diff --git a/pyo3-ffi/src/compat/mod.rs b/pyo3-ffi/src/compat/mod.rs new file mode 100644 index 00000000000..044ea46762b --- /dev/null +++ b/pyo3-ffi/src/compat/mod.rs @@ -0,0 +1,61 @@ +//! C API Compatibility Shims +//! +//! Some CPython C API functions added in recent versions of Python are +//! inherently safer to use than older C API constructs. This module +//! exposes functions available on all Python versions that wrap the +//! old C API on old Python versions and wrap the function directly +//! on newer Python versions. + +// Unless otherwise noted, the compatibility shims are adapted from +// the pythoncapi-compat project: https://github.com/python/pythoncapi-compat + +/// Internal helper macro which defines compatibility shims for C API functions, deferring to a +/// re-export when that's available. +macro_rules! compat_function { + ( + originally_defined_for($cfg:meta); + + $(#[$attrs:meta])* + pub unsafe fn $name:ident($($arg_names:ident: $arg_types:ty),* $(,)?) -> $ret:ty $body:block + ) => { + // Define as a standalone function under docsrs cfg so that this shows as a unique function in the docs, + // not a re-export (the re-export has the wrong visibility) + #[cfg(any(docsrs, not($cfg)))] + #[cfg_attr(docsrs, doc(cfg(all())))] + $(#[$attrs])* + pub unsafe fn $name( + $($arg_names: $arg_types,)* + ) -> $ret $body + + #[cfg(all($cfg, not(docsrs)))] + pub use $crate::$name; + + #[cfg(test)] + paste::paste! { + // Test that the compat function does not overlap with the original function. If the + // cfgs line up, then the the two glob imports will resolve to the same item via the + // re-export. If the cfgs mismatch, then the use of $name will be ambiguous in cases + // where the function is defined twice, and the test will fail to compile. + #[allow(unused_imports)] + mod [] { + use $crate::*; + use $crate::compat::*; + + #[test] + fn test_export() { + let _ = $name; + } + } + } + }; +} + +mod py_3_10; +mod py_3_13; +mod py_3_14; +mod py_3_9; + +pub use self::py_3_10::*; +pub use self::py_3_13::*; +pub use self::py_3_14::*; +pub use self::py_3_9::*; diff --git a/pyo3-ffi/src/compat/py_3_10.rs b/pyo3-ffi/src/compat/py_3_10.rs new file mode 100644 index 00000000000..2d4daa194c6 --- /dev/null +++ b/pyo3-ffi/src/compat/py_3_10.rs @@ -0,0 +1,45 @@ +compat_function!( + originally_defined_for(Py_3_10); + + #[inline] + pub unsafe fn Py_NewRef(obj: *mut crate::PyObject) -> *mut crate::PyObject { + crate::Py_INCREF(obj); + obj + } +); + +compat_function!( + originally_defined_for(Py_3_10); + + #[inline] + pub unsafe fn Py_XNewRef(obj: *mut crate::PyObject) -> *mut crate::PyObject { + crate::Py_XINCREF(obj); + obj + } +); + +compat_function!( + originally_defined_for(Py_3_10); + + #[inline] + pub unsafe fn PyModule_AddObjectRef( + module: *mut crate::PyObject, + name: *const std::ffi::c_char, + value: *mut crate::PyObject, + ) -> std::ffi::c_int { + if value.is_null() && crate::PyErr_Occurred().is_null() { + crate::PyErr_SetString( + crate::PyExc_SystemError, + c"PyModule_AddObjectRef() must be called with an exception raised if value is NULL".as_ptr(), + ); + return -1; + } + + crate::Py_XINCREF(value); + let result = crate::PyModule_AddObject(module, name, value); + if result < 0 { + crate::Py_XDECREF(value); + } + result + } +); diff --git a/pyo3-ffi/src/compat/py_3_13.rs b/pyo3-ffi/src/compat/py_3_13.rs new file mode 100644 index 00000000000..a433f7ac334 --- /dev/null +++ b/pyo3-ffi/src/compat/py_3_13.rs @@ -0,0 +1,121 @@ +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyDict_GetItemRef( + dp: *mut crate::PyObject, + key: *mut crate::PyObject, + result: *mut *mut crate::PyObject, + ) -> std::ffi::c_int { + use crate::{compat::Py_NewRef, PyDict_GetItemWithError, PyErr_Occurred}; + + let item = PyDict_GetItemWithError(dp, key); + if !item.is_null() { + *result = Py_NewRef(item); + return 1; // found + } + *result = std::ptr::null_mut(); + if PyErr_Occurred().is_null() { + return 0; // not found + } + -1 + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_GetItemRef( + arg1: *mut crate::PyObject, + arg2: crate::Py_ssize_t, + ) -> *mut crate::PyObject { + use crate::{PyList_GetItem, Py_XINCREF}; + + let item = PyList_GetItem(arg1, arg2); + Py_XINCREF(item); + item + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyImport_AddModuleRef( + name: *const std::ffi::c_char, + ) -> *mut crate::PyObject { + use crate::{compat::Py_XNewRef, PyImport_AddModule}; + + Py_XNewRef(PyImport_AddModule(name)) + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyWeakref_GetRef( + reference: *mut crate::PyObject, + pobj: *mut *mut crate::PyObject, + ) -> std::ffi::c_int { + use crate::{ + compat::Py_NewRef, PyErr_SetString, PyExc_TypeError, PyWeakref_Check, + PyWeakref_GetObject, Py_None, + }; + + if !reference.is_null() && PyWeakref_Check(reference) == 0 { + *pobj = std::ptr::null_mut(); + PyErr_SetString(PyExc_TypeError, c"expected a weakref".as_ptr()); + return -1; + } + let obj = PyWeakref_GetObject(reference); + if obj.is_null() { + // SystemError if reference is NULL + *pobj = std::ptr::null_mut(); + return -1; + } + if obj == Py_None() { + *pobj = std::ptr::null_mut(); + return 0; + } + *pobj = Py_NewRef(obj); + 1 + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_Extend( + list: *mut crate::PyObject, + iterable: *mut crate::PyObject, + ) -> std::ffi::c_int { + crate::PyList_SetSlice(list, crate::PY_SSIZE_T_MAX, crate::PY_SSIZE_T_MAX, iterable) + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_Clear(list: *mut crate::PyObject) -> std::ffi::c_int { + crate::PyList_SetSlice(list, 0, crate::PY_SSIZE_T_MAX, std::ptr::null_mut()) + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyModule_Add( + module: *mut crate::PyObject, + name: *const std::ffi::c_char, + value: *mut crate::PyObject, + ) -> std::ffi::c_int { + let result = crate::compat::PyModule_AddObjectRef(module, name, value); + crate::Py_XDECREF(value); + result + } +); diff --git a/pyo3-ffi/src/compat/py_3_14.rs b/pyo3-ffi/src/compat/py_3_14.rs new file mode 100644 index 00000000000..859d26ea777 --- /dev/null +++ b/pyo3-ffi/src/compat/py_3_14.rs @@ -0,0 +1,26 @@ +compat_function!( + originally_defined_for(all(Py_3_14, not(Py_LIMITED_API))); + + #[inline] + pub unsafe fn Py_HashBuffer( + ptr: *const std::ffi::c_void, + len: crate::Py_ssize_t, + ) -> crate::Py_hash_t { + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + { + crate::_Py_HashBytes(ptr, len) + } + + #[cfg(any(Py_LIMITED_API, PyPy))] + { + let bytes = crate::PyBytes_FromStringAndSize(ptr as *const std::ffi::c_char, len); + if bytes.is_null() { + -1 + } else { + let result = crate::PyObject_Hash(bytes); + crate::Py_DECREF(bytes); + result + } + } + } +); diff --git a/pyo3-ffi/src/compat/py_3_9.rs b/pyo3-ffi/src/compat/py_3_9.rs new file mode 100644 index 00000000000..6b3521cc167 --- /dev/null +++ b/pyo3-ffi/src/compat/py_3_9.rs @@ -0,0 +1,20 @@ +compat_function!( + originally_defined_for(all( + not(PyPy), + any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10 + )); + + #[inline] + pub unsafe fn PyObject_CallNoArgs(obj: *mut crate::PyObject) -> *mut crate::PyObject { + crate::PyObject_CallObject(obj, std::ptr::null_mut()) + } +); + +compat_function!( + originally_defined_for(all(Py_3_9, not(any(Py_LIMITED_API, PyPy)))); + + #[inline] + pub unsafe fn PyObject_CallMethodNoArgs(obj: *mut crate::PyObject, name: *mut crate::PyObject) -> *mut crate::PyObject { + crate::PyObject_CallMethodObjArgs(obj, name, std::ptr::null_mut::()) + } +); diff --git a/pyo3-ffi/src/compile.rs b/pyo3-ffi/src/compile.rs index 189a1a8b8a4..50128a815d0 100644 --- a/pyo3-ffi/src/compile.rs +++ b/pyo3-ffi/src/compile.rs @@ -1,4 +1,4 @@ -use std::os::raw::c_int; +use std::ffi::c_int; pub const Py_single_input: c_int = 256; pub const Py_file_input: c_int = 257; diff --git a/pyo3-ffi/src/complexobject.rs b/pyo3-ffi/src/complexobject.rs index 339f5d8c81a..a8920765424 100644 --- a/pyo3-ffi/src/complexobject.rs +++ b/pyo3-ffi/src/complexobject.rs @@ -1,38 +1,7 @@ use crate::object::*; -use std::os::raw::{c_double, c_int}; +use std::ffi::{c_double, c_int}; use std::ptr::addr_of_mut; -#[repr(C)] -#[derive(Copy, Clone)] -// non-limited -pub struct Py_complex { - pub real: c_double, - pub imag: c_double, -} - -#[cfg(not(Py_LIMITED_API))] -extern "C" { - pub fn _Py_c_sum(left: Py_complex, right: Py_complex) -> Py_complex; - pub fn _Py_c_diff(left: Py_complex, right: Py_complex) -> Py_complex; - pub fn _Py_c_neg(complex: Py_complex) -> Py_complex; - pub fn _Py_c_prod(left: Py_complex, right: Py_complex) -> Py_complex; - pub fn _Py_c_quot(dividend: Py_complex, divisor: Py_complex) -> Py_complex; - pub fn _Py_c_pow(num: Py_complex, exp: Py_complex) -> Py_complex; - pub fn _Py_c_abs(arg: Py_complex) -> c_double; - #[cfg_attr(PyPy, link_name = "PyPyComplex_FromCComplex")] - pub fn PyComplex_FromCComplex(v: Py_complex) -> *mut PyObject; - #[cfg_attr(PyPy, link_name = "PyPyComplex_AsCComplex")] - pub fn PyComplex_AsCComplex(op: *mut PyObject) -> Py_complex; -} - -#[repr(C)] -#[derive(Copy, Clone)] -// non-limited -pub struct PyComplexObject { - pub ob_base: PyObject, - pub cval: Py_complex, -} - #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg_attr(PyPy, link_name = "PyPyComplex_Type")] @@ -46,17 +15,16 @@ pub unsafe fn PyComplex_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyComplex_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == addr_of_mut!(PyComplex_Type)) as c_int + Py_IS_TYPE(op, addr_of_mut!(PyComplex_Type)) } extern "C" { // skipped non-limited PyComplex_FromCComplex #[cfg_attr(PyPy, link_name = "PyPyComplex_FromDoubles")] pub fn PyComplex_FromDoubles(real: c_double, imag: c_double) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyComplex_RealAsDouble")] pub fn PyComplex_RealAsDouble(op: *mut PyObject) -> c_double; #[cfg_attr(PyPy, link_name = "PyPyComplex_ImagAsDouble")] pub fn PyComplex_ImagAsDouble(op: *mut PyObject) -> c_double; - // skipped non-limited PyComplex_AsCComplex - // skipped non-limited _PyComplex_FormatAdvancedWriter } diff --git a/pyo3-ffi/src/context.rs b/pyo3-ffi/src/context.rs index 210d6e21f04..8f2f1576bb1 100644 --- a/pyo3-ffi/src/context.rs +++ b/pyo3-ffi/src/context.rs @@ -1,5 +1,5 @@ use crate::object::{PyObject, PyTypeObject, Py_TYPE}; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; use std::ptr::addr_of_mut; extern "C" { diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index d2e3ca9d67a..53ef71e0710 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -1,11 +1,11 @@ use crate::{PyObject, Py_ssize_t}; -use std::os::raw::{c_char, c_int}; +#[cfg(any(all(Py_3_8, not(PyPy)), not(Py_3_11)))] +use std::ffi::c_char; +use std::ffi::c_int; #[cfg(not(Py_3_11))] use crate::Py_buffer; -#[cfg(Py_3_8)] -use crate::pyport::PY_SSIZE_T_MAX; #[cfg(all(Py_3_8, not(PyPy)))] use crate::{ vectorcallfunc, PyCallable_Check, PyThreadState, PyThreadState_GET, PyTuple_Check, @@ -15,11 +15,11 @@ use crate::{ use libc::size_t; extern "C" { - #[cfg(all(Py_3_8, not(PyPy)))] + #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] pub fn _PyStack_AsDict(values: *const *mut PyObject, kwnames: *mut PyObject) -> *mut PyObject; } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] const _PY_FASTCALL_SMALL_STACK: size_t = 5; extern "C" { @@ -41,15 +41,15 @@ extern "C" { ) -> *mut PyObject; } -#[cfg(Py_3_8)] -const PY_VECTORCALL_ARGUMENTS_OFFSET: Py_ssize_t = - 1 << (8 * std::mem::size_of::() as Py_ssize_t - 1); +#[cfg(Py_3_8)] // NB exported as public in abstract.rs from 3.12 +const PY_VECTORCALL_ARGUMENTS_OFFSET: size_t = + 1 << (8 * std::mem::size_of::() as size_t - 1); #[cfg(Py_3_8)] #[inline(always)] pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t { - assert!(n <= (PY_SSIZE_T_MAX as size_t)); - (n as Py_ssize_t) & !PY_VECTORCALL_ARGUMENTS_OFFSET + let n = n & !PY_VECTORCALL_ARGUMENTS_OFFSET; + n.try_into().expect("cannot fail due to mask") } #[cfg(all(Py_3_8, not(PyPy)))] @@ -63,7 +63,7 @@ pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option 0); let offset = (*tp).tp_vectorcall_offset; assert!(offset > 0); - let ptr = (callable as *const c_char).offset(offset) as *const Option; + let ptr = callable.cast::().offset(offset).cast(); *ptr } @@ -91,7 +91,7 @@ pub unsafe fn _PyObject_VectorcallTstate( } } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy, Py_3_11))))] // exported as a function from 3.11, see abstract.rs #[inline(always)] pub unsafe fn PyObject_Vectorcall( callable: *mut PyObject, @@ -103,18 +103,11 @@ pub unsafe fn PyObject_Vectorcall( } extern "C" { - #[cfg(all(PyPy, Py_3_8))] - #[cfg_attr(not(Py_3_9), link_name = "_PyPyObject_Vectorcall")] - #[cfg_attr(Py_3_9, link_name = "PyPyObject_Vectorcall")] - pub fn PyObject_Vectorcall( - callable: *mut PyObject, - args: *const *mut PyObject, - nargsf: size_t, - kwnames: *mut PyObject, - ) -> *mut PyObject; - #[cfg(Py_3_8)] - #[cfg_attr(all(not(PyPy), not(Py_3_9)), link_name = "_PyObject_VectorcallDict")] + #[cfg_attr( + all(not(any(PyPy, GraalPy)), not(Py_3_9)), + link_name = "_PyObject_VectorcallDict" + )] #[cfg_attr(all(PyPy, not(Py_3_9)), link_name = "_PyPyObject_VectorcallDict")] #[cfg_attr(all(PyPy, Py_3_9), link_name = "PyPyObject_VectorcallDict")] pub fn PyObject_VectorcallDict( @@ -134,7 +127,7 @@ extern "C" { ) -> *mut PyObject; } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] #[inline(always)] pub unsafe fn _PyObject_FastCallTstate( tstate: *mut PyThreadState, @@ -145,7 +138,7 @@ pub unsafe fn _PyObject_FastCallTstate( _PyObject_VectorcallTstate(tstate, func, args, nargs as size_t, std::ptr::null_mut()) } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] #[inline(always)] pub unsafe fn _PyObject_FastCall( func: *mut PyObject, @@ -181,17 +174,7 @@ pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *m let args = args_array.as_ptr().offset(1); // For PY_VECTORCALL_ARGUMENTS_OFFSET let tstate = PyThreadState_GET(); let nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - _PyObject_VectorcallTstate(tstate, func, args, nargsf as size_t, std::ptr::null_mut()) -} - -extern "C" { - #[cfg(all(Py_3_9, not(PyPy)))] - pub fn PyObject_VectorcallMethod( - name: *mut PyObject, - args: *const *mut PyObject, - nargsf: size_t, - kwnames: *mut PyObject, - ) -> *mut PyObject; + _PyObject_VectorcallTstate(tstate, func, args, nargsf, std::ptr::null_mut()) } #[cfg(all(Py_3_9, not(PyPy)))] @@ -200,10 +183,10 @@ pub unsafe fn PyObject_CallMethodNoArgs( self_: *mut PyObject, name: *mut PyObject, ) -> *mut PyObject { - PyObject_VectorcallMethod( + crate::PyObject_VectorcallMethod( name, &self_, - 1 | PY_VECTORCALL_ARGUMENTS_OFFSET as size_t, + 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, std::ptr::null_mut(), ) } @@ -217,10 +200,10 @@ pub unsafe fn PyObject_CallMethodOneArg( ) -> *mut PyObject { let args = [self_, arg]; assert!(!arg.is_null()); - PyObject_VectorcallMethod( + crate::PyObject_VectorcallMethod( name, args.as_ptr(), - 2 | PY_VECTORCALL_ARGUMENTS_OFFSET as size_t, + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, std::ptr::null_mut(), ) } @@ -255,12 +238,12 @@ extern "C" { pub fn PyBuffer_GetPointer( view: *mut Py_buffer, indices: *mut Py_ssize_t, - ) -> *mut std::os::raw::c_void; + ) -> *mut std::ffi::c_void; #[cfg_attr(PyPy, link_name = "PyPyBuffer_SizeFromFormat")] pub fn PyBuffer_SizeFromFormat(format: *const c_char) -> Py_ssize_t; #[cfg_attr(PyPy, link_name = "PyPyBuffer_ToContiguous")] pub fn PyBuffer_ToContiguous( - buf: *mut std::os::raw::c_void, + buf: *mut std::ffi::c_void, view: *mut Py_buffer, len: Py_ssize_t, order: c_char, @@ -268,7 +251,7 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyBuffer_FromContiguous")] pub fn PyBuffer_FromContiguous( view: *mut Py_buffer, - buf: *mut std::os::raw::c_void, + buf: *mut std::ffi::c_void, len: Py_ssize_t, order: c_char, ) -> c_int; @@ -286,7 +269,7 @@ extern "C" { pub fn PyBuffer_FillInfo( view: *mut Py_buffer, o: *mut PyObject, - buf: *mut std::os::raw::c_void, + buf: *mut std::ffi::c_void, len: Py_ssize_t, readonly: c_int, flags: c_int, @@ -308,7 +291,7 @@ pub const PY_ITERSEARCH_INDEX: c_int = 2; pub const PY_ITERSEARCH_CONTAINS: c_int = 3; extern "C" { - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn _PySequence_IterSearch( seq: *mut PyObject, obj: *mut PyObject, diff --git a/pyo3-ffi/src/cpython/bytesobject.rs b/pyo3-ffi/src/cpython/bytesobject.rs index 912fc0ac427..56344d35dd9 100644 --- a/pyo3-ffi/src/cpython/bytesobject.rs +++ b/pyo3-ffi/src/cpython/bytesobject.rs @@ -1,22 +1,34 @@ use crate::object::*; use crate::Py_ssize_t; -#[cfg(not(any(PyPy, Py_LIMITED_API)))] -use std::os::raw::c_char; -use std::os::raw::c_int; +#[cfg(not(Py_LIMITED_API))] +use std::ffi::c_char; +use std::ffi::c_int; -#[cfg(not(any(PyPy, Py_LIMITED_API)))] +#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyBytesObject { pub ob_base: PyVarObject, + #[cfg_attr( + Py_3_11, + deprecated(note = "Deprecated in Python 3.11 and will be removed in a future version.") + )] pub ob_shash: crate::Py_hash_t, pub ob_sval: [c_char; 1], } -#[cfg(any(PyPy, Py_LIMITED_API))] -opaque_struct!(PyBytesObject); +#[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] +opaque_struct!(pub PyBytesObject); extern "C" { #[cfg_attr(PyPy, link_name = "_PyPyBytes_Resize")] pub fn _PyBytes_Resize(bytes: *mut *mut PyObject, newsize: Py_ssize_t) -> c_int; } + +#[cfg(not(Py_LIMITED_API))] +#[inline] +pub unsafe fn PyBytes_AS_STRING(op: *mut PyObject) -> *const c_char { + #[cfg(not(any(PyPy, GraalPy)))] + return &(*op.cast::()).ob_sval as *const c_char; + #[cfg(any(PyPy, GraalPy))] + return crate::PyBytes_AsString(op); +} diff --git a/pyo3-ffi/src/cpython/ceval.rs b/pyo3-ffi/src/cpython/ceval.rs index 6df10627d2e..cbec47195ec 100644 --- a/pyo3-ffi/src/cpython/ceval.rs +++ b/pyo3-ffi/src/cpython/ceval.rs @@ -1,6 +1,6 @@ use crate::cpython::pystate::Py_tracefunc; use crate::object::{freefunc, PyObject}; -use std::os::raw::c_int; +use std::ffi::c_int; extern "C" { // skipped non-limited _PyEval_CallTracing diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index 05f21a137b5..aa02a80adb9 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -1,202 +1,43 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -#[allow(unused_imports)] -use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; +#[cfg(not(GraalPy))] +use std::ffi::c_char; +use std::ffi::{c_int, c_void}; #[cfg(not(PyPy))] use std::ptr::addr_of_mut; -#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] -opaque_struct!(_PyOpcache); +// skipped private _PY_MONITORING_LOCAL_EVENTS +// skipped private _PY_MONITORING_UNGROUPED_EVENTS +// skipped private _PY_MONITORING_EVENTS -#[cfg(Py_3_12)] -pub const _PY_MONITORING_LOCAL_EVENTS: usize = 10; -#[cfg(Py_3_12)] -pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 15; -#[cfg(Py_3_12)] -pub const _PY_MONITORING_EVENTS: usize = 17; +// skipped private _PyLocalMonitors +// skipped private _Py_GlobalMonitors -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct _Py_LocalMonitors { - pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], -} - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct _Py_GlobalMonitors { - pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], -} - -// skipped _Py_CODEUNIT +// skipped private _Py_CODEUNIT -// skipped _Py_OPCODE -// skipped _Py_OPARG +// skipped private _Py_OPCODE +// skipped private _Py_OPARG -// skipped _py_make_codeunit +// skipped private _py_make_codeunit -// skipped _py_set_opcode +// skipped private _py_set_opcode -// skipped _Py_MAKE_CODEUNIT -// skipped _Py_SET_OPCODE - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoCached { - pub _co_code: *mut PyObject, - pub _co_varnames: *mut PyObject, - pub _co_cellvars: *mut PyObject, - pub _co_freevars: *mut PyObject, -} - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoLineInstrumentationData { - pub original_opcode: u8, - pub line_delta: i8, -} - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoMonitoringData { - pub local_monitors: _Py_LocalMonitors, - pub active_monitors: _Py_LocalMonitors, - pub tools: *mut u8, - pub lines: *mut _PyCoLineInstrumentationData, - pub line_tools: *mut u8, - pub per_instruction_opcodes: *mut u8, - pub per_instruction_tools: *mut u8, -} - -#[cfg(all(not(PyPy), not(Py_3_7)))] -opaque_struct!(PyCodeObject); - -#[cfg(all(not(PyPy), Py_3_7, not(Py_3_8)))] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_argcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_nlocals: c_int, - pub co_stacksize: c_int, - pub co_flags: c_int, - pub co_firstlineno: c_int, - pub co_code: *mut PyObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_varnames: *mut PyObject, - pub co_freevars: *mut PyObject, - pub co_cellvars: *mut PyObject, - pub co_cell2arg: *mut Py_ssize_t, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - pub co_lnotab: *mut PyObject, - pub co_zombieframe: *mut c_void, - pub co_weakreflist: *mut PyObject, - pub co_extra: *mut c_void, -} - -#[cfg(all(not(PyPy), Py_3_8, not(Py_3_11)))] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_argcount: c_int, - pub co_posonlyargcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_nlocals: c_int, - pub co_stacksize: c_int, - pub co_flags: c_int, - pub co_firstlineno: c_int, - pub co_code: *mut PyObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_varnames: *mut PyObject, - pub co_freevars: *mut PyObject, - pub co_cellvars: *mut PyObject, - pub co_cell2arg: *mut Py_ssize_t, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - #[cfg(not(Py_3_10))] - pub co_lnotab: *mut PyObject, - #[cfg(Py_3_10)] - pub co_linetable: *mut PyObject, - pub co_zombieframe: *mut c_void, - pub co_weakreflist: *mut PyObject, - pub co_extra: *mut c_void, - pub co_opcache_map: *mut c_uchar, - pub co_opcache: *mut _PyOpcache, - pub co_opcache_flag: c_int, - pub co_opcache_size: c_uchar, -} +// skipped private _Py_MAKE_CODEUNIT +// skipped private _Py_SET_OPCODE -#[cfg(all(not(PyPy), Py_3_11))] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct PyCodeObject { - pub ob_base: PyVarObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_exceptiontable: *mut PyObject, - pub co_flags: c_int, - #[cfg(not(Py_3_12))] - pub co_warmup: c_int, +// skipped private _PyCoCached +// skipped private _PyCoLineInstrumentationData +// skipped private _PyCoMontoringData - pub co_argcount: c_int, - pub co_posonlyargcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_stacksize: c_int, - pub co_firstlineno: c_int, +// skipped private _PyExecutorArray - pub co_nlocalsplus: c_int, - #[cfg(Py_3_12)] - pub co_framesize: c_int, - pub co_nlocals: c_int, - #[cfg(not(Py_3_12))] - pub co_nplaincellvars: c_int, - pub co_ncellvars: c_int, - pub co_nfreevars: c_int, - #[cfg(Py_3_12)] - pub co_version: u32, - - pub co_localsplusnames: *mut PyObject, - pub co_localspluskinds: *mut PyObject, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - pub co_qualname: *mut PyObject, - pub co_linetable: *mut PyObject, - pub co_weakreflist: *mut PyObject, - #[cfg(not(Py_3_12))] - pub _co_code: *mut PyObject, - #[cfg(not(Py_3_12))] - pub _co_linearray: *mut c_char, - #[cfg(Py_3_12)] - pub _co_cached: *mut _PyCoCached, - #[cfg(Py_3_12)] - pub _co_instrumentation_version: u64, - #[cfg(Py_3_12)] - pub _co_monitoring: *mut _PyCoMonitoringData, - pub _co_firsttraceable: c_int, - pub co_extra: *mut c_void, - pub co_code_adaptive: [c_char; 1], -} - -#[cfg(PyPy)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_name: *mut PyObject, - pub co_filename: *mut PyObject, - pub co_argcount: c_int, - pub co_flags: c_int, -} +opaque_struct!( + #[doc = "A Python code object.\n"] + #[doc = "\n"] + #[doc = "`pyo3-ffi` does not expose the contents of this struct, as it has no stability guarantees."] + pub PyCodeObject +); /* Masks for co_flags */ pub const CO_OPTIMIZED: c_int = 0x0001; @@ -230,6 +71,7 @@ pub const CO_FUTURE_GENERATOR_STOP: c_int = 0x8_0000; pub const CO_MAXBLOCKS: usize = 20; +#[cfg(not(PyPy))] #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { pub static mut PyCode_Type: PyTypeObject; @@ -241,29 +83,16 @@ pub unsafe fn PyCode_Check(op: *mut PyObject) -> c_int { (Py_TYPE(op) == addr_of_mut!(PyCode_Type)) as c_int } -#[inline] -#[cfg(all(not(PyPy), Py_3_10, not(Py_3_11)))] -pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t { - crate::PyTuple_GET_SIZE((*op).co_freevars) -} - -#[inline] -#[cfg(all(not(Py_3_10), Py_3_11, not(PyPy)))] -pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> c_int { - (*op).co_nfreevars -} - extern "C" { #[cfg(PyPy)] #[link_name = "PyPyCode_Check"] pub fn PyCode_Check(op: *mut PyObject) -> c_int; - - #[cfg(PyPy)] - #[link_name = "PyPyCode_GetNumFree"] - pub fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t; } +// skipped PyCode_GetNumFree (requires knowledge of code object layout) + extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyCode_New")] pub fn PyCode_New( argcount: c_int, @@ -282,6 +111,7 @@ extern "C" { firstlineno: c_int, lnotab: *mut PyObject, ) -> *mut PyCodeObject; + #[cfg(not(GraalPy))] #[cfg(Py_3_8)] pub fn PyCode_NewWithPosOnlyArgs( argcount: c_int, @@ -301,12 +131,14 @@ extern "C" { firstlineno: c_int, lnotab: *mut PyObject, ) -> *mut PyCodeObject; + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyCode_NewEmpty")] pub fn PyCode_NewEmpty( filename: *const c_char, funcname: *const c_char, firstlineno: c_int, ) -> *mut PyCodeObject; + #[cfg(not(GraalPy))] pub fn PyCode_Addr2Line(arg1: *mut PyCodeObject, arg2: c_int) -> c_int; // skipped PyCodeAddressRange "for internal use only" // skipped _PyCode_CheckLineNumber diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 71af81e83e5..df0b982b147 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -6,19 +6,21 @@ use crate::pyarena::*; use crate::pythonrun::*; #[cfg(not(any(PyPy, Py_3_10)))] use crate::PyCodeObject; +use crate::INT_MAX; #[cfg(not(any(PyPy, Py_3_10)))] -use std::os::raw::c_char; -use std::os::raw::c_int; - -// skipped non-limited PyCF_MASK -// skipped non-limited PyCF_MASK_OBSOLETE -// skipped non-limited PyCF_SOURCE_IS_UTF8 -// skipped non-limited PyCF_DONT_IMPLY_DEDENT -// skipped non-limited PyCF_ONLY_AST -// skipped non-limited PyCF_IGNORE_COOKIE -// skipped non-limited PyCF_TYPE_COMMENTS -// skipped non-limited PyCF_ALLOW_TOP_LEVEL_AWAIT -// skipped non-limited PyCF_COMPILE_MASK +use std::ffi::c_char; +use std::ffi::c_int; + +// skipped PyCF_MASK +// skipped PyCF_MASK_OBSOLETE +// skipped PyCF_SOURCE_IS_UTF8 +// skipped PyCF_DONT_IMPLY_DEDENT +// skipped PyCF_ONLY_AST +// skipped PyCF_IGNORE_COOKIE +// skipped PyCF_TYPE_COMMENTS +// skipped PyCF_ALLOW_TOP_LEVEL_AWAIT +// skipped PyCF_OPTIMIZED_AST +// skipped PyCF_COMPILE_MASK #[repr(C)] #[derive(Copy, Clone)] @@ -28,31 +30,23 @@ pub struct PyCompilerFlags { pub cf_feature_version: c_int, } -// skipped non-limited _PyCompilerFlags_INIT - -#[cfg(all(Py_3_12, not(PyPy)))] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCompilerSrcLocation { - pub lineno: c_int, - pub end_lineno: c_int, - pub col_offset: c_int, - pub end_col_offset: c_int, -} - -// skipped SRC_LOCATION_FROM_AST +// skipped _PyCompilerFlags_INIT -#[cfg(not(PyPy))] +// NB this type technically existed in the header until 3.13, when it was +// moved to the internal CPython headers. +// +// We choose not to expose it in the public API past 3.10, as it is +// not used in the public API past that point. +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { pub ff_features: c_int, - #[cfg(not(Py_3_12))] pub ff_lineno: c_int, - #[cfg(Py_3_12)] - pub ff_location: _PyCompilerSrcLocation, } +// FIXME: these constants should probably be &CStr, if they are used at all + pub const FUTURE_NESTED_SCOPES: &str = "nested_scopes"; pub const FUTURE_GENERATORS: &str = "generators"; pub const FUTURE_DIVISION: &str = "division"; @@ -62,13 +56,12 @@ pub const FUTURE_PRINT_FUNCTION: &str = "print_function"; pub const FUTURE_UNICODE_LITERALS: &str = "unicode_literals"; pub const FUTURE_BARRY_AS_BDFL: &str = "barry_as_FLUFL"; pub const FUTURE_GENERATOR_STOP: &str = "generator_stop"; -// skipped non-limited FUTURE_ANNOTATIONS +pub const FUTURE_ANNOTATIONS: &str = "annotations"; +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] extern "C" { - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyNode_Compile(arg1: *mut _node, arg2: *const c_char) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyAST_CompileEx( _mod: *mut _mod, filename: *const c_char, @@ -77,7 +70,6 @@ extern "C" { arena: *mut PyArena, ) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyAST_CompileObject( _mod: *mut _mod, filename: *mut PyObject, @@ -86,23 +78,20 @@ extern "C" { arena: *mut PyArena, ) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyFuture_FromAST(_mod: *mut _mod, filename: *const c_char) -> *mut PyFutureFeatures; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyFuture_FromASTObject( _mod: *mut _mod, filename: *mut PyObject, ) -> *mut PyFutureFeatures; +} - // skipped non-limited _Py_Mangle - // skipped non-limited PY_INVALID_STACK_EFFECT +pub const PY_INVALID_STACK_EFFECT: c_int = INT_MAX; + +extern "C" { pub fn PyCompile_OpcodeStackEffect(opcode: c_int, oparg: c_int) -> c_int; #[cfg(Py_3_8)] pub fn PyCompile_OpcodeStackEffectWithJump(opcode: c_int, oparg: c_int, jump: c_int) -> c_int; - - // skipped non-limited _PyASTOptimizeState - // skipped non-limited _PyAST_Optimize } diff --git a/pyo3-ffi/src/cpython/complexobject.rs b/pyo3-ffi/src/cpython/complexobject.rs new file mode 100644 index 00000000000..3283fc4e52f --- /dev/null +++ b/pyo3-ffi/src/cpython/complexobject.rs @@ -0,0 +1,30 @@ +use crate::PyObject; +use std::ffi::c_double; + +#[repr(C)] +#[derive(Copy, Clone)] +pub struct Py_complex { + pub real: c_double, + pub imag: c_double, +} + +// skipped private function _Py_c_sum +// skipped private function _Py_c_diff +// skipped private function _Py_c_neg +// skipped private function _Py_c_prod +// skipped private function _Py_c_quot +// skipped private function _Py_c_pow +// skipped private function _Py_c_abs + +#[repr(C)] +pub struct PyComplexObject { + pub ob_base: PyObject, + pub cval: Py_complex, +} + +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyComplex_FromCComplex")] + pub fn PyComplex_FromCComplex(v: Py_complex) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyComplex_AsCComplex")] + pub fn PyComplex_AsCComplex(op: *mut PyObject) -> Py_complex; +} diff --git a/pyo3-ffi/src/cpython/critical_section.rs b/pyo3-ffi/src/cpython/critical_section.rs new file mode 100644 index 00000000000..808dba870c6 --- /dev/null +++ b/pyo3-ffi/src/cpython/critical_section.rs @@ -0,0 +1,30 @@ +#[cfg(Py_GIL_DISABLED)] +use crate::PyMutex; +use crate::PyObject; + +#[repr(C)] +#[cfg(Py_GIL_DISABLED)] +pub struct PyCriticalSection { + _cs_prev: usize, + _cs_mutex: *mut PyMutex, +} + +#[repr(C)] +#[cfg(Py_GIL_DISABLED)] +pub struct PyCriticalSection2 { + _cs_base: PyCriticalSection, + _cs_mutex2: *mut PyMutex, +} + +#[cfg(not(Py_GIL_DISABLED))] +opaque_struct!(pub PyCriticalSection); + +#[cfg(not(Py_GIL_DISABLED))] +opaque_struct!(pub PyCriticalSection2); + +extern "C" { + pub fn PyCriticalSection_Begin(c: *mut PyCriticalSection, op: *mut PyObject); + pub fn PyCriticalSection_End(c: *mut PyCriticalSection); + pub fn PyCriticalSection2_Begin(c: *mut PyCriticalSection2, a: *mut PyObject, b: *mut PyObject); + pub fn PyCriticalSection2_End(c: *mut PyCriticalSection2); +} diff --git a/pyo3-ffi/src/cpython/descrobject.rs b/pyo3-ffi/src/cpython/descrobject.rs index 1b5ee466c8e..03d64789fa7 100644 --- a/pyo3-ffi/src/cpython/descrobject.rs +++ b/pyo3-ffi/src/cpython/descrobject.rs @@ -1,5 +1,5 @@ use crate::{PyGetSetDef, PyMethodDef, PyObject, PyTypeObject}; -use std::os::raw::{c_char, c_int, c_void}; +use std::ffi::{c_char, c_int, c_void}; pub type wrapperfunc = Option< unsafe extern "C" fn( @@ -69,10 +69,7 @@ pub struct PyWrapperDescrObject { pub d_wrapped: *mut c_void, } -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _PyMethodWrapper_Type: PyTypeObject; -} +// skipped _PyMethodWrapper_Type // skipped non-limited PyDescr_NewWrapper // skipped non-limited PyDescr_IsData diff --git a/pyo3-ffi/src/cpython/dictobject.rs b/pyo3-ffi/src/cpython/dictobject.rs index 4af990a2d9a..0ee90b53fc6 100644 --- a/pyo3-ffi/src/cpython/dictobject.rs +++ b/pyo3-ffi/src/cpython/dictobject.rs @@ -1,18 +1,26 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::c_int; +use std::ffi::c_int; -opaque_struct!(PyDictKeysObject); +opaque_struct!(pub PyDictKeysObject); #[cfg(Py_3_11)] -opaque_struct!(PyDictValues); +opaque_struct!(pub PyDictValues); +#[cfg(not(GraalPy))] #[repr(C)] #[derive(Debug)] pub struct PyDictObject { pub ob_base: PyObject, pub ma_used: Py_ssize_t, + #[cfg_attr( + Py_3_12, + deprecated(note = "Deprecated in Python 3.12 and will be removed in the future.") + )] + #[cfg(not(Py_3_14))] pub ma_version_tag: u64, + #[cfg(Py_3_14)] + _ma_watcher_tag: u64, pub ma_keys: *mut PyDictKeysObject, #[cfg(not(Py_3_11))] pub ma_values: *mut *mut PyObject, diff --git a/pyo3-ffi/src/cpython/floatobject.rs b/pyo3-ffi/src/cpython/floatobject.rs index e33da0b91b9..4b9ef1a2484 100644 --- a/pyo3-ffi/src/cpython/floatobject.rs +++ b/pyo3-ffi/src/cpython/floatobject.rs @@ -1,5 +1,7 @@ +#[cfg(GraalPy)] +use crate::PyFloat_AsDouble; use crate::{PyFloat_Check, PyObject}; -use std::os::raw::c_double; +use std::ffi::c_double; #[repr(C)] pub struct PyFloatObject { @@ -15,7 +17,10 @@ pub unsafe fn _PyFloat_CAST(op: *mut PyObject) -> *mut PyFloatObject { #[inline] pub unsafe fn PyFloat_AS_DOUBLE(op: *mut PyObject) -> c_double { - (*_PyFloat_CAST(op)).ob_fval + #[cfg(not(GraalPy))] + return (*_PyFloat_CAST(op)).ob_fval; + #[cfg(GraalPy)] + return PyFloat_AsDouble(op); } // skipped PyFloat_Pack2 diff --git a/pyo3-ffi/src/cpython/frameobject.rs b/pyo3-ffi/src/cpython/frameobject.rs index 7410000ef45..5bd9a620f19 100644 --- a/pyo3-ffi/src/cpython/frameobject.rs +++ b/pyo3-ffi/src/cpython/frameobject.rs @@ -1,74 +1,32 @@ +#[cfg(not(GraalPy))] use crate::cpython::code::PyCodeObject; +#[cfg(not(GraalPy))] use crate::object::*; +#[cfg(not(GraalPy))] use crate::pystate::PyThreadState; -#[cfg(not(any(PyPy, Py_3_11)))] -use std::os::raw::c_char; -use std::os::raw::c_int; -use std::ptr::addr_of_mut; +use crate::PyFrameObject; +#[cfg(not(any(PyPy, GraalPy, Py_3_11)))] +use std::ffi::c_char; +use std::ffi::c_int; -#[cfg(not(any(PyPy, Py_3_11)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_11)))] pub type PyFrameState = c_char; #[repr(C)] #[derive(Copy, Clone)] -#[cfg(not(any(PyPy, Py_3_11)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_11)))] pub struct PyTryBlock { pub b_type: c_int, pub b_handler: c_int, pub b_level: c_int, } -#[repr(C)] -#[derive(Copy, Clone)] -#[cfg(not(any(PyPy, Py_3_11)))] -pub struct PyFrameObject { - pub ob_base: PyVarObject, - pub f_back: *mut PyFrameObject, - pub f_code: *mut PyCodeObject, - pub f_builtins: *mut PyObject, - pub f_globals: *mut PyObject, - pub f_locals: *mut PyObject, - pub f_valuestack: *mut *mut PyObject, - - #[cfg(not(Py_3_10))] - pub f_stacktop: *mut *mut PyObject, - pub f_trace: *mut PyObject, - #[cfg(Py_3_10)] - pub f_stackdepth: c_int, - pub f_trace_lines: c_char, - pub f_trace_opcodes: c_char, - - pub f_gen: *mut PyObject, - - pub f_lasti: c_int, - pub f_lineno: c_int, - pub f_iblock: c_int, - #[cfg(not(Py_3_10))] - pub f_executing: c_char, - #[cfg(Py_3_10)] - pub f_state: PyFrameState, - pub f_blockstack: [PyTryBlock; crate::CO_MAXBLOCKS], - pub f_localsplus: [*mut PyObject; 1], -} - -#[cfg(any(PyPy, Py_3_11))] -opaque_struct!(PyFrameObject); - // skipped _PyFrame_IsRunnable // skipped _PyFrame_IsExecuting // skipped _PyFrameHasCompleted -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut PyFrame_Type: PyTypeObject; -} - -#[inline] -pub unsafe fn PyFrame_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == addr_of_mut!(PyFrame_Type)) as c_int -} - extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyFrame_New")] pub fn PyFrame_New( tstate: *mut PyThreadState, @@ -79,7 +37,7 @@ extern "C" { // skipped _PyFrame_New_NoTrack pub fn PyFrame_BlockSetup(f: *mut PyFrameObject, _type: c_int, handler: c_int, level: c_int); - #[cfg(not(any(PyPy, Py_3_11)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_11)))] pub fn PyFrame_BlockPop(f: *mut PyFrameObject) -> *mut PyTryBlock; pub fn PyFrame_LocalsToFast(f: *mut PyFrameObject, clear: c_int); @@ -87,7 +45,6 @@ extern "C" { pub fn PyFrame_FastToLocals(f: *mut PyFrameObject); // skipped _PyFrame_DebugMallocStats - // skipped PyFrame_GetBack #[cfg(not(Py_3_9))] pub fn PyFrame_ClearFreeList() -> c_int; diff --git a/pyo3-ffi/src/cpython/funcobject.rs b/pyo3-ffi/src/cpython/funcobject.rs index 1e9ee0cc18c..7a242e85abd 100644 --- a/pyo3-ffi/src/cpython/funcobject.rs +++ b/pyo3-ffi/src/cpython/funcobject.rs @@ -1,10 +1,10 @@ -use std::os::raw::c_int; +use std::ffi::c_int; #[cfg(not(all(PyPy, not(Py_3_8))))] use std::ptr::addr_of_mut; use crate::PyObject; -#[cfg(all(not(PyPy), not(Py_3_10)))] +#[cfg(all(not(any(PyPy, GraalPy)), not(Py_3_10)))] #[repr(C)] pub struct PyFunctionObject { pub ob_base: PyObject, @@ -24,7 +24,7 @@ pub struct PyFunctionObject { pub vectorcall: Option, } -#[cfg(all(not(PyPy), Py_3_10))] +#[cfg(all(not(any(PyPy, GraalPy)), Py_3_10))] #[repr(C)] pub struct PyFunctionObject { pub ob_base: PyObject, @@ -41,6 +41,8 @@ pub struct PyFunctionObject { pub func_weakreflist: *mut PyObject, pub func_module: *mut PyObject, pub func_annotations: *mut PyObject, + #[cfg(Py_3_14)] + pub func_annotate: *mut PyObject, #[cfg(Py_3_12)] pub func_typeparams: *mut PyObject, pub vectorcall: Option, @@ -55,6 +57,11 @@ pub struct PyFunctionObject { pub func_name: *mut PyObject, } +#[cfg(GraalPy)] +pub struct PyFunctionObject { + pub ob_base: PyObject, +} + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg(not(all(PyPy, not(Py_3_8))))] diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index aaa03f82eef..680a261c0de 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -1,15 +1,12 @@ use crate::object::*; use crate::PyFrameObject; -#[cfg(not(PyPy))] -use crate::_PyErr_StackItem; -#[cfg(Py_3_11)] -use std::os::raw::c_char; -use std::os::raw::c_int; +#[cfg(all(Py_3_11, not(any(PyPy, GraalPy, Py_3_14))))] +use std::ffi::c_char; +use std::ffi::c_int; use std::ptr::addr_of_mut; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy, Py_3_14)))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyGenObject { pub ob_base: PyObject, #[cfg(not(Py_3_11))] @@ -21,7 +18,8 @@ pub struct PyGenObject { pub gi_weakreflist: *mut PyObject, pub gi_name: *mut PyObject, pub gi_qualname: *mut PyObject, - pub gi_exc_state: _PyErr_StackItem, + #[allow(private_interfaces)] + pub gi_exc_state: crate::cpython::pystate::_PyErr_StackItem, #[cfg(Py_3_11)] pub gi_origin_or_finalizer: *mut PyObject, #[cfg(Py_3_11)] @@ -36,6 +34,9 @@ pub struct PyGenObject { pub gi_iframe: [*mut PyObject; 1], } +#[cfg(all(Py_3_14, not(any(PyPy, GraalPy))))] +opaque_struct!(pub PyGenObject); + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { pub static mut PyGen_Type: PyTypeObject; @@ -68,9 +69,10 @@ extern "C" { #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { pub static mut PyCoro_Type: PyTypeObject; - pub static mut _PyCoroWrapper_Type: PyTypeObject; } +// skipped _PyCoroWrapper_Type + #[inline] pub unsafe fn PyCoro_CheckExact(op: *mut PyObject) -> c_int { PyObject_TypeCheck(op, addr_of_mut!(PyCoro_Type)) diff --git a/pyo3-ffi/src/cpython/import.rs b/pyo3-ffi/src/cpython/import.rs index aafd71a8355..3b9374f68e3 100644 --- a/pyo3-ffi/src/cpython/import.rs +++ b/pyo3-ffi/src/cpython/import.rs @@ -1,7 +1,7 @@ use crate::{PyInterpreterState, PyObject}; #[cfg(not(PyPy))] -use std::os::raw::c_uchar; -use std::os::raw::{c_char, c_int}; +use std::ffi::c_uchar; +use std::ffi::{c_char, c_int}; // skipped PyInit__imp @@ -57,7 +57,7 @@ pub struct _frozen { pub size: c_int, #[cfg(Py_3_11)] pub is_package: c_int, - #[cfg(Py_3_11)] + #[cfg(all(Py_3_11, not(Py_3_13)))] pub get_code: Option *mut PyObject>, } @@ -65,10 +65,8 @@ pub struct _frozen { extern "C" { #[cfg(not(PyPy))] pub static mut PyImport_FrozenModules: *const _frozen; - #[cfg(all(not(PyPy), Py_3_11))] - pub static mut _PyImport_FrozenBootstrap: *const _frozen; - #[cfg(all(not(PyPy), Py_3_11))] - pub static mut _PyImport_FrozenStdlib: *const _frozen; - #[cfg(all(not(PyPy), Py_3_11))] - pub static mut _PyImport_FrozenTest: *const _frozen; } + +// skipped _PyImport_FrozenBootstrap +// skipped _PyImport_FrozenStdlib +// skipped _PyImport_FrozenTest diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index 17fe7559e1b..6b0ae2e5dec 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -2,7 +2,7 @@ use crate::Py_ssize_t; use libc::wchar_t; -use std::os::raw::{c_char, c_int, c_ulong}; +use std::ffi::{c_char, c_int, c_ulong}; #[repr(C)] #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -93,6 +93,8 @@ pub struct PyConfig { pub tracemalloc: c_int, #[cfg(Py_3_12)] pub perf_profiling: c_int, + #[cfg(Py_3_14)] + pub remote_debug: c_int, pub import_time: c_int, #[cfg(Py_3_11)] pub code_debug_ranges: c_int, @@ -141,6 +143,18 @@ pub struct PyConfig { pub safe_path: c_int, #[cfg(Py_3_12)] pub int_max_str_digits: c_int, + #[cfg(Py_3_14)] + pub thread_inherit_context: c_int, + #[cfg(Py_3_14)] + pub context_aware_warnings: c_int, + #[cfg(all(Py_3_14, target_os = "macos"))] + pub use_system_logger: c_int, + #[cfg(Py_3_13)] + pub cpu_count: c_int, + #[cfg(Py_GIL_DISABLED)] + pub enable_gil: c_int, + #[cfg(all(Py_3_14, Py_GIL_DISABLED))] + pub tlbc_enabled: c_int, pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, @@ -165,6 +179,8 @@ pub struct PyConfig { pub run_command: *mut wchar_t, pub run_module: *mut wchar_t, pub run_filename: *mut wchar_t, + #[cfg(Py_3_13)] + pub sys_path_0: *mut wchar_t, pub _install_importlib: c_int, pub _init_main: c_int, #[cfg(all(Py_3_9, not(Py_3_12)))] @@ -173,6 +189,8 @@ pub struct PyConfig { pub _is_python_build: c_int, #[cfg(all(Py_3_9, not(Py_3_10)))] pub _orig_argv: PyWideStringList, + #[cfg(all(Py_3_13, py_sys_config = "Py_DEBUG"))] + pub run_presite: *mut wchar_t, } extern "C" { diff --git a/pyo3-ffi/src/cpython/listobject.rs b/pyo3-ffi/src/cpython/listobject.rs index 7fb2228fadd..694e6bc4290 100644 --- a/pyo3-ffi/src/cpython/listobject.rs +++ b/pyo3-ffi/src/cpython/listobject.rs @@ -4,7 +4,6 @@ use crate::pyport::Py_ssize_t; #[cfg(not(PyPy))] #[repr(C)] -#[derive(Copy, Clone)] pub struct PyListObject { pub ob_base: PyVarObject, pub ob_item: *mut *mut PyObject, @@ -22,14 +21,14 @@ pub struct PyListObject { /// Macro, trading safety for speed #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyList_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject { *(*(op as *mut PyListObject)).ob_item.offset(i) } /// Macro, *only* to be used to fill in brand new lists #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyList_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) { *(*(op as *mut PyListObject)).ob_item.offset(i) = v; } diff --git a/pyo3-ffi/src/cpython/lock.rs b/pyo3-ffi/src/cpython/lock.rs new file mode 100644 index 00000000000..487173f9afb --- /dev/null +++ b/pyo3-ffi/src/cpython/lock.rs @@ -0,0 +1,26 @@ +#[cfg(Py_3_14)] +use std::os::raw::c_int; +use std::sync::atomic::AtomicU8; + +#[repr(transparent)] +#[derive(Debug)] +pub struct PyMutex { + pub(crate) _bits: AtomicU8, +} + +// we don't impl Default because PyO3's safe wrappers don't need it +#[allow(clippy::new_without_default)] +impl PyMutex { + pub const fn new() -> PyMutex { + PyMutex { + _bits: AtomicU8::new(0), + } + } +} + +extern "C" { + pub fn PyMutex_Lock(m: *mut PyMutex); + pub fn PyMutex_Unlock(m: *mut PyMutex); + #[cfg(Py_3_14)] + pub fn PyMutex_IsLocked(m: *mut PyMutex) -> c_int; +} diff --git a/pyo3-ffi/src/cpython/longobject.rs b/pyo3-ffi/src/cpython/longobject.rs new file mode 100644 index 00000000000..a778d8c26d2 --- /dev/null +++ b/pyo3-ffi/src/cpython/longobject.rs @@ -0,0 +1,74 @@ +use crate::longobject::*; +use crate::object::*; +#[cfg(Py_3_13)] +use crate::pyport::Py_ssize_t; +use libc::size_t; +#[cfg(Py_3_13)] +use std::ffi::c_void; +use std::ffi::{c_int, c_uchar}; + +#[cfg(Py_3_13)] +extern "C" { + pub fn PyLong_FromUnicodeObject(u: *mut PyObject, base: c_int) -> *mut PyObject; +} + +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_DEFAULTS: c_int = -1; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_BIG_ENDIAN: c_int = 0; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_LITTLE_ENDIAN: c_int = 1; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_NATIVE_ENDIAN: c_int = 3; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_UNSIGNED_BUFFER: c_int = 4; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_REJECT_NEGATIVE: c_int = 8; + +extern "C" { + // skipped _PyLong_Sign + + #[cfg(Py_3_13)] + pub fn PyLong_AsNativeBytes( + v: *mut PyObject, + buffer: *mut c_void, + n_bytes: Py_ssize_t, + flags: c_int, + ) -> Py_ssize_t; + + #[cfg(Py_3_13)] + pub fn PyLong_FromNativeBytes( + buffer: *const c_void, + n_bytes: size_t, + flags: c_int, + ) -> *mut PyObject; + + #[cfg(Py_3_13)] + pub fn PyLong_FromUnsignedNativeBytes( + buffer: *const c_void, + n_bytes: size_t, + flags: c_int, + ) -> *mut PyObject; + + // skipped PyUnstable_Long_IsCompact + // skipped PyUnstable_Long_CompactValue + + #[cfg_attr(PyPy, link_name = "_PyPyLong_FromByteArray")] + pub fn _PyLong_FromByteArray( + bytes: *const c_uchar, + n: size_t, + little_endian: c_int, + is_signed: c_int, + ) -> *mut PyObject; + + #[cfg_attr(PyPy, link_name = "_PyPyLong_AsByteArrayO")] + pub fn _PyLong_AsByteArray( + v: *mut PyLongObject, + bytes: *mut c_uchar, + n: size_t, + little_endian: c_int, + is_signed: c_int, + ) -> c_int; + + // skipped _PyLong_GCD +} diff --git a/pyo3-ffi/src/cpython/methodobject.rs b/pyo3-ffi/src/cpython/methodobject.rs index 7d9659785ba..096bb02ae13 100644 --- a/pyo3-ffi/src/cpython/methodobject.rs +++ b/pyo3-ffi/src/cpython/methodobject.rs @@ -1,8 +1,10 @@ use crate::object::*; +#[cfg(not(GraalPy))] use crate::{PyCFunctionObject, PyMethodDefPointer, METH_METHOD, METH_STATIC}; -use std::os::raw::c_int; +use std::ffi::c_int; use std::ptr::addr_of_mut; +#[cfg(not(GraalPy))] pub struct PyCMethodObject { pub func: PyCFunctionObject, pub mm_class: *mut PyTypeObject, @@ -23,6 +25,7 @@ pub unsafe fn PyCMethod_Check(op: *mut PyObject) -> c_int { PyObject_TypeCheck(op, addr_of_mut!(PyCMethod_Type)) } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyCFunction_GET_FUNCTION(func: *mut PyObject) -> PyMethodDefPointer { debug_assert_eq!(PyCMethod_Check(func), 1); @@ -31,6 +34,7 @@ pub unsafe fn PyCFunction_GET_FUNCTION(func: *mut PyObject) -> PyMethodDefPointe (*(*func).m_ml).ml_meth } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyCFunction_GET_SELF(func: *mut PyObject) -> *mut PyObject { debug_assert_eq!(PyCMethod_Check(func), 1); @@ -43,6 +47,7 @@ pub unsafe fn PyCFunction_GET_SELF(func: *mut PyObject) -> *mut PyObject { } } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyCFunction_GET_FLAGS(func: *mut PyObject) -> c_int { debug_assert_eq!(PyCMethod_Check(func), 1); @@ -51,6 +56,7 @@ pub unsafe fn PyCFunction_GET_FLAGS(func: *mut PyObject) -> c_int { (*(*func).m_ml).ml_flags } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyCFunction_GET_CLASS(func: *mut PyObject) -> *mut PyTypeObject { debug_assert_eq!(PyCMethod_Check(func), 1); diff --git a/pyo3-ffi/src/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs index 738ba37652e..d2b2274628b 100644 --- a/pyo3-ffi/src/cpython/mod.rs +++ b/pyo3-ffi/src/cpython/mod.rs @@ -5,6 +5,9 @@ pub(crate) mod bytesobject; pub(crate) mod ceval; pub(crate) mod code; pub(crate) mod compile; +pub(crate) mod complexobject; +#[cfg(Py_3_13)] +pub(crate) mod critical_section; pub(crate) mod descrobject; #[cfg(not(PyPy))] pub(crate) mod dictobject; @@ -18,6 +21,9 @@ pub(crate) mod import; pub(crate) mod initconfig; // skipped interpreteridobject.h pub(crate) mod listobject; +#[cfg(Py_3_13)] +pub(crate) mod lock; +pub(crate) mod longobject; #[cfg(all(Py_3_9, not(PyPy)))] pub(crate) mod methodobject; pub(crate) mod object; @@ -32,6 +38,7 @@ pub(crate) mod pythonrun; // skipped sysmodule.h pub(crate) mod floatobject; pub(crate) mod pyframe; +pub(crate) mod pyhash; pub(crate) mod tupleobject; pub(crate) mod unicodeobject; pub(crate) mod weakrefobject; @@ -42,6 +49,9 @@ pub use self::bytesobject::*; pub use self::ceval::*; pub use self::code::*; pub use self::compile::*; +pub use self::complexobject::*; +#[cfg(Py_3_13)] +pub use self::critical_section::*; pub use self::descrobject::*; #[cfg(not(PyPy))] pub use self::dictobject::*; @@ -53,14 +63,18 @@ pub use self::import::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::initconfig::*; pub use self::listobject::*; +#[cfg(Py_3_13)] +pub use self::lock::*; +pub use self::longobject::*; #[cfg(all(Py_3_9, not(PyPy)))] pub use self::methodobject::*; pub use self::object::*; pub use self::objimpl::*; pub use self::pydebug::*; pub use self::pyerrors::*; -#[cfg(Py_3_11)] pub use self::pyframe::*; +#[cfg(any(not(PyPy), Py_3_13))] +pub use self::pyhash::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::pylifecycle::*; pub use self::pymem::*; @@ -68,5 +82,5 @@ pub use self::pystate::*; pub use self::pythonrun::*; pub use self::tupleobject::*; pub use self::unicodeobject::*; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub use self::weakrefobject::*; diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 161fb50cf24..9230a1f65e8 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -1,25 +1,28 @@ #[cfg(Py_3_8)] use crate::vectorcallfunc; -#[cfg(Py_3_11)] -use crate::PyModuleDef; use crate::{object, PyGetSetDef, PyMemberDef, PyMethodDef, PyObject, Py_ssize_t}; +use std::ffi::{c_char, c_int, c_uint, c_void}; use std::mem; -use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; -// skipped _Py_NewReference -// skipped _Py_ForgetReference -// skipped _Py_GetRefTotal +// skipped private _Py_NewReference +// skipped private _Py_NewReferenceNoTotal +// skipped private _Py_ResurrectReference + +// skipped private _Py_GetGlobalRefTotal +// skipped private _Py_GetRefTotal +// skipped private _Py_GetLegacyRefTotal +// skipped private _PyInterpreterState_GetRefTotal -// skipped _Py_Identifier +// skipped private _Py_Identifier -// skipped _Py_static_string_init -// skipped _Py_static_string -// skipped _Py_IDENTIFIER +// skipped private _Py_static_string_init +// skipped private _Py_static_string +// skipped private _Py_IDENTIFIER #[cfg(not(Py_3_11))] // moved to src/buffer.rs from Python mod bufferinfo { use crate::Py_ssize_t; - use std::os::raw::{c_char, c_int, c_void}; + use std::ffi::{c_char, c_int, c_void}; use std::ptr; #[repr(C)] @@ -46,6 +49,7 @@ mod bufferinfo { } impl Py_buffer { + #[allow(clippy::new_without_default)] pub const fn new() -> Self { Py_buffer { buf: ptr::null_mut(), @@ -204,17 +208,8 @@ pub type printfunc = unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut ::libc::FILE, arg3: c_int) -> c_int; #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] pub struct PyTypeObject { - #[cfg(all(PyPy, not(Py_3_9)))] - pub ob_refcnt: Py_ssize_t, - #[cfg(all(PyPy, not(Py_3_9)))] - pub ob_pypy_link: Py_ssize_t, - #[cfg(all(PyPy, not(Py_3_9)))] - pub ob_type: *mut PyTypeObject, - #[cfg(all(PyPy, not(Py_3_9)))] - pub ob_size: Py_ssize_t, - #[cfg(not(all(PyPy, not(Py_3_9))))] pub ob_base: object::PyVarObject, pub tp_name: *const c_char, pub tp_basicsize: Py_ssize_t, @@ -237,7 +232,10 @@ pub struct PyTypeObject { pub tp_getattro: Option, pub tp_setattro: Option, pub tp_as_buffer: *mut PyBufferProcs, - pub tp_flags: c_ulong, + #[cfg(not(Py_GIL_DISABLED))] + pub tp_flags: std::ffi::c_ulong, + #[cfg(Py_GIL_DISABLED)] + pub tp_flags: crate::impl_::AtomicCULong, pub tp_doc: *const c_char, pub tp_traverse: Option, pub tp_clear: Option, @@ -270,10 +268,8 @@ pub struct PyTypeObject { pub tp_vectorcall: Option, #[cfg(Py_3_12)] pub tp_watched: c_char, - #[cfg(any(all(PyPy, Py_3_8, not(Py_3_10)), all(not(PyPy), Py_3_8, not(Py_3_9))))] + #[cfg(all(not(PyPy), Py_3_8, not(Py_3_9)))] pub tp_print: Option, - #[cfg(all(PyPy, not(Py_3_10)))] - pub tp_pypy_flags: std::os::raw::c_long, #[cfg(py_sys_config = "COUNT_ALLOCS")] pub tp_allocs: Py_ssize_t, #[cfg(py_sys_config = "COUNT_ALLOCS")] @@ -289,14 +285,15 @@ pub struct PyTypeObject { #[cfg(Py_3_11)] #[repr(C)] #[derive(Clone)] -pub struct _specialization_cache { - pub getitem: *mut PyObject, +struct _specialization_cache { + getitem: *mut PyObject, #[cfg(Py_3_12)] - pub getitem_version: u32, + getitem_version: u32, + #[cfg(Py_3_13)] + init: *mut PyObject, } #[repr(C)] -#[derive(Clone)] pub struct PyHeapTypeObject { pub ht_type: PyTypeObject, pub as_async: PyAsyncMethods, @@ -311,10 +308,14 @@ pub struct PyHeapTypeObject { pub ht_cached_keys: *mut c_void, #[cfg(Py_3_9)] pub ht_module: *mut object::PyObject, - #[cfg(Py_3_11)] - pub _ht_tpname: *mut c_char, - #[cfg(Py_3_11)] - pub _spec_cache: _specialization_cache, + #[cfg(all(Py_3_11, not(PyPy)))] + _ht_tpname: *mut c_char, + #[cfg(Py_3_14)] + pub ht_token: *mut c_void, + #[cfg(all(Py_3_11, not(PyPy)))] + _spec_cache: _specialization_cache, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + pub unique_id: Py_ssize_t, } impl Default for PyHeapTypeObject { @@ -325,82 +326,75 @@ impl Default for PyHeapTypeObject { } #[inline] +#[cfg(not(Py_3_11))] pub unsafe fn PyHeapType_GET_MEMBERS(etype: *mut PyHeapTypeObject) -> *mut PyMemberDef { let py_type = object::Py_TYPE(etype as *mut object::PyObject); let ptr = etype.offset((*py_type).tp_basicsize); ptr as *mut PyMemberDef } -// skipped _PyType_Name -// skipped _PyType_Lookup -// skipped _PyType_LookupId -// skipped _PyObject_LookupSpecial -// skipped _PyType_CalculateMetaclass -// skipped _PyType_GetDocFromInternalDoc -// skipped _PyType_GetTextSignatureFromInternalDoc +// skipped private _PyType_Name +// skipped private _PyType_Lookup +// skipped private _PyType_LookupRef extern "C" { - #[cfg(Py_3_11)] - #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleByDef")] - pub fn PyType_GetModuleByDef(ty: *mut PyTypeObject, def: *mut PyModuleDef) -> *mut PyObject; - #[cfg(Py_3_12)] pub fn PyType_GetDict(o: *mut PyTypeObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_Print")] pub fn PyObject_Print(o: *mut PyObject, fp: *mut ::libc::FILE, flags: c_int) -> c_int; - // skipped _Py_BreakPoint - // skipped _PyObject_Dump - // skipped _PyObject_IsFreed - // skipped _PyObject_IsAbstract + // skipped private _Py_BreakPoint + // skipped private _PyObject_Dump + // skipped _PyObject_GetAttrId - // skipped _PyObject_SetAttrId - // skipped _PyObject_LookupAttr - // skipped _PyObject_LookupAttrId - // skipped _PyObject_GetMethod - #[cfg(not(PyPy))] - pub fn _PyObject_GetDictPtr(obj: *mut PyObject) -> *mut *mut PyObject; - #[cfg(not(PyPy))] - pub fn _PyObject_NextNotImplemented(arg1: *mut PyObject) -> *mut PyObject; + // skipped private _PyObject_GetDictPtr pub fn PyObject_CallFinalizer(arg1: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPyObject_CallFinalizerFromDealloc")] pub fn PyObject_CallFinalizerFromDealloc(arg1: *mut PyObject) -> c_int; - // skipped _PyObject_GenericGetAttrWithDict - // skipped _PyObject_GenericSetAttrWithDict - // skipped _PyObject_FunctionStr + // skipped private _PyObject_GenericGetAttrWithDict + // skipped private _PyObject_GenericSetAttrWithDict + // skipped private _PyObject_FunctionStr } // skipped Py_SETREF // skipped Py_XSETREF -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _PyNone_Type: PyTypeObject; - pub static mut _PyNotImplemented_Type: PyTypeObject; -} - -// skipped _Py_SwappedOp +// skipped private _PyObject_ASSERT_FROM +// skipped private _PyObject_ASSERT_WITH_MSG +// skipped private _PyObject_ASSERT +// skipped private _PyObject_ASSERT_FAILED_MSG +// skipped private _PyObject_AssertFailed -// skipped _PyDebugAllocatorStats -// skipped _PyObject_DebugTypeStats -// skipped _PyObject_ASSERT_FROM -// skipped _PyObject_ASSERT_WITH_MSG -// skipped _PyObject_ASSERT -// skipped _PyObject_ASSERT_FAILED_MSG -// skipped _PyObject_AssertFailed -// skipped _PyObject_CheckConsistency +// skipped private _PyTrash_begin +// skipped private _PyTrash_end // skipped _PyTrash_thread_deposit_object // skipped _PyTrash_thread_destroy_chain -// skipped _PyTrash_begin -// skipped _PyTrash_end -// skipped _PyTrash_cond -// skipped PyTrash_UNWIND_LEVEL -// skipped Py_TRASHCAN_BEGIN_CONDITION -// skipped Py_TRASHCAN_END + // skipped Py_TRASHCAN_BEGIN -// skipped Py_TRASHCAN_SAFE_BEGIN -// skipped Py_TRASHCAN_SAFE_END +// skipped Py_TRASHCAN_END + +// skipped PyObject_GetItemData + +// skipped PyObject_VisitManagedDict +// skipped _PyObject_SetManagedDict +// skipped PyObject_ClearManagedDict + +// skipped TYPE_MAX_WATCHERS + +// skipped PyType_WatchCallback +// skipped PyType_AddWatcher +// skipped PyType_ClearWatcher +// skipped PyType_Watch +// skipped PyType_Unwatch + +// skipped PyUnstable_Type_AssignVersionTag + +// skipped PyRefTracerEvent + +// skipped PyRefTracer +// skipped PyRefTracer_SetTracer +// skipped PyRefTracer_GetTracer diff --git a/pyo3-ffi/src/cpython/objimpl.rs b/pyo3-ffi/src/cpython/objimpl.rs index 36a4380d122..71087d28d2c 100644 --- a/pyo3-ffi/src/cpython/objimpl.rs +++ b/pyo3-ffi/src/cpython/objimpl.rs @@ -1,8 +1,9 @@ +#[cfg(not(all(Py_3_11, any(PyPy, GraalPy))))] use libc::size_t; -use std::os::raw::c_int; +use std::ffi::c_int; -#[cfg(not(PyPy))] -use std::os::raw::c_void; +#[cfg(not(any(PyPy, GraalPy)))] +use std::ffi::c_void; use crate::object::*; @@ -14,7 +15,7 @@ extern "C" { pub fn _Py_GetAllocatedBlocks() -> crate::Py_ssize_t; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyObjectArenaAllocator { @@ -23,7 +24,7 @@ pub struct PyObjectArenaAllocator { pub free: Option, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] impl Default for PyObjectArenaAllocator { #[inline] fn default() -> Self { @@ -32,9 +33,9 @@ impl Default for PyObjectArenaAllocator { } extern "C" { - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyObject_GetArenaAllocator(allocator: *mut PyObjectArenaAllocator); - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyObject_SetArenaAllocator(allocator: *mut PyObjectArenaAllocator); #[cfg(Py_3_9)] diff --git a/pyo3-ffi/src/cpython/pydebug.rs b/pyo3-ffi/src/cpython/pydebug.rs index a42848e8fdb..6878554aead 100644 --- a/pyo3-ffi/src/cpython/pydebug.rs +++ b/pyo3-ffi/src/cpython/pydebug.rs @@ -1,4 +1,4 @@ -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; #[cfg(not(Py_LIMITED_API))] #[cfg_attr(windows, link(name = "pythonXY"))] diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index fe7b4d4b045..c9831669ac7 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -1,5 +1,5 @@ use crate::PyObject; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] use crate::Py_ssize_t; #[repr(C)] @@ -22,7 +22,7 @@ pub struct PyBaseExceptionObject { pub suppress_context: char, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PySyntaxErrorObject { @@ -46,9 +46,11 @@ pub struct PySyntaxErrorObject { pub end_offset: *mut PyObject, pub text: *mut PyObject, pub print_file_and_line: *mut PyObject, + #[cfg(Py_3_14)] + pub metadata: *mut PyObject, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PyImportErrorObject { @@ -69,7 +71,7 @@ pub struct PyImportErrorObject { pub name_from: *mut PyObject, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PyUnicodeErrorObject { @@ -90,7 +92,7 @@ pub struct PyUnicodeErrorObject { pub reason: *mut PyObject, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PySystemExitObject { @@ -107,7 +109,7 @@ pub struct PySystemExitObject { pub code: *mut PyObject, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PyOSErrorObject { @@ -152,10 +154,7 @@ pub struct PyStopIterationObject { pub value: *mut PyObject, } -extern "C" { - #[cfg(not(PyPy))] - pub fn _PyErr_ChainExceptions(typ: *mut PyObject, val: *mut PyObject, tb: *mut PyObject); -} +// skipped _PyErr_ChainExceptions // skipped PyNameErrorObject // skipped PyAttributeErrorObject diff --git a/pyo3-ffi/src/cpython/pyframe.rs b/pyo3-ffi/src/cpython/pyframe.rs index d0cfa0a2c6d..6da9ecd5b53 100644 --- a/pyo3-ffi/src/cpython/pyframe.rs +++ b/pyo3-ffi/src/cpython/pyframe.rs @@ -1,2 +1,62 @@ -#[cfg(Py_3_11)] -opaque_struct!(_PyInterpreterFrame); +#[cfg(any(Py_3_11, all(Py_3_9, not(PyPy))))] +use crate::PyFrameObject; +use crate::{PyObject, PyTypeObject, Py_TYPE}; +#[cfg(Py_3_12)] +use std::ffi::c_char; +use std::ffi::c_int; +use std::ptr::addr_of_mut; + +// NB used in `_PyEval_EvalFrameDefault`, maybe we remove this too. +#[cfg(all(Py_3_11, not(PyPy)))] +opaque_struct!(pub _PyInterpreterFrame); + +#[cfg_attr(windows, link(name = "pythonXY"))] +extern "C" { + pub static mut PyFrame_Type: PyTypeObject; + + #[cfg(Py_3_13)] + pub static mut PyFrameLocalsProxy_Type: PyTypeObject; +} + +#[inline] +pub unsafe fn PyFrame_Check(op: *mut PyObject) -> c_int { + (Py_TYPE(op) == addr_of_mut!(PyFrame_Type)) as c_int +} + +#[cfg(Py_3_13)] +#[inline] +pub unsafe fn PyFrameLocalsProxy_Check(op: *mut PyObject) -> c_int { + (Py_TYPE(op) == addr_of_mut!(PyFrameLocalsProxy_Type)) as c_int +} + +extern "C" { + #[cfg(all(Py_3_9, not(PyPy)))] + pub fn PyFrame_GetBack(frame: *mut PyFrameObject) -> *mut PyFrameObject; + + #[cfg(Py_3_11)] + pub fn PyFrame_GetLocals(frame: *mut PyFrameObject) -> *mut PyObject; + + #[cfg(Py_3_11)] + pub fn PyFrame_GetGlobals(frame: *mut PyFrameObject) -> *mut PyObject; + + #[cfg(Py_3_11)] + pub fn PyFrame_GetBuiltins(frame: *mut PyFrameObject) -> *mut PyObject; + + #[cfg(Py_3_11)] + pub fn PyFrame_GetGenerator(frame: *mut PyFrameObject) -> *mut PyObject; + + #[cfg(Py_3_11)] + pub fn PyFrame_GetLasti(frame: *mut PyFrameObject) -> c_int; + + #[cfg(Py_3_12)] + pub fn PyFrame_GetVar(frame: *mut PyFrameObject, name: *mut PyObject) -> *mut PyObject; + + #[cfg(Py_3_12)] + pub fn PyFrame_GetVarString(frame: *mut PyFrameObject, name: *mut c_char) -> *mut PyObject; + + // skipped PyUnstable_InterpreterFrame_GetCode + // skipped PyUnstable_InterpreterFrame_GetLasti + // skipped PyUnstable_InterpreterFrame_GetLine + // skipped PyUnstable_ExecutableKinds + +} diff --git a/pyo3-ffi/src/cpython/pyhash.rs b/pyo3-ffi/src/cpython/pyhash.rs new file mode 100644 index 00000000000..f5c98c23256 --- /dev/null +++ b/pyo3-ffi/src/cpython/pyhash.rs @@ -0,0 +1,38 @@ +#[cfg(Py_3_14)] +use crate::Py_ssize_t; +#[cfg(Py_3_13)] +use crate::{PyObject, Py_hash_t}; +#[cfg(any(Py_3_13, not(PyPy)))] +use std::ffi::c_void; +#[cfg(not(PyPy))] +use std::ffi::{c_char, c_int}; + +#[cfg(not(PyPy))] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct PyHash_FuncDef { + pub hash: + Option crate::Py_hash_t>, + pub name: *const c_char, + pub hash_bits: c_int, + pub seed_bits: c_int, +} + +#[cfg(not(PyPy))] +impl Default for PyHash_FuncDef { + #[inline] + fn default() -> Self { + unsafe { std::mem::zeroed() } + } +} + +extern "C" { + #[cfg(not(PyPy))] + pub fn PyHash_GetFuncDef() -> *mut PyHash_FuncDef; + #[cfg(Py_3_13)] + pub fn Py_HashPointer(ptr: *const c_void) -> Py_hash_t; + #[cfg(Py_3_13)] + pub fn PyObject_GenericHash(obj: *mut PyObject) -> Py_hash_t; + #[cfg(Py_3_14)] + pub fn Py_HashBuffer(ptr: *const c_void, len: Py_ssize_t) -> Py_hash_t; +} diff --git a/pyo3-ffi/src/cpython/pylifecycle.rs b/pyo3-ffi/src/cpython/pylifecycle.rs index c259c369efd..975dbce1915 100644 --- a/pyo3-ffi/src/cpython/pylifecycle.rs +++ b/pyo3-ffi/src/cpython/pylifecycle.rs @@ -1,10 +1,11 @@ use crate::{PyConfig, PyPreConfig, PyStatus, Py_ssize_t}; use libc::wchar_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; -// "private" functions in cpython/pylifecycle.h accepted in PEP 587 extern "C" { - // skipped _Py_SetStandardStreamEncoding; + + // skipped Py_FrozenMain + pub fn Py_PreInitialize(src_config: *const PyPreConfig) -> PyStatus; pub fn Py_PreInitializeFromBytesArgs( src_config: *const PyPreConfig, @@ -16,34 +17,14 @@ extern "C" { argc: Py_ssize_t, argv: *mut *mut wchar_t, ) -> PyStatus; - pub fn _Py_IsCoreInitialized() -> c_int; pub fn Py_InitializeFromConfig(config: *const PyConfig) -> PyStatus; - pub fn _Py_InitializeMain() -> PyStatus; pub fn Py_RunMain() -> c_int; pub fn Py_ExitStatusException(status: PyStatus) -> !; - // skipped _Py_RestoreSignals - // skipped Py_FdIsInteractive - // skipped _Py_FdIsInteractive - - // skipped _Py_SetProgramFullPath - - // skipped _Py_gitidentifier - // skipped _Py_getversion - - // skipped _Py_IsFinalizing - - // skipped _PyOS_URandom - // skipped _PyOS_URandomNonblock - - // skipped _Py_CoerceLegacyLocale - // skipped _Py_LegacyLocaleDetected - // skipped _Py_SetLocaleFromEnv - } #[cfg(Py_3_12)] @@ -76,6 +57,11 @@ pub const _PyInterpreterConfig_INIT: PyInterpreterConfig = PyInterpreterConfig { gil: PyInterpreterConfig_OWN_GIL, }; +// https://github.com/python/cpython/blob/902de283a8303177eb95bf5bc252d2421fcbd758/Include/cpython/pylifecycle.h#L63-L65 +#[cfg(Py_3_12)] +const _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS: c_int = + if cfg!(Py_GIL_DISABLED) { 1 } else { 0 }; + #[cfg(Py_3_12)] pub const _PyInterpreterConfig_LEGACY_INIT: PyInterpreterConfig = PyInterpreterConfig { use_main_obmalloc: 1, @@ -83,7 +69,7 @@ pub const _PyInterpreterConfig_LEGACY_INIT: PyInterpreterConfig = PyInterpreterC allow_exec: 1, allow_threads: 1, allow_daemon_threads: 1, - check_multi_interp_extensions: 0, + check_multi_interp_extensions: _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS, gil: PyInterpreterConfig_SHARED_GIL, }; @@ -96,4 +82,4 @@ extern "C" { } // skipped atexit_datacallbackfunc -// skipped _Py_AtExit +// skipped PyUnstable_AtExit diff --git a/pyo3-ffi/src/cpython/pymem.rs b/pyo3-ffi/src/cpython/pymem.rs index 2dfb3f3bcfa..762e850820f 100644 --- a/pyo3-ffi/src/cpython/pymem.rs +++ b/pyo3-ffi/src/cpython/pymem.rs @@ -1,5 +1,5 @@ use libc::size_t; -use std::os::raw::c_void; +use std::ffi::c_void; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyMem_RawMalloc")] @@ -26,7 +26,7 @@ pub enum PyMemAllocatorDomain { } // skipped PyMemAllocatorName -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyMemAllocatorEx { @@ -40,10 +40,10 @@ pub struct PyMemAllocatorEx { } extern "C" { - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyMem_GetAllocator(domain: PyMemAllocatorDomain, allocator: *mut PyMemAllocatorEx); - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyMem_SetAllocator(domain: PyMemAllocatorDomain, allocator: *mut PyMemAllocatorEx); - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyMem_SetupDebugHooks(); } diff --git a/pyo3-ffi/src/cpython/pystate.rs b/pyo3-ffi/src/cpython/pystate.rs index 5481265b55d..96e2635a157 100644 --- a/pyo3-ffi/src/cpython/pystate.rs +++ b/pyo3-ffi/src/cpython/pystate.rs @@ -1,7 +1,7 @@ #[cfg(not(PyPy))] use crate::PyThreadState; use crate::{PyFrameObject, PyInterpreterState, PyObject}; -use std::os::raw::c_int; +use std::ffi::c_int; // skipped _PyInterpreterState_RequiresIDRef // skipped _PyInterpreterState_RequireIDRef @@ -27,16 +27,17 @@ pub const PyTrace_OPCODE: c_int = 7; // skipped PyTraceInfo // skipped CFrame +/// Private structure used inline in `PyGenObject` #[cfg(not(PyPy))] #[repr(C)] #[derive(Clone, Copy)] -pub struct _PyErr_StackItem { +pub(crate) struct _PyErr_StackItem { #[cfg(not(Py_3_11))] - pub exc_type: *mut PyObject, - pub exc_value: *mut PyObject, + exc_type: *mut PyObject, + exc_value: *mut PyObject, #[cfg(not(Py_3_11))] - pub exc_traceback: *mut PyObject, - pub previous_item: *mut _PyErr_StackItem, + exc_traceback: *mut PyObject, + previous_item: *mut _PyErr_StackItem, } // skipped _PyStackChunk @@ -69,21 +70,21 @@ extern "C" { pub fn PyThreadState_DeleteCurrent(); } -#[cfg(all(Py_3_9, not(Py_3_11)))] +#[cfg(all(Py_3_9, not(any(Py_3_11, PyPy))))] pub type _PyFrameEvalFunction = extern "C" fn( *mut crate::PyThreadState, *mut crate::PyFrameObject, c_int, ) -> *mut crate::object::PyObject; -#[cfg(Py_3_11)] +#[cfg(all(Py_3_11, not(PyPy)))] pub type _PyFrameEvalFunction = extern "C" fn( *mut crate::PyThreadState, *mut crate::_PyInterpreterFrame, c_int, ) -> *mut crate::object::PyObject; -#[cfg(Py_3_9)] +#[cfg(all(Py_3_9, not(PyPy)))] extern "C" { /// Get the frame evaluation function. pub fn _PyInterpreterState_GetEvalFrameFunc( diff --git a/pyo3-ffi/src/cpython/pythonrun.rs b/pyo3-ffi/src/cpython/pythonrun.rs index a92528b7fdc..db3767cb60c 100644 --- a/pyo3-ffi/src/cpython/pythonrun.rs +++ b/pyo3-ffi/src/cpython/pythonrun.rs @@ -1,11 +1,11 @@ use crate::object::*; -#[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] +#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API, Py_3_10)))] use crate::pyarena::PyArena; use crate::PyCompilerFlags; -#[cfg(not(any(PyPy, Py_3_10)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] use crate::{_mod, _node}; use libc::FILE; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; extern "C" { pub fn PyRun_SimpleStringFlags(arg1: *const c_char, arg2: *mut PyCompilerFlags) -> c_int; @@ -54,7 +54,7 @@ extern "C" { flags: *mut PyCompilerFlags, ) -> c_int; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] pub fn PyParser_ASTFromString( s: *const c_char, filename: *const c_char, @@ -62,7 +62,7 @@ extern "C" { flags: *mut PyCompilerFlags, arena: *mut PyArena, ) -> *mut _mod; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] pub fn PyParser_ASTFromStringObject( s: *const c_char, filename: *mut PyObject, @@ -70,7 +70,7 @@ extern "C" { flags: *mut PyCompilerFlags, arena: *mut PyArena, ) -> *mut _mod; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] pub fn PyParser_ASTFromFile( fp: *mut FILE, filename: *const c_char, @@ -82,7 +82,7 @@ extern "C" { errcode: *mut c_int, arena: *mut PyArena, ) -> *mut _mod; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] pub fn PyParser_ASTFromFileObject( fp: *mut FILE, filename: *mut PyObject, @@ -105,7 +105,7 @@ extern "C" { arg4: *mut PyObject, arg5: *mut PyCompilerFlags, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_FileExFlags( fp: *mut FILE, filename: *const c_char, @@ -116,7 +116,7 @@ extern "C" { flags: *mut PyCompilerFlags, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn Py_CompileStringExFlags( str: *const c_char, filename: *const c_char, @@ -135,16 +135,13 @@ extern "C" { } #[inline] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn Py_CompileString(string: *const c_char, p: *const c_char, s: c_int) -> *mut PyObject { - #[cfg(not(PyPy))] - return Py_CompileStringExFlags(string, p, s, std::ptr::null_mut(), -1); - - #[cfg(PyPy)] - Py_CompileStringFlags(string, p, s, std::ptr::null_mut()) + Py_CompileStringExFlags(string, p, s, std::ptr::null_mut(), -1) } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn Py_CompileStringFlags( string: *const c_char, p: *const c_char, @@ -164,11 +161,11 @@ extern "C" { g: *mut PyObject, l: *mut PyObject, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_AnyFile(fp: *mut FILE, name: *const c_char) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_AnyFileEx(fp: *mut FILE, name: *const c_char, closeit: c_int) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_AnyFileFlags( arg1: *mut FILE, arg2: *const c_char, @@ -176,13 +173,13 @@ extern "C" { ) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyRun_SimpleString")] pub fn PyRun_SimpleString(s: *const c_char) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_SimpleFile(f: *mut FILE, p: *const c_char) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_SimpleFileEx(f: *mut FILE, p: *const c_char, c: c_int) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_InteractiveOne(f: *mut FILE, p: *const c_char) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_InteractiveLoop(f: *mut FILE, p: *const c_char) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyRun_File")] pub fn PyRun_File( @@ -192,7 +189,7 @@ extern "C" { g: *mut PyObject, l: *mut PyObject, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_FileEx( fp: *mut FILE, p: *const c_char, @@ -201,7 +198,7 @@ extern "C" { l: *mut PyObject, c: c_int, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_FileFlags( fp: *mut FILE, p: *const c_char, @@ -218,14 +215,14 @@ extern "C" { // skipped macro PyRun_AnyFileFlags extern "C" { - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] pub fn PyParser_SimpleParseStringFlags( arg1: *const c_char, arg2: c_int, arg3: c_int, ) -> *mut _node; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] pub fn PyParser_SimpleParseStringFlagsFilename( arg1: *const c_char, @@ -233,7 +230,7 @@ extern "C" { arg3: c_int, arg4: c_int, ) -> *mut _node; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] pub fn PyParser_SimpleParseFileFlags( arg1: *mut FILE, diff --git a/pyo3-ffi/src/cpython/tupleobject.rs b/pyo3-ffi/src/cpython/tupleobject.rs index 24dde268526..dc1bf8e40d0 100644 --- a/pyo3-ffi/src/cpython/tupleobject.rs +++ b/pyo3-ffi/src/cpython/tupleobject.rs @@ -1,20 +1,23 @@ use crate::object::*; +#[cfg(Py_3_14)] +use crate::pyport::Py_hash_t; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; #[repr(C)] pub struct PyTupleObject { pub ob_base: PyVarObject, + #[cfg(Py_3_14)] + pub ob_hash: Py_hash_t, pub ob_item: [*mut PyObject; 1], } // skipped _PyTuple_Resize // skipped _PyTuple_MaybeUntrack -/// Macro, trading safety for speed - // skipped _PyTuple_CAST +/// Macro, trading safety for speed #[inline] #[cfg(not(PyPy))] pub unsafe fn PyTuple_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { @@ -22,14 +25,14 @@ pub unsafe fn PyTuple_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyTuple_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject { *(*(op as *mut PyTupleObject)).ob_item.as_ptr().offset(i) } /// Macro, *only* to be used to fill in brand new tuples #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyTuple_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) { *(*(op as *mut PyTupleObject)).ob_item.as_mut_ptr().offset(i) = v; } diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index f618ecf0a84..cf0ef54484e 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -1,9 +1,8 @@ -#[cfg(not(PyPy))] +#[cfg(any(Py_3_11, not(PyPy)))] use crate::Py_hash_t; -use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_UNICODE, Py_ssize_t}; -#[cfg(not(Py_3_12))] +use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_ssize_t}; use libc::wchar_t; -use std::os::raw::{c_char, c_int, c_uint, c_void}; +use std::ffi::{c_char, c_int, c_uint, c_void}; // skipped Py_UNICODE_ISSPACE() // skipped Py_UNICODE_ISLOWER() @@ -32,11 +31,13 @@ use std::os::raw::{c_char, c_int, c_uint, c_void}; // skipped Py_UNICODE_LOW_SURROGATE // generated by bindgen v0.63.0 (with small adaptations) +#[cfg(not(Py_3_14))] #[repr(C)] struct BitfieldUnit { storage: Storage, } +#[cfg(not(Py_3_14))] impl BitfieldUnit { #[inline] pub const fn new(storage: Storage) -> Self { @@ -44,6 +45,7 @@ impl BitfieldUnit { } } +#[cfg(not(any(GraalPy, Py_3_14)))] impl BitfieldUnit where Storage: AsRef<[u8]> + AsMut<[u8]>, @@ -117,23 +119,37 @@ where } } +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_INTERNED_INDEX: usize = 0; +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_INTERNED_WIDTH: u8 = 2; +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_KIND_INDEX: usize = STATE_INTERNED_WIDTH as usize; +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_KIND_WIDTH: u8 = 3; +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_COMPACT_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH) as usize; +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_COMPACT_WIDTH: u8 = 1; +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_ASCII_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH) as usize; +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_ASCII_WIDTH: u8 = 1; -#[cfg(not(Py_3_12))] +#[cfg(all(not(any(GraalPy, Py_3_14)), Py_3_12))] +const STATE_STATICALLY_ALLOCATED_INDEX: usize = + (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH + STATE_ASCII_WIDTH) as usize; +#[cfg(all(not(any(GraalPy, Py_3_14)), Py_3_12))] +const STATE_STATICALLY_ALLOCATED_WIDTH: u8 = 1; + +#[cfg(not(any(Py_3_12, GraalPy)))] const STATE_READY_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH + STATE_ASCII_WIDTH) as usize; -#[cfg(not(Py_3_12))] +#[cfg(not(any(Py_3_12, GraalPy)))] const STATE_READY_WIDTH: u8 = 1; // generated by bindgen v0.63.0 (with small adaptations) @@ -145,14 +161,15 @@ const STATE_READY_WIDTH: u8 = 1; /// /// Memory layout of C bitfields is implementation defined, so these functions are still /// unsafe. Users must verify that they work as expected on the architectures they target. +#[cfg(not(Py_3_14))] #[repr(C)] -#[repr(align(4))] struct PyASCIIObjectState { bitfield_align: [u8; 0], bitfield: BitfieldUnit<[u8; 4usize]>, } // c_uint and u32 are not necessarily the same type on all targets / architectures +#[cfg(not(any(GraalPy, Py_3_14)))] #[allow(clippy::useless_transmute)] impl PyASCIIObjectState { #[inline] @@ -206,6 +223,26 @@ impl PyASCIIObjectState { .set(STATE_ASCII_INDEX, STATE_ASCII_WIDTH, val as u64) } + #[cfg(Py_3_12)] + #[inline] + unsafe fn statically_allocated(&self) -> c_uint { + std::mem::transmute(self.bitfield.get( + STATE_STATICALLY_ALLOCATED_INDEX, + STATE_STATICALLY_ALLOCATED_WIDTH, + ) as u32) + } + + #[cfg(Py_3_12)] + #[inline] + unsafe fn set_statically_allocated(&mut self, val: c_uint) { + let val: u32 = std::mem::transmute(val); + self.bitfield.set( + STATE_STATICALLY_ALLOCATED_INDEX, + STATE_STATICALLY_ALLOCATED_WIDTH, + val as u64, + ) + } + #[cfg(not(Py_3_12))] #[inline] unsafe fn ready(&self) -> c_uint { @@ -221,6 +258,7 @@ impl PyASCIIObjectState { } } +#[cfg(not(Py_3_14))] impl From for PyASCIIObjectState { #[inline] fn from(value: u32) -> Self { @@ -231,6 +269,7 @@ impl From for PyASCIIObjectState { } } +#[cfg(not(Py_3_14))] impl From for u32 { #[inline] fn from(value: PyASCIIObjectState) -> Self { @@ -242,25 +281,36 @@ impl From for u32 { pub struct PyASCIIObject { pub ob_base: PyObject, pub length: Py_ssize_t, - #[cfg(not(PyPy))] + #[cfg(any(Py_3_11, not(PyPy)))] pub hash: Py_hash_t, /// A bit field with various properties. /// /// Rust doesn't expose bitfields. So we have accessor functions for /// retrieving values. /// + /// Before 3.12: /// unsigned int interned:2; // SSTATE_* constants. /// unsigned int kind:3; // PyUnicode_*_KIND constants. /// unsigned int compact:1; /// unsigned int ascii:1; /// unsigned int ready:1; /// unsigned int :24; + /// + /// 3.12, and 3.13 + /// unsigned int interned:2; // SSTATE_* constants. + /// unsigned int kind:3; // PyUnicode_*_KIND constants. + /// unsigned int compact:1; + /// unsigned int ascii:1; + /// unsigned int statically_allocated:1; + /// unsigned int :24; + /// on 3.14 and higher PyO3 doesn't access the internal state pub state: u32, #[cfg(not(Py_3_12))] pub wstr: *mut wchar_t, } /// Interacting with the bitfield is not actually well-defined, so we mark these APIs unsafe. +#[cfg(not(any(GraalPy, Py_3_14)))] impl PyASCIIObject { #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12 /// Get the `interned` field of the [`PyASCIIObject`] state bitfield. @@ -337,6 +387,7 @@ impl PyASCIIObject { /// /// Calling this function with an argument that is neither `0` nor `1` is invalid. #[inline] + #[cfg(not(all(Py_3_14, Py_GIL_DISABLED)))] pub unsafe fn set_ascii(&mut self, val: c_uint) { let mut state = PyASCIIObjectState::from(self.state); state.set_ascii(val); @@ -362,6 +413,26 @@ impl PyASCIIObject { state.set_ready(val); self.state = u32::from(state); } + + /// Get the `statically_allocated` field of the [`PyASCIIObject`] state bitfield. + /// + /// Returns either `0` or `1`. + #[inline] + #[cfg(Py_3_12)] + pub unsafe fn statically_allocated(&self) -> c_uint { + PyASCIIObjectState::from(self.state).statically_allocated() + } + + /// Set the `statically_allocated` flag of the [`PyASCIIObject`] state bitfield. + /// + /// Calling this function with an argument that is neither `0` nor `1` is invalid. + #[inline] + #[cfg(Py_3_12)] + pub unsafe fn set_statically_allocated(&mut self, val: c_uint) { + let mut state = PyASCIIObjectState::from(self.state); + state.set_statically_allocated(val); + self.state = u32::from(state); + } } #[repr(C)] @@ -388,7 +459,7 @@ pub struct PyUnicodeObject { } extern "C" { - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn _PyUnicode_CheckConsistency(op: *mut PyObject, check_content: c_int) -> c_int; } @@ -403,6 +474,7 @@ pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; #[cfg(Py_3_12)] pub const SSTATE_INTERNED_IMMORTAL_STATIC: c_uint = 3; +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -412,11 +484,13 @@ pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).ascii() } +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_IS_COMPACT(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).compact() } +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint { ((*(op as *mut PyASCIIObject)).ascii() != 0 && PyUnicode_IS_COMPACT(op) != 0).into() @@ -430,21 +504,31 @@ pub const PyUnicode_1BYTE_KIND: c_uint = 1; pub const PyUnicode_2BYTE_KIND: c_uint = 2; pub const PyUnicode_4BYTE_KIND: c_uint = 4; +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_1BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS1 { PyUnicode_DATA(op) as *mut Py_UCS1 } +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_2BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS2 { PyUnicode_DATA(op) as *mut Py_UCS2 } +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { PyUnicode_DATA(op) as *mut Py_UCS4 } +#[cfg(all(not(GraalPy), Py_3_14))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyUnicode_KIND")] + pub fn PyUnicode_KIND(op: *mut PyObject) -> c_uint; +} + +#[cfg(all(not(GraalPy), not(Py_3_14)))] #[inline] pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -454,6 +538,7 @@ pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).kind() } +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { if PyUnicode_IS_ASCII(op) != 0 { @@ -463,6 +548,7 @@ pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { } } +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(!(*(op as *mut PyUnicodeObject)).data.any.is_null()); @@ -470,6 +556,7 @@ pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { (*(op as *mut PyUnicodeObject)).data.any } +#[cfg(not(any(GraalPy, PyPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -481,10 +568,18 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { } } +#[cfg(Py_3_14)] +#[cfg(all(not(GraalPy), Py_3_14))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyUnicode_DATA")] + pub fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void; +} + // skipped PyUnicode_WRITE // skipped PyUnicode_READ // skipped PyUnicode_READ_CHAR +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -494,26 +589,26 @@ pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { (*(op as *mut PyASCIIObject)).length } -#[cfg(Py_3_12)] +#[cfg(any(Py_3_12, GraalPy))] #[inline] pub unsafe fn PyUnicode_IS_READY(_op: *mut PyObject) -> c_uint { // kept in CPython for backwards compatibility 1 } -#[cfg(not(Py_3_12))] +#[cfg(not(any(GraalPy, Py_3_12)))] #[inline] pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).ready() } -#[cfg(Py_3_12)] +#[cfg(any(Py_3_12, GraalPy))] #[inline] pub unsafe fn PyUnicode_READY(_op: *mut PyObject) -> c_int { 0 } -#[cfg(not(Py_3_12))] +#[cfg(not(any(Py_3_12, GraalPy)))] #[inline] pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -561,7 +656,7 @@ extern "C" { #[cfg(not(Py_3_12))] #[deprecated] #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromUnicode")] - pub fn PyUnicode_FromUnicode(u: *const Py_UNICODE, size: Py_ssize_t) -> *mut PyObject; + pub fn PyUnicode_FromUnicode(u: *const wchar_t, size: Py_ssize_t) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyUnicode_FromKindAndData")] pub fn PyUnicode_FromKindAndData( @@ -576,7 +671,7 @@ extern "C" { #[cfg(not(Py_3_12))] #[deprecated] #[cfg_attr(PyPy, link_name = "PyPyUnicode_AsUnicode")] - pub fn PyUnicode_AsUnicode(unicode: *mut PyObject) -> *mut Py_UNICODE; + pub fn PyUnicode_AsUnicode(unicode: *mut PyObject) -> *mut wchar_t; // skipped _PyUnicode_AsUnicode @@ -586,7 +681,7 @@ extern "C" { pub fn PyUnicode_AsUnicodeAndSize( unicode: *mut PyObject, size: *mut Py_ssize_t, - ) -> *mut Py_UNICODE; + ) -> *mut wchar_t; // skipped PyUnicode_GetMax } @@ -615,14 +710,14 @@ extern "C" { // skipped _PyUnicode_AsString pub fn PyUnicode_Encode( - s: *const Py_UNICODE, + s: *const wchar_t, size: Py_ssize_t, encoding: *const c_char, errors: *const c_char, ) -> *mut PyObject; pub fn PyUnicode_EncodeUTF7( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, base64SetO: c_int, base64WhiteSpace: c_int, @@ -634,13 +729,13 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeUTF8")] pub fn PyUnicode_EncodeUTF8( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, ) -> *mut PyObject; pub fn PyUnicode_EncodeUTF32( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, byteorder: c_int, @@ -649,7 +744,7 @@ extern "C" { // skipped _PyUnicode_EncodeUTF32 pub fn PyUnicode_EncodeUTF16( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, byteorder: c_int, @@ -658,13 +753,11 @@ extern "C" { // skipped _PyUnicode_EncodeUTF16 // skipped _PyUnicode_DecodeUnicodeEscape - pub fn PyUnicode_EncodeUnicodeEscape( - data: *const Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; + pub fn PyUnicode_EncodeUnicodeEscape(data: *const wchar_t, length: Py_ssize_t) + -> *mut PyObject; pub fn PyUnicode_EncodeRawUnicodeEscape( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, ) -> *mut PyObject; @@ -672,7 +765,7 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeLatin1")] pub fn PyUnicode_EncodeLatin1( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, ) -> *mut PyObject; @@ -681,13 +774,13 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeASCII")] pub fn PyUnicode_EncodeASCII( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, errors: *const c_char, ) -> *mut PyObject; pub fn PyUnicode_EncodeCharmap( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, mapping: *mut PyObject, errors: *const c_char, @@ -696,7 +789,7 @@ extern "C" { // skipped _PyUnicode_EncodeCharmap pub fn PyUnicode_TranslateCharmap( - data: *const Py_UNICODE, + data: *const wchar_t, length: Py_ssize_t, table: *mut PyObject, errors: *const c_char, @@ -706,17 +799,14 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyUnicode_EncodeDecimal")] pub fn PyUnicode_EncodeDecimal( - s: *mut Py_UNICODE, + s: *mut wchar_t, length: Py_ssize_t, output: *mut c_char, errors: *const c_char, ) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyUnicode_TransformDecimalToASCII")] - pub fn PyUnicode_TransformDecimalToASCII( - s: *mut Py_UNICODE, - length: Py_ssize_t, - ) -> *mut PyObject; + pub fn PyUnicode_TransformDecimalToASCII(s: *mut wchar_t, length: Py_ssize_t) -> *mut PyObject; // skipped _PyUnicode_TransformDecimalAndSpaceToASCII } diff --git a/pyo3-ffi/src/cpython/weakrefobject.rs b/pyo3-ffi/src/cpython/weakrefobject.rs index 5a5f85c5f0c..1c50c7a759f 100644 --- a/pyo3-ffi/src/cpython/weakrefobject.rs +++ b/pyo3-ffi/src/cpython/weakrefobject.rs @@ -1,4 +1,5 @@ -#[cfg(not(PyPy))] +// NB publicly re-exported in `src/weakrefobject.rs` +#[cfg(not(any(PyPy, GraalPy)))] pub struct _PyWeakReference { pub ob_base: crate::PyObject, pub wr_object: *mut crate::PyObject, @@ -8,6 +9,8 @@ pub struct _PyWeakReference { pub wr_next: *mut crate::PyWeakReference, #[cfg(Py_3_11)] pub vectorcall: Option, + #[cfg(all(Py_3_13, Py_GIL_DISABLED))] + pub weakrefs_lock: *mut crate::PyMutex, } // skipped _PyWeakref_GetWeakrefCount diff --git a/pyo3-ffi/src/datetime.rs b/pyo3-ffi/src/datetime.rs index 7e5a250990f..dc3a8b32dec 100644 --- a/pyo3-ffi/src/datetime.rs +++ b/pyo3-ffi/src/datetime.rs @@ -3,30 +3,26 @@ //! This is the unsafe thin wrapper around the [CPython C API](https://docs.python.org/3/c-api/datetime.html), //! and covers the various date and time related objects in the Python `datetime` //! standard library module. -//! -//! A note regarding PyPy (cpyext) support: -//! -//! Support for `PyDateTime_CAPI` is limited as of PyPy 7.0.0. -//! `DateTime_FromTimestamp` and `Date_FromTimestamp` are currently not supported. +#[cfg(not(PyPy))] +use crate::PyCapsule_Import; +#[cfg(GraalPy)] +use crate::{PyLong_AsLong, PyLong_Check, PyObject_GetAttrString, Py_DecRef}; use crate::{PyObject, PyObject_TypeCheck, PyTypeObject, Py_TYPE}; -use std::cell::UnsafeCell; -use std::os::raw::{c_char, c_int}; +use std::ffi::c_char; +use std::ffi::c_int; use std::ptr; +use std::sync::Once; +use std::{cell::UnsafeCell, ffi::CStr}; #[cfg(not(PyPy))] -use { - crate::{PyCapsule_Import, Py_hash_t}, - std::ffi::CString, - std::os::raw::c_uchar, -}; - +use {crate::Py_hash_t, std::ffi::c_uchar}; // Type struct wrappers const _PyDateTime_DATE_DATASIZE: usize = 4; const _PyDateTime_TIME_DATASIZE: usize = 6; const _PyDateTime_DATETIME_DATASIZE: usize = 10; #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.timedelta`. pub struct PyDateTime_Delta { pub ob_base: PyObject, @@ -40,21 +36,19 @@ pub struct PyDateTime_Delta { // skipped non-limited PyDateTime_TZInfo // skipped non-limited _PyDateTime_BaseTZInfo -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.time` without a `tzinfo` member. pub struct _PyDateTime_BaseTime { pub ob_base: PyObject, - #[cfg(not(PyPy))] pub hashcode: Py_hash_t, pub hastzinfo: c_char, - #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_TIME_DATASIZE], } #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.time`. pub struct PyDateTime_Time { pub ob_base: PyObject, @@ -73,7 +67,7 @@ pub struct PyDateTime_Time { } #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.date` pub struct PyDateTime_Date { pub ob_base: PyObject, @@ -85,21 +79,19 @@ pub struct PyDateTime_Date { pub data: [c_uchar; _PyDateTime_DATE_DATASIZE], } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.datetime` without a `tzinfo` member. pub struct _PyDateTime_BaseDateTime { pub ob_base: PyObject, - #[cfg(not(PyPy))] pub hashcode: Py_hash_t, pub hastzinfo: c_char, - #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATETIME_DATASIZE], } #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] /// Structure representing a `datetime.datetime`. pub struct PyDateTime_DateTime { pub ob_base: PyObject, @@ -121,56 +113,56 @@ pub struct PyDateTime_DateTime { // Accessor functions for PyDateTime_Date and PyDateTime_DateTime #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the year component of a `PyDateTime_Date` or `PyDateTime_DateTime`. /// Returns a signed integer greater than 0. pub unsafe fn PyDateTime_GET_YEAR(o: *mut PyObject) -> c_int { // This should work for Date or DateTime - let d = *(o as *mut PyDateTime_Date); - c_int::from(d.data[0]) << 8 | c_int::from(d.data[1]) + let data = (*(o as *mut PyDateTime_Date)).data; + (c_int::from(data[0]) << 8) | c_int::from(data[1]) } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the month component of a `PyDateTime_Date` or `PyDateTime_DateTime`. /// Returns a signed integer in the range `[1, 12]`. pub unsafe fn PyDateTime_GET_MONTH(o: *mut PyObject) -> c_int { - let d = *(o as *mut PyDateTime_Date); - c_int::from(d.data[2]) + let data = (*(o as *mut PyDateTime_Date)).data; + c_int::from(data[2]) } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the day component of a `PyDateTime_Date` or `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[1, 31]`. pub unsafe fn PyDateTime_GET_DAY(o: *mut PyObject) -> c_int { - let d = *(o as *mut PyDateTime_Date); - c_int::from(d.data[3]) + let data = (*(o as *mut PyDateTime_Date)).data; + c_int::from(data[3]) } // Accessor macros for times -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_HOUR { ($o: expr, $offset:expr) => { c_int::from((*$o).data[$offset + 0]) }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_MINUTE { ($o: expr, $offset:expr) => { c_int::from((*$o).data[$offset + 1]) }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_SECOND { ($o: expr, $offset:expr) => { c_int::from((*$o).data[$offset + 2]) }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_MICROSECOND { ($o: expr, $offset:expr) => { (c_int::from((*$o).data[$offset + 3]) << 16) @@ -179,14 +171,14 @@ macro_rules! _PyDateTime_GET_MICROSECOND { }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_FOLD { ($o: expr) => { (*$o).fold }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_TZINFO { ($o: expr) => { if (*$o).hastzinfo != 0 { @@ -199,7 +191,7 @@ macro_rules! _PyDateTime_GET_TZINFO { // Accessor functions for DateTime #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the hour component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 23]` pub unsafe fn PyDateTime_DATE_GET_HOUR(o: *mut PyObject) -> c_int { @@ -207,7 +199,7 @@ pub unsafe fn PyDateTime_DATE_GET_HOUR(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the minute component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 59]` pub unsafe fn PyDateTime_DATE_GET_MINUTE(o: *mut PyObject) -> c_int { @@ -215,7 +207,7 @@ pub unsafe fn PyDateTime_DATE_GET_MINUTE(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the second component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 59]` pub unsafe fn PyDateTime_DATE_GET_SECOND(o: *mut PyObject) -> c_int { @@ -223,7 +215,7 @@ pub unsafe fn PyDateTime_DATE_GET_SECOND(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the microsecond component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 999999]` pub unsafe fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int { @@ -231,7 +223,7 @@ pub unsafe fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the fold component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 1]` pub unsafe fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_uchar { @@ -239,7 +231,7 @@ pub unsafe fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_uchar { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the tzinfo component of a `PyDateTime_DateTime`. /// Returns a pointer to a `PyObject` that should be either NULL or an instance /// of a `datetime.tzinfo` subclass. @@ -249,7 +241,7 @@ pub unsafe fn PyDateTime_DATE_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { // Accessor functions for Time #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the hour component of a `PyDateTime_Time`. /// Returns a signed integer in the interval `[0, 23]` pub unsafe fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int { @@ -257,7 +249,7 @@ pub unsafe fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the minute component of a `PyDateTime_Time`. /// Returns a signed integer in the interval `[0, 59]` pub unsafe fn PyDateTime_TIME_GET_MINUTE(o: *mut PyObject) -> c_int { @@ -265,7 +257,7 @@ pub unsafe fn PyDateTime_TIME_GET_MINUTE(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the second component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 59]` pub unsafe fn PyDateTime_TIME_GET_SECOND(o: *mut PyObject) -> c_int { @@ -273,14 +265,14 @@ pub unsafe fn PyDateTime_TIME_GET_SECOND(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the microsecond component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 999999]` pub unsafe fn PyDateTime_TIME_GET_MICROSECOND(o: *mut PyObject) -> c_int { _PyDateTime_GET_MICROSECOND!((o as *mut PyDateTime_Time), 0) } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[inline] /// Retrieve the fold component of a `PyDateTime_Time`. /// Returns a signed integer in the interval `[0, 1]` @@ -289,7 +281,7 @@ pub unsafe fn PyDateTime_TIME_GET_FOLD(o: *mut PyObject) -> c_uchar { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the tzinfo component of a `PyDateTime_Time`. /// Returns a pointer to a `PyObject` that should be either NULL or an instance /// of a `datetime.tzinfo` subclass. @@ -298,7 +290,7 @@ pub unsafe fn PyDateTime_TIME_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { } // Accessor functions -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _access_field { ($obj:expr, $type: ident, $field:ident) => { (*($obj as *mut $type)).$field @@ -306,7 +298,7 @@ macro_rules! _access_field { } // Accessor functions for PyDateTime_Delta -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _access_delta_field { ($obj:expr, $field:ident) => { _access_field!($obj, PyDateTime_Delta, $field) @@ -314,7 +306,7 @@ macro_rules! _access_delta_field { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the days component of a `PyDateTime_Delta`. /// /// Returns a signed integer in the interval [-999999999, 999999999]. @@ -326,7 +318,7 @@ pub unsafe fn PyDateTime_DELTA_GET_DAYS(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the seconds component of a `PyDateTime_Delta`. /// /// Returns a signed integer in the interval [0, 86399]. @@ -338,7 +330,7 @@ pub unsafe fn PyDateTime_DELTA_GET_SECONDS(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the seconds component of a `PyDateTime_Delta`. /// /// Returns a signed integer in the interval [0, 999999]. @@ -349,6 +341,132 @@ pub unsafe fn PyDateTime_DELTA_GET_MICROSECONDS(o: *mut PyObject) -> c_int { _access_delta_field!(o, microseconds) } +// Accessor functions for GraalPy. The macros on GraalPy work differently, +// but copying them seems suboptimal +#[inline] +#[cfg(GraalPy)] +pub unsafe fn _get_attr(obj: *mut PyObject, field: &std::ffi::CStr) -> c_int { + let result = PyObject_GetAttrString(obj, field.as_ptr()); + Py_DecRef(result); // the original macros are borrowing + if PyLong_Check(result) == 1 { + PyLong_AsLong(result) as c_int + } else { + 0 + } +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_GET_YEAR(o: *mut PyObject) -> c_int { + _get_attr(o, c"year") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_GET_MONTH(o: *mut PyObject) -> c_int { + _get_attr(o, c"month") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_GET_DAY(o: *mut PyObject) -> c_int { + _get_attr(o, c"day") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_HOUR(o: *mut PyObject) -> c_int { + _get_attr(o, c"hour") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_MINUTE(o: *mut PyObject) -> c_int { + _get_attr(o, c"minute") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_SECOND(o: *mut PyObject) -> c_int { + _get_attr(o, c"second") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int { + _get_attr(o, c"microsecond") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_int { + _get_attr(o, c"fold") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { + let res = PyObject_GetAttrString(o, c"tzinfo".as_ptr().cast()); + Py_DecRef(res); // the original macros are borrowing + res +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int { + _get_attr(o, c"hour") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_MINUTE(o: *mut PyObject) -> c_int { + _get_attr(o, c"minute") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_SECOND(o: *mut PyObject) -> c_int { + _get_attr(o, c"second") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_MICROSECOND(o: *mut PyObject) -> c_int { + _get_attr(o, c"microsecond") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_FOLD(o: *mut PyObject) -> c_int { + _get_attr(o, c"fold") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { + let res = PyObject_GetAttrString(o, c"tzinfo".as_ptr().cast()); + Py_DecRef(res); // the original macros are borrowing + res +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DELTA_GET_DAYS(o: *mut PyObject) -> c_int { + _get_attr(o, c"days") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DELTA_GET_SECONDS(o: *mut PyObject) -> c_int { + _get_attr(o, c"seconds") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DELTA_GET_MICROSECONDS(o: *mut PyObject) -> c_int { + _get_attr(o, c"microseconds") +} + #[cfg(PyPy)] extern "C" { // skipped _PyDateTime_HAS_TZINFO (not in PyPy) @@ -369,7 +487,9 @@ extern "C" { pub fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int; #[link_name = "PyPyDateTime_GET_FOLD"] pub fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_int; - // skipped PyDateTime_DATE_GET_TZINFO (not in PyPy) + #[link_name = "PyPyDateTime_DATE_GET_TZINFO"] + #[cfg(Py_3_10)] + pub fn PyDateTime_DATE_GET_TZINFO(o: *mut PyObject) -> *mut PyObject; #[link_name = "PyPyDateTime_TIME_GET_HOUR"] pub fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int; @@ -381,7 +501,9 @@ extern "C" { pub fn PyDateTime_TIME_GET_MICROSECOND(o: *mut PyObject) -> c_int; #[link_name = "PyPyDateTime_TIME_GET_FOLD"] pub fn PyDateTime_TIME_GET_FOLD(o: *mut PyObject) -> c_int; - // skipped PyDateTime_TIME_GET_TZINFO (not in PyPy) + #[link_name = "PyPyDateTime_TIME_GET_TZINFO"] + #[cfg(Py_3_10)] + pub fn PyDateTime_TIME_GET_TZINFO(o: *mut PyObject) -> *mut PyObject; #[link_name = "PyPyDateTime_DELTA_GET_DAYS"] pub fn PyDateTime_DELTA_GET_DAYS(o: *mut PyObject) -> c_int; @@ -468,6 +590,8 @@ pub struct PyDateTime_CAPI { // Python already shares this object between threads, so it's no more evil for us to do it too! unsafe impl Sync for PyDateTime_CAPI {} +pub const PyDateTime_CAPSULE_NAME: &CStr = c"datetime.datetime_CAPI"; + /// Returns a pointer to a `PyDateTime_CAPI` instance /// /// # Note @@ -475,33 +599,38 @@ unsafe impl Sync for PyDateTime_CAPI {} /// `PyDateTime_IMPORT` is called #[inline] pub unsafe fn PyDateTimeAPI() -> *mut PyDateTime_CAPI { - *PyDateTimeAPI_impl.0.get() -} - -#[inline] -pub unsafe fn PyDateTime_TimeZone_UTC() -> *mut PyObject { - (*PyDateTimeAPI()).TimeZone_UTC + *PyDateTimeAPI_impl.ptr.get() } /// Populates the `PyDateTimeAPI` object pub unsafe fn PyDateTime_IMPORT() { - // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use - // `PyCapsule_Import` will behave unexpectedly in pypy. - #[cfg(PyPy)] - let py_datetime_c_api = PyDateTime_Import(); - - #[cfg(not(PyPy))] - let py_datetime_c_api = { - // PyDateTime_CAPSULE_NAME is a macro in C - let PyDateTime_CAPSULE_NAME = CString::new("datetime.datetime_CAPI").unwrap(); - - PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) as *mut PyDateTime_CAPI - }; + if !PyDateTimeAPI_impl.once.is_completed() { + // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use + // `PyCapsule_Import` will behave unexpectedly in pypy. + #[cfg(PyPy)] + let py_datetime_c_api = PyDateTime_Import(); + + #[cfg(not(PyPy))] + let py_datetime_c_api = + PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) as *mut PyDateTime_CAPI; + + if py_datetime_c_api.is_null() { + return; + } - *PyDateTimeAPI_impl.0.get() = py_datetime_c_api; + // Protect against race conditions when the datetime API is concurrently + // initialized in multiple threads. UnsafeCell.get() cannot panic so this + // won't panic either. + PyDateTimeAPI_impl.once.call_once(|| { + *PyDateTimeAPI_impl.ptr.get() = py_datetime_c_api; + }); + } } -// skipped non-limited PyDateTime_TimeZone_UTC +#[inline] +pub unsafe fn PyDateTime_TimeZone_UTC() -> *mut PyObject { + (*PyDateTimeAPI()).TimeZone_UTC +} /// Type Check macros /// @@ -614,8 +743,13 @@ extern "C" { // Rust specific implementation details -struct PyDateTimeAPISingleton(UnsafeCell<*mut PyDateTime_CAPI>); +struct PyDateTimeAPISingleton { + once: Once, + ptr: UnsafeCell<*mut PyDateTime_CAPI>, +} unsafe impl Sync for PyDateTimeAPISingleton {} -static PyDateTimeAPI_impl: PyDateTimeAPISingleton = - PyDateTimeAPISingleton(UnsafeCell::new(ptr::null_mut())); +static PyDateTimeAPI_impl: PyDateTimeAPISingleton = PyDateTimeAPISingleton { + once: Once::new(), + ptr: UnsafeCell::new(ptr::null_mut()), +}; diff --git a/pyo3-ffi/src/descrobject.rs b/pyo3-ffi/src/descrobject.rs index f4a5ce00bf6..8a4d5839a37 100644 --- a/pyo3-ffi/src/descrobject.rs +++ b/pyo3-ffi/src/descrobject.rs @@ -1,7 +1,7 @@ use crate::methodobject::PyMethodDef; use crate::object::{PyObject, PyTypeObject}; use crate::Py_ssize_t; -use std::os::raw::{c_char, c_int, c_void}; +use std::ffi::{c_char, c_int, c_void}; use std::ptr; pub type getter = unsafe extern "C" fn(slf: *mut PyObject, closure: *mut c_void) -> *mut PyObject; @@ -14,7 +14,7 @@ pub type setter = /// Note that CPython may leave fields uninitialized. You must ensure that /// `name` != NULL before dereferencing or reading other fields. #[repr(C)] -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug)] pub struct PyGetSetDef { pub name: *const c_char, pub get: Option, diff --git a/pyo3-ffi/src/dictobject.rs b/pyo3-ffi/src/dictobject.rs index 8d522df97e2..d8419ba4317 100644 --- a/pyo3-ffi/src/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -1,6 +1,6 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -66,6 +66,13 @@ extern "C" { ) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyDict_DelItemString")] pub fn PyDict_DelItemString(dp: *mut PyObject, key: *const c_char) -> c_int; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyDict_GetItemRef")] + pub fn PyDict_GetItemRef( + dp: *mut PyObject, + key: *mut PyObject, + result: *mut *mut PyObject, + ) -> c_int; // skipped 3.10 / ex-non-limited PyObject_GenericGetDict } @@ -109,6 +116,6 @@ extern "C" { pub static mut PyDictRevIterItem_Type: PyTypeObject; } -#[cfg(any(PyPy, Py_LIMITED_API))] +#[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) -opaque_struct!(PyDictObject); +opaque_struct!(pub PyDictObject); diff --git a/pyo3-ffi/src/fileobject.rs b/pyo3-ffi/src/fileobject.rs index 91618ab7fd2..ec3a0409930 100644 --- a/pyo3-ffi/src/fileobject.rs +++ b/pyo3-ffi/src/fileobject.rs @@ -1,5 +1,5 @@ use crate::object::PyObject; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; pub const PY_STDIOTEXTMODE: &str = "b"; diff --git a/pyo3-ffi/src/fileutils.rs b/pyo3-ffi/src/fileutils.rs index 3f053b72b51..649751babf5 100644 --- a/pyo3-ffi/src/fileutils.rs +++ b/pyo3-ffi/src/fileutils.rs @@ -1,6 +1,6 @@ use crate::pyport::Py_ssize_t; use libc::wchar_t; -use std::os::raw::c_char; +use std::ffi::c_char; extern "C" { pub fn Py_DecodeLocale(arg1: *const c_char, size: *mut Py_ssize_t) -> *mut wchar_t; diff --git a/pyo3-ffi/src/floatobject.rs b/pyo3-ffi/src/floatobject.rs index 65fc1d4c316..18bbc8bece0 100644 --- a/pyo3-ffi/src/floatobject.rs +++ b/pyo3-ffi/src/floatobject.rs @@ -1,10 +1,10 @@ use crate::object::*; -use std::os::raw::{c_double, c_int}; +use std::ffi::{c_double, c_int}; use std::ptr::addr_of_mut; #[cfg(Py_LIMITED_API)] // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) -opaque_struct!(PyFloatObject); +opaque_struct!(pub PyFloatObject); #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/pyo3-ffi/src/genericaliasobject.rs b/pyo3-ffi/src/genericaliasobject.rs new file mode 100644 index 00000000000..7979d7d863e --- /dev/null +++ b/pyo3-ffi/src/genericaliasobject.rs @@ -0,0 +1,12 @@ +#[cfg(Py_3_9)] +use crate::object::{PyObject, PyTypeObject}; + +#[cfg_attr(windows, link(name = "pythonXY"))] +extern "C" { + #[cfg(Py_3_9)] + #[cfg_attr(PyPy, link_name = "PyPy_GenericAlias")] + pub fn Py_GenericAlias(origin: *mut PyObject, args: *mut PyObject) -> *mut PyObject; + + #[cfg(Py_3_9)] + pub static mut Py_GenericAliasType: PyTypeObject; +} diff --git a/pyo3-ffi/src/impl_/mod.rs b/pyo3-ffi/src/impl_/mod.rs new file mode 100644 index 00000000000..064df213ba6 --- /dev/null +++ b/pyo3-ffi/src/impl_/mod.rs @@ -0,0 +1,22 @@ +#[cfg(Py_GIL_DISABLED)] +mod atomic_c_ulong { + pub struct GetAtomicCULong(); + + pub trait AtomicCULongType { + type Type; + } + impl AtomicCULongType for GetAtomicCULong<32> { + type Type = std::sync::atomic::AtomicU32; + } + impl AtomicCULongType for GetAtomicCULong<64> { + type Type = std::sync::atomic::AtomicU64; + } + + pub type TYPE = + () * 8 }> as AtomicCULongType>::Type; +} + +/// Typedef for an atomic integer to match the platform-dependent c_ulong type. +#[cfg(Py_GIL_DISABLED)] +#[doc(hidden)] +pub type AtomicCULong = atomic_c_ulong::TYPE; diff --git a/pyo3-ffi/src/import.rs b/pyo3-ffi/src/import.rs index e00843466e8..ce224af28e1 100644 --- a/pyo3-ffi/src/import.rs +++ b/pyo3-ffi/src/import.rs @@ -1,5 +1,5 @@ use crate::object::PyObject; -use std::os::raw::{c_char, c_int, c_long}; +use std::ffi::{c_char, c_int, c_long}; extern "C" { pub fn PyImport_GetMagicNumber() -> c_long; @@ -30,6 +30,9 @@ extern "C" { pub fn PyImport_AddModuleObject(name: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyImport_AddModule")] pub fn PyImport_AddModule(name: *const c_char) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyImport_AddModuleRef")] + pub fn PyImport_AddModuleRef(name: *const c_char) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyImport_ImportModule")] pub fn PyImport_ImportModule(name: *const c_char) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyImport_ImportModuleNoBlock")] diff --git a/pyo3-ffi/src/intrcheck.rs b/pyo3-ffi/src/intrcheck.rs index 3d8a58f7f2f..05400557d5a 100644 --- a/pyo3-ffi/src/intrcheck.rs +++ b/pyo3-ffi/src/intrcheck.rs @@ -1,4 +1,4 @@ -use std::os::raw::c_int; +use std::ffi::c_int; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyOS_InterruptOccurred")] diff --git a/pyo3-ffi/src/iterobject.rs b/pyo3-ffi/src/iterobject.rs index aa0c7b26db1..04a28d4826f 100644 --- a/pyo3-ffi/src/iterobject.rs +++ b/pyo3-ffi/src/iterobject.rs @@ -1,5 +1,5 @@ use crate::object::*; -use std::os::raw::c_int; +use std::ffi::c_int; use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs index 5e241fbb2c7..3faf4d4c2c3 100644 --- a/pyo3-ffi/src/lib.rs +++ b/pyo3-ffi/src/lib.rs @@ -1,3 +1,4 @@ +#![cfg_attr(docsrs, feature(doc_cfg))] //! Raw FFI declarations for Python's C API. //! //! PyO3 can be used to write native Python modules or run Python code and modules from Rust. @@ -16,7 +17,7 @@ //! generally the following apply: //! - Pointer arguments have to point to a valid Python object of the correct type, //! although null pointers are sometimes valid input. -//! - The vast majority can only be used safely while the GIL is held. +//! - The vast majority can only be used safely while the thread is attached to the Python interpreter. //! - Some functions have additional safety requirements, consult the //! [Python/C API Reference Manual][capi] //! for more information. @@ -42,16 +43,46 @@ //! PyO3 uses `rustc`'s `--cfg` flags to enable or disable code used for different Python versions. //! If you want to do this for your own crate, you can do so with the [`pyo3-build-config`] crate. //! -//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`: Marks code that is only enabled when -//! compiling for a given minimum Python version. +//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`, `Py_3_11`, `Py_3_12`, `Py_3_13`: Marks code that is +//! only enabled when compiling for a given minimum Python version. //! - `Py_LIMITED_API`: Marks code enabled when the `abi3` feature flag is enabled. +//! - `Py_GIL_DISABLED`: Marks code that runs only in the free-threaded build of CPython. //! - `PyPy` - Marks code enabled when compiling for PyPy. +//! - `GraalPy` - Marks code enabled when compiling for GraalPy. +//! +//! Additionally, you can query for the values `Py_DEBUG`, `Py_REF_DEBUG`, +//! `Py_TRACE_REFS`, and `COUNT_ALLOCS` from `py_sys_config` to query for the +//! corresponding C build-time defines. For example, to conditionally define +//! debug code using `Py_DEBUG`, you could do: +//! +//! ```rust,ignore +//! #[cfg(py_sys_config = "Py_DEBUG")] +//! println!("only runs if python was compiled with Py_DEBUG") +//! ``` +//! +//! To use these attributes, add [`pyo3-build-config`] as a build dependency in +//! your `Cargo.toml`: +//! +//! ```toml +//! [build-dependencies] +#![doc = concat!("pyo3-build-config =\"", env!("CARGO_PKG_VERSION"), "\"")] +//! ``` +//! +//! And then either create a new `build.rs` file in the project root or modify +//! the existing `build.rs` file to call `use_pyo3_cfgs()`: +//! +//! ```rust,ignore +//! fn main() { +//! pyo3_build_config::use_pyo3_cfgs(); +//! } +//! ``` //! //! # Minimum supported Rust and Python versions //! -//! PyO3 supports the following software versions: -//! - Python 3.7 and up (CPython and PyPy) -//! - Rust 1.56 and up +//! `pyo3-ffi` supports the following Python distributions: +//! - CPython 3.7 or greater +//! - PyPy 7.3 (Python 3.11+) +//! - GraalPy 24.0 or greater (Python 3.10+) //! //! # Example: Building Python Native modules //! @@ -77,99 +108,148 @@ //! [dependencies.pyo3-ffi] #![doc = concat!("version = \"", env!("CARGO_PKG_VERSION"), "\"")] //! features = ["extension-module"] +//! +//! [build-dependencies] +//! # This is only necessary if you need to configure your build based on +//! # the Python version or the compile-time configuration for the interpreter. +#![doc = concat!("pyo3_build_config = \"", env!("CARGO_PKG_VERSION"), "\"")] +//! ``` +//! +//! If you need to use conditional compilation based on Python version or how +//! Python was compiled, you need to add `pyo3-build-config` as a +//! `build-dependency` in your `Cargo.toml` as in the example above and either +//! create a new `build.rs` file or modify an existing one so that +//! `pyo3_build_config::use_pyo3_cfgs()` gets called at build time: +//! +//! **`build.rs`** +//! ```rust,ignore +//! fn main() { +//! pyo3_build_config::use_pyo3_cfgs() +//! } //! ``` //! //! **`src/lib.rs`** -//! ```rust -//! use std::os::raw::c_char; +//! ```rust,no_run +//! use std::ffi::{c_char, c_long}; //! use std::ptr; //! //! use pyo3_ffi::*; //! //! static mut MODULE_DEF: PyModuleDef = PyModuleDef { //! m_base: PyModuleDef_HEAD_INIT, -//! m_name: "string_sum\0".as_ptr().cast::(), -//! m_doc: "A Python module written in Rust.\0" -//! .as_ptr() -//! .cast::(), +//! m_name: c"string_sum".as_ptr(), +//! m_doc: c"A Python module written in Rust.".as_ptr(), //! m_size: 0, -//! m_methods: unsafe { METHODS.as_mut_ptr().cast() }, +//! m_methods: unsafe { METHODS as *const [PyMethodDef] as *mut PyMethodDef }, //! m_slots: std::ptr::null_mut(), //! m_traverse: None, //! m_clear: None, //! m_free: None, //! }; //! -//! static mut METHODS: [PyMethodDef; 2] = [ +//! static mut METHODS: &[PyMethodDef] = &[ //! PyMethodDef { -//! ml_name: "sum_as_string\0".as_ptr().cast::(), +//! ml_name: c"sum_as_string".as_ptr(), //! ml_meth: PyMethodDefPointer { -//! _PyCFunctionFast: sum_as_string, +//! PyCFunctionFast: sum_as_string, //! }, //! ml_flags: METH_FASTCALL, -//! ml_doc: "returns the sum of two integers as a string\0" -//! .as_ptr() -//! .cast::(), +//! ml_doc: c"returns the sum of two integers as a string".as_ptr(), //! }, //! // A zeroed PyMethodDef to mark the end of the array. -//! PyMethodDef::zeroed() +//! PyMethodDef::zeroed(), //! ]; //! //! // The module initialization function, which must be named `PyInit_`. //! #[allow(non_snake_case)] //! #[no_mangle] //! pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { -//! PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)) +//! let module = PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)); +//! if module.is_null() { +//! return module; +//! } +//! #[cfg(Py_GIL_DISABLED)] +//! { +//! if PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED) < 0 { +//! Py_DECREF(module); +//! return std::ptr::null_mut(); +//! } +//! } +//! module //! } //! -//! pub unsafe extern "C" fn sum_as_string( -//! _self: *mut PyObject, -//! args: *mut *mut PyObject, -//! nargs: Py_ssize_t, -//! ) -> *mut PyObject { -//! if nargs != 2 { -//! PyErr_SetString( -//! PyExc_TypeError, -//! "sum_as_string() expected 2 positional arguments\0" -//! .as_ptr() -//! .cast::(), +//! /// A helper to parse function arguments +//! /// If we used PyO3's proc macros they'd handle all of this boilerplate for us :) +//! unsafe fn parse_arg_as_i32(obj: *mut PyObject, n_arg: usize) -> Option { +//! if PyLong_Check(obj) == 0 { +//! let msg = format!( +//! "sum_as_string expected an int for positional argument {}\0", +//! n_arg //! ); -//! return std::ptr::null_mut(); +//! PyErr_SetString(PyExc_TypeError, msg.as_ptr().cast::()); +//! return None; //! } //! -//! let arg1 = *args; -//! if PyLong_Check(arg1) == 0 { -//! PyErr_SetString( -//! PyExc_TypeError, -//! "sum_as_string() expected an int for positional argument 1\0" -//! .as_ptr() -//! .cast::(), -//! ); -//! return std::ptr::null_mut(); +//! // Let's keep the behaviour consistent on platforms where `c_long` is bigger than 32 bits. +//! // In particular, it is an i32 on Windows but i64 on most Linux systems +//! let mut overflow = 0; +//! let i_long: c_long = PyLong_AsLongAndOverflow(obj, &mut overflow); +//! +//! #[allow(irrefutable_let_patterns)] // some platforms have c_long equal to i32 +//! if overflow != 0 { +//! raise_overflowerror(obj); +//! None +//! } else if let Ok(i) = i_long.try_into() { +//! Some(i) +//! } else { +//! raise_overflowerror(obj); +//! None //! } +//! } //! -//! let arg1 = PyLong_AsLong(arg1); -//! if !PyErr_Occurred().is_null() { -//! return ptr::null_mut(); +//! unsafe fn raise_overflowerror(obj: *mut PyObject) { +//! let obj_repr = PyObject_Str(obj); +//! if !obj_repr.is_null() { +//! let mut size = 0; +//! let p = PyUnicode_AsUTF8AndSize(obj_repr, &mut size); +//! if !p.is_null() { +//! let s = std::str::from_utf8_unchecked(std::slice::from_raw_parts( +//! p.cast::(), +//! size as usize, +//! )); +//! let msg = format!("cannot fit {} in 32 bits\0", s); +//! +//! PyErr_SetString(PyExc_OverflowError, msg.as_ptr().cast::()); +//! } +//! Py_DECREF(obj_repr); //! } +//! } //! -//! let arg2 = *args.add(1); -//! if PyLong_Check(arg2) == 0 { +//! pub unsafe extern "C" fn sum_as_string( +//! _self: *mut PyObject, +//! args: *mut *mut PyObject, +//! nargs: Py_ssize_t, +//! ) -> *mut PyObject { +//! if nargs != 2 { //! PyErr_SetString( //! PyExc_TypeError, -//! "sum_as_string() expected an int for positional argument 2\0" -//! .as_ptr() -//! .cast::(), +//! c"sum_as_string expected 2 positional arguments".as_ptr(), //! ); //! return std::ptr::null_mut(); //! } //! -//! let arg2 = PyLong_AsLong(arg2); -//! if !PyErr_Occurred().is_null() { -//! return ptr::null_mut(); -//! } +//! let (first, second) = (*args, *args.add(1)); +//! +//! let first = match parse_arg_as_i32(first, 1) { +//! Some(x) => x, +//! None => return std::ptr::null_mut(), +//! }; +//! let second = match parse_arg_as_i32(second, 2) { +//! Some(x) => x, +//! None => return std::ptr::null_mut(), +//! }; //! -//! match arg1.checked_add(arg2) { +//! match first.checked_add(second) { //! Some(sum) => { //! let string = sum.to_string(); //! PyUnicode_FromStringAndSize(string.as_ptr().cast::(), string.len() as isize) @@ -177,7 +257,7 @@ //! None => { //! PyErr_SetString( //! PyExc_OverflowError, -//! "arguments too large to add\0".as_ptr().cast::(), +//! c"arguments too large to add".as_ptr(), //! ); //! std::ptr::null_mut() //! } @@ -209,6 +289,12 @@ //! [manually][manual_builds]. Both offer more flexibility than `maturin` but require further //! configuration. //! +//! This example stores the module definition statically and uses the `PyModule_Create` function +//! in the CPython C API to register the module. This is the "old" style for registering modules +//! and has the limitation that it cannot support subinterpreters. You can also create a module +//! using the new multi-phase initialization API that does support subinterpreters. See the +//! `sequential` project located in the `examples` directory at the root of the `pyo3-ffi` crate +//! for a worked example of how to this using `pyo3-ffi`. //! //! # Using Python from Rust //! @@ -231,31 +317,73 @@ //! [`maturin`]: https://github.com/PyO3/maturin "Build and publish crates with pyo3, rust-cpython and cffi bindings as well as rust binaries as python packages" //! [`pyo3-build-config`]: https://docs.rs/pyo3-build-config //! [feature flags]: https://doc.rust-lang.org/cargo/reference/features.html "Features - The Cargo Book" -//! [manual_builds]: https://pyo3.rs/latest/building_and_distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide" +#![doc = concat!("[manual_builds]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/building-and-distribution.html#manual-builds \"Manual builds - Building and Distribution - PyO3 user guide\"")] //! [setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions" //! [PEP 384]: https://www.python.org/dev/peps/pep-0384 "PEP 384 -- Defining a Stable ABI" -//! [Features chapter of the guide]: https://pyo3.rs/latest/features.html#features-reference "Features Reference - PyO3 user guide" - +#![doc = concat!("[Features chapter of the guide]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#features-reference \"Features eference - PyO3 user guide\"")] #![allow( missing_docs, non_camel_case_types, non_snake_case, non_upper_case_globals, clippy::upper_case_acronyms, - clippy::missing_safety_doc + clippy::missing_safety_doc, + clippy::ptr_eq )] #![warn(elided_lifetimes_in_paths, unused_lifetimes)] +// This crate is a hand-maintained translation of CPython's headers, so requiring "unsafe" +// blocks within those translations increases maintenance burden without providing any +// additional safety. The safety of the functions in this crate is determined by the +// original CPython headers +#![allow(unsafe_op_in_unsafe_fn)] // Until `extern type` is stabilized, use the recommended approach to // model opaque types: // https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs macro_rules! opaque_struct { - ($name:ident) => { + ($(#[$attrs:meta])* $pub:vis $name:ident) => { + $(#[$attrs])* #[repr(C)] - pub struct $name([u8; 0]); + $pub struct $name([u8; 0]); }; } +/// This is a helper macro to create a `&'static CStr`. +/// +/// It can be used on all Rust versions supported by PyO3, unlike c"" literals which +/// were stabilised in Rust 1.77. +/// +/// Due to the nature of PyO3 making heavy use of C FFI interop with Python, it is +/// common for PyO3 to use CStr. +/// +/// Examples: +/// +/// ```rust,no_run +/// use std::ffi::CStr; +/// +/// const HELLO: &CStr = pyo3_ffi::c_str!("hello"); +/// static WORLD: &CStr = pyo3_ffi::c_str!("world"); +/// ``` +#[macro_export] +macro_rules! c_str { + // TODO: deprecate this now MSRV is above 1.77 + ($s:expr) => { + $crate::_cstr_from_utf8_with_nul_checked(concat!($s, "\0")) + }; +} + +/// Private helper for `c_str!` macro. +#[doc(hidden)] +pub const fn _cstr_from_utf8_with_nul_checked(s: &str) -> &std::ffi::CStr { + match std::ffi::CStr::from_bytes_with_nul(s.as_bytes()) { + Ok(cstr) => cstr, + Err(_) => panic!("string contains nul bytes"), + } +} + +pub mod compat; +mod impl_; + pub use self::abstract_::*; pub use self::bltinmodule::*; pub use self::boolobject::*; @@ -277,6 +405,8 @@ pub use self::enumobject::*; pub use self::fileobject::*; pub use self::fileutils::*; pub use self::floatobject::*; +#[cfg(Py_3_9)] +pub use self::genericaliasobject::*; pub use self::import::*; pub use self::intrcheck::*; pub use self::iterobject::*; @@ -305,7 +435,9 @@ pub use self::pyport::*; pub use self::pystate::*; pub use self::pystrtod::*; pub use self::pythonrun::*; +pub use self::pytypedefs::*; pub use self::rangeobject::*; +pub use self::refcount::*; pub use self::setobject::*; pub use self::sliceobject::*; pub use self::structseq::*; @@ -346,7 +478,7 @@ mod fileobject; mod fileutils; mod floatobject; // skipped empty frameobject.h -// skipped genericaliasobject.h +mod genericaliasobject; mod import; // skipped interpreteridobject.h mod intrcheck; @@ -397,7 +529,9 @@ mod pythonrun; mod pystrtod; // skipped pythread.h // skipped pytime.h +mod pytypedefs; mod rangeobject; +mod refcount; mod setobject; mod sliceobject; mod structseq; diff --git a/pyo3-ffi/src/listobject.rs b/pyo3-ffi/src/listobject.rs index 32c6a2dc26a..6fd28ff0c99 100644 --- a/pyo3-ffi/src/listobject.rs +++ b/pyo3-ffi/src/listobject.rs @@ -1,6 +1,6 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::c_int; +use std::ffi::c_int; use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -28,6 +28,9 @@ extern "C" { pub fn PyList_Size(arg1: *mut PyObject) -> Py_ssize_t; #[cfg_attr(PyPy, link_name = "PyPyList_GetItem")] pub fn PyList_GetItem(arg1: *mut PyObject, arg2: Py_ssize_t) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyList_GetItemRef")] + pub fn PyList_GetItemRef(arg1: *mut PyObject, arg2: Py_ssize_t) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyList_SetItem")] pub fn PyList_SetItem(arg1: *mut PyObject, arg2: Py_ssize_t, arg3: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Insert")] @@ -47,6 +50,10 @@ extern "C" { arg3: Py_ssize_t, arg4: *mut PyObject, ) -> c_int; + #[cfg(Py_3_13)] + pub fn PyList_Extend(list: *mut PyObject, iterable: *mut PyObject) -> c_int; + #[cfg(Py_3_13)] + pub fn PyList_Clear(list: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Sort")] pub fn PyList_Sort(arg1: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Reverse")] @@ -54,14 +61,16 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyList_AsTuple")] pub fn PyList_AsTuple(arg1: *mut PyObject) -> *mut PyObject; - // CPython macros exported as functions on PyPy - #[cfg(PyPy)] + // CPython macros exported as functions on PyPy or GraalPy + #[cfg(any(PyPy, GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyList_GET_ITEM")] + #[cfg_attr(GraalPy, link_name = "PyList_GetItem")] pub fn PyList_GET_ITEM(arg1: *mut PyObject, arg2: Py_ssize_t) -> *mut PyObject; #[cfg(PyPy)] #[cfg_attr(PyPy, link_name = "PyPyList_GET_SIZE")] pub fn PyList_GET_SIZE(arg1: *mut PyObject) -> Py_ssize_t; - #[cfg(PyPy)] + #[cfg(any(PyPy, GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyList_SET_ITEM")] + #[cfg_attr(GraalPy, link_name = "_PyList_SET_ITEM")] pub fn PyList_SET_ITEM(arg1: *mut PyObject, arg2: Py_ssize_t, arg3: *mut PyObject); } diff --git a/pyo3-ffi/src/longobject.rs b/pyo3-ffi/src/longobject.rs index 55ea8fa1462..6927fcbef5a 100644 --- a/pyo3-ffi/src/longobject.rs +++ b/pyo3-ffi/src/longobject.rs @@ -1,18 +1,10 @@ use crate::object::*; use crate::pyport::Py_ssize_t; use libc::size_t; -#[cfg(not(Py_LIMITED_API))] -use std::os::raw::c_uchar; -use std::os::raw::{c_char, c_double, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void}; +use std::ffi::{c_char, c_double, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void}; use std::ptr::addr_of_mut; -opaque_struct!(PyLongObject); - -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - #[cfg_attr(PyPy, link_name = "PyPyLong_Type")] - pub static mut PyLong_Type: PyTypeObject; -} +opaque_struct!(pub PyLongObject); #[inline] pub unsafe fn PyLong_Check(op: *mut PyObject) -> c_int { @@ -90,34 +82,11 @@ extern "C" { arg3: c_int, ) -> *mut PyObject; } -// skipped non-limited PyLong_FromUnicodeObject -// skipped non-limited _PyLong_FromBytes #[cfg(not(Py_LIMITED_API))] extern "C" { - // skipped non-limited _PyLong_Sign - #[cfg_attr(PyPy, link_name = "_PyPyLong_NumBits")] pub fn _PyLong_NumBits(obj: *mut PyObject) -> size_t; - - // skipped _PyLong_DivmodNear - - #[cfg_attr(PyPy, link_name = "_PyPyLong_FromByteArray")] - pub fn _PyLong_FromByteArray( - bytes: *const c_uchar, - n: size_t, - little_endian: c_int, - is_signed: c_int, - ) -> *mut PyObject; - - #[cfg_attr(PyPy, link_name = "_PyPyLong_AsByteArrayO")] - pub fn _PyLong_AsByteArray( - v: *mut PyLongObject, - bytes: *mut c_uchar, - n: size_t, - little_endian: c_int, - is_signed: c_int, - ) -> c_int; } // skipped non-limited _PyLong_Format @@ -130,6 +99,5 @@ extern "C" { pub fn PyOS_strtol(arg1: *const c_char, arg2: *mut *mut c_char, arg3: c_int) -> c_long; } -// skipped non-limited _PyLong_GCD // skipped non-limited _PyLong_Rshift // skipped non-limited _PyLong_Lshift diff --git a/pyo3-ffi/src/marshal.rs b/pyo3-ffi/src/marshal.rs index 562f6f91620..bba8f5ece50 100644 --- a/pyo3-ffi/src/marshal.rs +++ b/pyo3-ffi/src/marshal.rs @@ -1,5 +1,5 @@ use super::{PyObject, Py_ssize_t}; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; // skipped Py_MARSHAL_VERSION // skipped PyMarshal_WriteLongToFile diff --git a/pyo3-ffi/src/memoryobject.rs b/pyo3-ffi/src/memoryobject.rs index b7ef9e2ef1d..01ceea3bdbe 100644 --- a/pyo3-ffi/src/memoryobject.rs +++ b/pyo3-ffi/src/memoryobject.rs @@ -1,13 +1,12 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; use std::ptr::addr_of_mut; +// skipped _PyManagedBuffer_Type + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { - #[cfg(not(Py_LIMITED_API))] - pub static mut _PyManagedBuffer_Type: PyTypeObject; - #[cfg_attr(PyPy, link_name = "PyPyMemoryView_Type")] pub static mut PyMemoryView_Type: PyTypeObject; } @@ -18,7 +17,7 @@ pub unsafe fn PyMemoryView_Check(op: *mut PyObject) -> c_int { } // skipped non-limited PyMemoryView_GET_BUFFER -// skipped non-limited PyMemeryView_GET_BASE +// skipped non-limited PyMemoryView_GET_BASE extern "C" { #[cfg_attr(PyPy, link_name = "PyPyMemoryView_FromObject")] diff --git a/pyo3-ffi/src/methodobject.rs b/pyo3-ffi/src/methodobject.rs index e80d5668e78..5f367c9bd48 100644 --- a/pyo3-ffi/src/methodobject.rs +++ b/pyo3-ffi/src/methodobject.rs @@ -1,10 +1,10 @@ use crate::object::{PyObject, PyTypeObject, Py_TYPE}; #[cfg(Py_3_9)] use crate::PyObject_TypeCheck; -use std::os::raw::{c_char, c_int, c_void}; +use std::ffi::{c_char, c_int, c_void}; use std::{mem, ptr}; -#[cfg(all(Py_3_9, not(Py_LIMITED_API)))] +#[cfg(all(Py_3_9, not(Py_LIMITED_API), not(GraalPy)))] pub struct PyCFunctionObject { pub ob_base: PyObject, pub m_ml: *mut PyMethodDef, @@ -43,26 +43,34 @@ pub type PyCFunction = unsafe extern "C" fn(slf: *mut PyObject, args: *mut PyObject) -> *mut PyObject; #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] -pub type _PyCFunctionFast = unsafe extern "C" fn( +pub type PyCFunctionFast = unsafe extern "C" fn( slf: *mut PyObject, args: *mut *mut PyObject, nargs: crate::pyport::Py_ssize_t, ) -> *mut PyObject; +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[deprecated(note = "renamed to `PyCFunctionFast`")] +pub type _PyCFunctionFast = PyCFunctionFast; + pub type PyCFunctionWithKeywords = unsafe extern "C" fn( slf: *mut PyObject, args: *mut PyObject, kwds: *mut PyObject, ) -> *mut PyObject; -#[cfg(not(Py_LIMITED_API))] -pub type _PyCFunctionFastWithKeywords = unsafe extern "C" fn( +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +pub type PyCFunctionFastWithKeywords = unsafe extern "C" fn( slf: *mut PyObject, args: *const *mut PyObject, nargs: crate::pyport::Py_ssize_t, kwnames: *mut PyObject, ) -> *mut PyObject; +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[deprecated(note = "renamed to `PyCFunctionFastWithKeywords`")] +pub type _PyCFunctionFastWithKeywords = PyCFunctionFastWithKeywords; + #[cfg(all(Py_3_9, not(Py_LIMITED_API)))] pub type PyCMethod = unsafe extern "C" fn( slf: *mut PyObject, @@ -77,6 +85,7 @@ extern "C" { pub fn PyCFunction_GetFunction(f: *mut PyObject) -> Option; pub fn PyCFunction_GetSelf(f: *mut PyObject) -> *mut PyObject; pub fn PyCFunction_GetFlags(f: *mut PyObject) -> c_int; + #[cfg(not(Py_3_13))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] pub fn PyCFunction_Call( f: *mut PyObject, @@ -144,11 +153,21 @@ pub union PyMethodDefPointer { /// This variant corresponds with [`METH_FASTCALL`]. #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] - pub _PyCFunctionFast: _PyCFunctionFast, + #[deprecated(note = "renamed to `PyCFunctionFast`")] + pub _PyCFunctionFast: PyCFunctionFast, + + /// This variant corresponds with [`METH_FASTCALL`]. + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + pub PyCFunctionFast: PyCFunctionFast, + + /// This variant corresponds with [`METH_FASTCALL`] | [`METH_KEYWORDS`]. + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + #[deprecated(note = "renamed to `PyCFunctionFastWithKeywords`")] + pub _PyCFunctionFastWithKeywords: PyCFunctionFastWithKeywords, /// This variant corresponds with [`METH_FASTCALL`] | [`METH_KEYWORDS`]. - #[cfg(not(Py_LIMITED_API))] - pub _PyCFunctionFastWithKeywords: _PyCFunctionFastWithKeywords, + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + pub PyCFunctionFastWithKeywords: PyCFunctionFastWithKeywords, /// This variant corresponds with [`METH_METHOD`] | [`METH_FASTCALL`] | [`METH_KEYWORDS`]. #[cfg(all(Py_3_9, not(Py_LIMITED_API)))] @@ -186,9 +205,8 @@ impl std::fmt::Pointer for PyMethodDefPointer { } } -// TODO: This can be a const assert on Rust 1.57 const _: () = - [()][mem::size_of::() - mem::size_of::>()]; + assert!(mem::size_of::() == mem::size_of::>()); #[cfg(not(Py_3_9))] extern "C" { diff --git a/pyo3-ffi/src/modsupport.rs b/pyo3-ffi/src/modsupport.rs index b259c70059e..8cda23a8c2d 100644 --- a/pyo3-ffi/src/modsupport.rs +++ b/pyo3-ffi/src/modsupport.rs @@ -2,7 +2,7 @@ use crate::methodobject::PyMethodDef; use crate::moduleobject::PyModuleDef; use crate::object::PyObject; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int, c_long}; +use std::ffi::{c_char, c_int, c_long}; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyArg_Parse")] @@ -14,9 +14,14 @@ extern "C" { arg1: *mut PyObject, arg2: *mut PyObject, arg3: *const c_char, - arg4: *mut *mut c_char, + #[cfg(not(Py_3_13))] arg4: *mut *mut c_char, + #[cfg(Py_3_13)] arg4: *const *const c_char, ... ) -> c_int; + + // skipped PyArg_VaParse + // skipped PyArg_VaParseTupleAndKeywords + pub fn PyArg_ValidateKeywordArguments(arg1: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyArg_UnpackTuple")] pub fn PyArg_UnpackTuple( @@ -26,33 +31,13 @@ extern "C" { arg4: Py_ssize_t, ... ) -> c_int; + #[cfg_attr(PyPy, link_name = "PyPy_BuildValue")] pub fn Py_BuildValue(arg1: *const c_char, ...) -> *mut PyObject; - // #[cfg_attr(PyPy, link_name = "_PyPy_BuildValue_SizeT")] - //pub fn _Py_BuildValue_SizeT(arg1: *const c_char, ...) - // -> *mut PyObject; - // #[cfg_attr(PyPy, link_name = "PyPy_VaBuildValue")] - - // skipped non-limited _PyArg_UnpackStack - // skipped non-limited _PyArg_NoKeywords - // skipped non-limited _PyArg_NoKwnames - // skipped non-limited _PyArg_NoPositional - // skipped non-limited _PyArg_BadArgument - // skipped non-limited _PyArg_CheckPositional - - //pub fn Py_VaBuildValue(arg1: *const c_char, arg2: va_list) - // -> *mut PyObject; - - // skipped non-limited _Py_VaBuildStack - // skipped non-limited _PyArg_Parser - - // skipped non-limited _PyArg_ParseTupleAndKeywordsFast - // skipped non-limited _PyArg_ParseStack - // skipped non-limited _PyArg_ParseStackAndKeywords - // skipped non-limited _PyArg_VaParseTupleAndKeywordsFast - // skipped non-limited _PyArg_UnpackKeywords - // skipped non-limited _PyArg_Fini + // skipped Py_VaBuildValue + #[cfg(Py_3_13)] + pub fn PyModule_Add(module: *mut PyObject, name: *const c_char, value: *mut PyObject) -> c_int; #[cfg(Py_3_10)] #[cfg_attr(PyPy, link_name = "PyPyModule_AddObjectRef")] pub fn PyModule_AddObjectRef( @@ -88,6 +73,7 @@ extern "C" { // skipped PyModule_AddStringMacro pub fn PyModule_SetDocString(arg1: *mut PyObject, arg2: *const c_char) -> c_int; pub fn PyModule_AddFunctions(arg1: *mut PyObject, arg2: *mut PyMethodDef) -> c_int; + #[cfg_attr(PyPy, link_name = "PyPyModule_ExecDef")] pub fn PyModule_ExecDef(module: *mut PyObject, def: *mut PyModuleDef) -> c_int; } @@ -105,6 +91,7 @@ extern "C" { fn PyModule_Create2TraceRefs(module: *mut PyModuleDef, apiver: c_int) -> *mut PyObject; #[cfg(not(py_sys_config = "Py_TRACE_REFS"))] + #[cfg_attr(PyPy, link_name = "PyPyModule_FromDefAndSpec2")] pub fn PyModule_FromDefAndSpec2( def: *mut PyModuleDef, spec: *mut PyObject, @@ -159,9 +146,3 @@ pub unsafe fn PyModule_FromDefAndSpec(def: *mut PyModuleDef, spec: *mut PyObject }, ) } - -#[cfg(not(Py_LIMITED_API))] -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _Py_PackageContext: *const c_char; -} diff --git a/pyo3-ffi/src/moduleobject.rs b/pyo3-ffi/src/moduleobject.rs index f4306b18639..0d46b6d3130 100644 --- a/pyo3-ffi/src/moduleobject.rs +++ b/pyo3-ffi/src/moduleobject.rs @@ -1,7 +1,7 @@ use crate::methodobject::PyMethodDef; use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int, c_void}; +use std::ffi::{c_char, c_int, c_void}; use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -21,6 +21,7 @@ pub unsafe fn PyModule_CheckExact(op: *mut PyObject) -> c_int { } extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyModule_NewObject")] pub fn PyModule_NewObject(name: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyModule_New")] pub fn PyModule_New(name: *const c_char) -> *mut PyObject; @@ -52,7 +53,6 @@ extern "C" { } #[repr(C)] -#[derive(Copy, Clone)] pub struct PyModuleDef_Base { pub ob_base: PyObject, pub m_init: Option *mut PyObject>, @@ -60,6 +60,7 @@ pub struct PyModuleDef_Base { pub m_copy: *mut PyObject, } +#[allow(clippy::declare_interior_mutable_const)] pub const PyModuleDef_HEAD_INIT: PyModuleDef_Base = PyModuleDef_Base { ob_base: PyObject_HEAD_INIT, m_init: None, @@ -87,18 +88,31 @@ pub const Py_mod_create: c_int = 1; pub const Py_mod_exec: c_int = 2; #[cfg(Py_3_12)] pub const Py_mod_multiple_interpreters: c_int = 3; +#[cfg(Py_3_13)] +pub const Py_mod_gil: c_int = 4; + +// skipped private _Py_mod_LAST_SLOT #[cfg(Py_3_12)] +#[allow(clippy::zero_ptr)] // matches the way that the rest of these constants are defined pub const Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED: *mut c_void = 0 as *mut c_void; #[cfg(Py_3_12)] pub const Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED: *mut c_void = 1 as *mut c_void; #[cfg(Py_3_12)] pub const Py_MOD_PER_INTERPRETER_GIL_SUPPORTED: *mut c_void = 2 as *mut c_void; -// skipped non-limited _Py_mod_LAST_SLOT +#[cfg(Py_3_13)] +#[allow(clippy::zero_ptr)] // matches the way that the rest of these constants are defined +pub const Py_MOD_GIL_USED: *mut c_void = 0 as *mut c_void; +#[cfg(Py_3_13)] +pub const Py_MOD_GIL_NOT_USED: *mut c_void = 1 as *mut c_void; + +#[cfg(all(not(Py_LIMITED_API), Py_GIL_DISABLED))] +extern "C" { + pub fn PyUnstable_Module_SetGIL(module: *mut PyObject, gil: *mut c_void) -> c_int; +} #[repr(C)] -#[derive(Copy, Clone)] pub struct PyModuleDef { pub m_base: PyModuleDef_Base, pub m_name: *const c_char, diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index 0e0243cd764..0842c5d2f5e 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -1,119 +1,221 @@ use crate::pyport::{Py_hash_t, Py_ssize_t}; +#[cfg(Py_GIL_DISABLED)] +use crate::refcount; +#[cfg(Py_GIL_DISABLED)] +use crate::PyMutex; +use std::ffi::{c_char, c_int, c_uint, c_ulong, c_void}; use std::mem; -use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; use std::ptr; +#[cfg(Py_GIL_DISABLED)] +use std::sync::atomic::{AtomicIsize, AtomicU32}; #[cfg(Py_LIMITED_API)] -opaque_struct!(PyTypeObject); +opaque_struct!(pub PyTypeObject); #[cfg(not(Py_LIMITED_API))] pub use crate::cpython::object::PyTypeObject; -// _PyObject_HEAD_EXTRA: conditionally defined in PyObject_HEAD_INIT -// _PyObject_EXTRA_INIT: conditionally defined in PyObject_HEAD_INIT +// skip PyObject_HEAD -#[cfg(Py_3_12)] -pub const _Py_IMMORTAL_REFCNT: Py_ssize_t = { - if cfg!(target_pointer_width = "64") { - c_uint::MAX as Py_ssize_t - } else { - // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h) - (c_uint::MAX >> 2) as Py_ssize_t - } -}; - -pub const PyObject_HEAD_INIT: PyObject = PyObject { - #[cfg(py_sys_config = "Py_TRACE_REFS")] - _ob_next: std::ptr::null_mut(), - #[cfg(py_sys_config = "Py_TRACE_REFS")] - _ob_prev: std::ptr::null_mut(), - #[cfg(Py_3_12)] - ob_refcnt: PyObjectObRefcnt { ob_refcnt: 1 }, - #[cfg(not(Py_3_12))] - ob_refcnt: 1, - #[cfg(PyPy)] - ob_pypy_link: 0, - ob_type: std::ptr::null_mut(), -}; +#[repr(C)] +#[derive(Copy, Clone)] +#[cfg(all( + target_pointer_width = "64", + Py_3_14, + not(Py_GIL_DISABLED), + target_endian = "big" +))] +/// This struct is anonymous in CPython, so the name was given by PyO3 because +/// Rust structs need a name. +pub struct PyObjectObFlagsAndRefcnt { + pub ob_flags: u16, + pub ob_overflow: u16, + pub ob_refcnt: u32, +} -// skipped PyObject_VAR_HEAD -// skipped Py_INVALID_SIZE +#[repr(C)] +#[derive(Copy, Clone)] +#[cfg(all( + target_pointer_width = "64", + Py_3_14, + not(Py_GIL_DISABLED), + target_endian = "little" +))] +/// This struct is anonymous in CPython, so the name was given by PyO3 because +/// Rust structs need a name. +pub struct PyObjectObFlagsAndRefcnt { + pub ob_refcnt: u32, + pub ob_overflow: u16, + pub ob_flags: u16, +} #[repr(C)] #[derive(Copy, Clone)] -#[cfg(Py_3_12)] +#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] /// This union is anonymous in CPython, so the name was given by PyO3 because -/// Rust unions need a name. +/// Rust union need a name. pub union PyObjectObRefcnt { + #[cfg(all(target_pointer_width = "64", Py_3_14))] + pub ob_refcnt_full: crate::PY_INT64_T, + #[cfg(all(target_pointer_width = "64", Py_3_14))] + pub refcnt_and_flags: PyObjectObFlagsAndRefcnt, pub ob_refcnt: Py_ssize_t, - #[cfg(target_pointer_width = "64")] + #[cfg(all(target_pointer_width = "64", not(Py_3_14)))] pub ob_refcnt_split: [crate::PY_UINT32_T; 2], } -#[cfg(Py_3_12)] +#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] impl std::fmt::Debug for PyObjectObRefcnt { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", unsafe { self.ob_refcnt }) } } -#[cfg(not(Py_3_12))] +#[cfg(all(not(Py_3_12), not(Py_GIL_DISABLED)))] pub type PyObjectObRefcnt = Py_ssize_t; +// PyObject_HEAD_INIT comes before the PyObject definition in object.h +// but we put it after PyObject because HEAD_INIT uses PyObject + #[repr(C)] -#[derive(Copy, Clone, Debug)] +#[derive(Debug)] pub struct PyObject { #[cfg(py_sys_config = "Py_TRACE_REFS")] pub _ob_next: *mut PyObject, #[cfg(py_sys_config = "Py_TRACE_REFS")] pub _ob_prev: *mut PyObject, + #[cfg(Py_GIL_DISABLED)] + pub ob_tid: libc::uintptr_t, + #[cfg(all(Py_GIL_DISABLED, not(Py_3_14)))] + pub _padding: u16, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + pub ob_flags: u16, + #[cfg(Py_GIL_DISABLED)] + pub ob_mutex: PyMutex, // per-object lock + #[cfg(Py_GIL_DISABLED)] + pub ob_gc_bits: u8, // gc-related state + #[cfg(Py_GIL_DISABLED)] + pub ob_ref_local: AtomicU32, // local reference count + #[cfg(Py_GIL_DISABLED)] + pub ob_ref_shared: AtomicIsize, // shared reference count + #[cfg(not(Py_GIL_DISABLED))] pub ob_refcnt: PyObjectObRefcnt, #[cfg(PyPy)] pub ob_pypy_link: Py_ssize_t, pub ob_type: *mut PyTypeObject, } +#[allow(clippy::declare_interior_mutable_const)] +pub const PyObject_HEAD_INIT: PyObject = PyObject { + #[cfg(py_sys_config = "Py_TRACE_REFS")] + _ob_next: std::ptr::null_mut(), + #[cfg(py_sys_config = "Py_TRACE_REFS")] + _ob_prev: std::ptr::null_mut(), + #[cfg(Py_GIL_DISABLED)] + ob_tid: 0, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + ob_flags: 0, + #[cfg(all(Py_GIL_DISABLED, not(Py_3_14)))] + _padding: 0, + #[cfg(Py_GIL_DISABLED)] + ob_mutex: PyMutex::new(), + #[cfg(Py_GIL_DISABLED)] + ob_gc_bits: 0, + #[cfg(Py_GIL_DISABLED)] + ob_ref_local: AtomicU32::new(refcount::_Py_IMMORTAL_REFCNT_LOCAL), + #[cfg(Py_GIL_DISABLED)] + ob_ref_shared: AtomicIsize::new(0), + #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] + ob_refcnt: PyObjectObRefcnt { ob_refcnt: 1 }, + #[cfg(not(Py_3_12))] + ob_refcnt: 1, + #[cfg(PyPy)] + ob_pypy_link: 0, + ob_type: std::ptr::null_mut(), +}; + +// skipped _Py_UNOWNED_TID + // skipped _PyObject_CAST #[repr(C)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] pub struct PyVarObject { pub ob_base: PyObject, + #[cfg(not(GraalPy))] pub ob_size: Py_ssize_t, + // On GraalPy the field is physically there, but not always populated. We hide it to prevent accidental misuse + #[cfg(GraalPy)] + pub _ob_size_graalpy: Py_ssize_t, } -// skipped _PyVarObject_CAST +// skipped private _PyVarObject_CAST #[inline] +#[cfg(not(any(GraalPy, PyPy)))] +#[cfg_attr(docsrs, doc(cfg(all())))] pub unsafe fn Py_Is(x: *mut PyObject, y: *mut PyObject) -> c_int { (x == y).into() } -#[inline] -#[cfg(Py_3_12)] -pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { - (*ob).ob_refcnt.ob_refcnt +#[cfg(any(GraalPy, PyPy))] +#[cfg_attr(docsrs, doc(cfg(all())))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPy_Is")] + pub fn Py_Is(x: *mut PyObject, y: *mut PyObject) -> c_int; } -#[inline] -#[cfg(not(Py_3_12))] -pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { - (*ob).ob_refcnt +// skipped _Py_GetThreadLocal_Addr + +// skipped _Py_ThreadID + +// skipped _Py_IsOwnedByCurrentThread + +#[cfg(GraalPy)] +extern "C" { + #[cfg(GraalPy)] + fn _Py_TYPE(arg1: *const PyObject) -> *mut PyTypeObject; + + #[cfg(GraalPy)] + fn _Py_SIZE(arg1: *const PyObject) -> Py_ssize_t; } #[inline] +#[cfg(not(Py_3_14))] pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { - (*ob).ob_type + #[cfg(not(GraalPy))] + return (*ob).ob_type; + #[cfg(GraalPy)] + return _Py_TYPE(ob); +} + +#[cfg_attr(windows, link(name = "pythonXY"))] +#[cfg(Py_3_14)] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPy_TYPE")] + pub fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject; } -// PyLong_Type defined in longobject.rs -// PyBool_Type defined in boolobject.rs +// skip _Py_TYPE compat shim + +#[cfg_attr(windows, link(name = "pythonXY"))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyLong_Type")] + pub static mut PyLong_Type: PyTypeObject; + #[cfg_attr(PyPy, link_name = "PyPyBool_Type")] + pub static mut PyBool_Type: PyTypeObject; +} #[inline] pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { - debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyLong_Type)); - debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyBool_Type)); - (*ob.cast::()).ob_size + #[cfg(not(GraalPy))] + { + debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyLong_Type)); + debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyBool_Type)); + (*ob.cast::()).ob_size + } + #[cfg(GraalPy)] + _Py_SIZE(ob) } #[inline] @@ -121,23 +223,8 @@ pub unsafe fn Py_IS_TYPE(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { (Py_TYPE(ob) == tp) as c_int } -#[inline(always)] -#[cfg(all(Py_3_12, target_pointer_width = "64"))] -pub unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { - (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int -} - -#[inline(always)] -#[cfg(all(Py_3_12, target_pointer_width = "32"))] -pub unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { - ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int -} - -// skipped _Py_SET_REFCNT -// skipped Py_SET_REFCNT -// skipped _Py_SET_TYPE // skipped Py_SET_TYPE -// skipped _Py_SET_SIZE + // skipped Py_SET_SIZE pub type unaryfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; @@ -249,6 +336,14 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyType_GetQualName")] pub fn PyType_GetQualName(arg1: *mut PyTypeObject) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyType_GetFullyQualifiedName")] + pub fn PyType_GetFullyQualifiedName(arg1: *mut PyTypeObject) -> *mut PyObject; + + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleName")] + pub fn PyType_GetModuleName(arg1: *mut PyTypeObject) -> *mut PyObject; + #[cfg(Py_3_12)] #[cfg_attr(PyPy, link_name = "PyPyType_FromMetaclass")] pub fn PyType_FromMetaclass( @@ -272,7 +367,7 @@ extern "C" { #[inline] pub unsafe fn PyObject_TypeCheck(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { - (Py_TYPE(ob) == tp || PyType_IsSubtype(Py_TYPE(ob), tp) != 0) as c_int + (Py_IS_TYPE(ob, tp) != 0 || PyType_IsSubtype(Py_TYPE(ob), tp) != 0) as c_int } #[cfg_attr(windows, link(name = "pythonXY"))] @@ -329,18 +424,43 @@ extern "C" { arg2: *const c_char, arg3: *mut PyObject, ) -> c_int; + #[cfg(any(Py_3_13, all(PyPy, not(Py_3_11))))] // CPython defined in 3.12 as an inline function in abstract.h + #[cfg_attr(PyPy, link_name = "PyPyObject_DelAttrString")] + pub fn PyObject_DelAttrString(arg1: *mut PyObject, arg2: *const c_char) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttrString")] pub fn PyObject_HasAttrString(arg1: *mut PyObject, arg2: *const c_char) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_GetAttr")] pub fn PyObject_GetAttr(arg1: *mut PyObject, arg2: *mut PyObject) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyObject_GetOptionalAttr")] + pub fn PyObject_GetOptionalAttr( + arg1: *mut PyObject, + arg2: *mut PyObject, + arg3: *mut *mut PyObject, + ) -> c_int; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyObject_GetOptionalAttrString")] + pub fn PyObject_GetOptionalAttrString( + arg1: *mut PyObject, + arg2: *const c_char, + arg3: *mut *mut PyObject, + ) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_SetAttr")] pub fn PyObject_SetAttr(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; + #[cfg(any(Py_3_13, all(PyPy, not(Py_3_11))))] // CPython defined in 3.12 as an inline function in abstract.h + #[cfg_attr(PyPy, link_name = "PyPyObject_DelAttr")] + pub fn PyObject_DelAttr(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttr")] pub fn PyObject_HasAttr(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttrWithError")] + pub fn PyObject_HasAttrWithError(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyObject_HasAttrStringWithError")] + pub fn PyObject_HasAttrStringWithError(arg1: *mut PyObject, arg2: *const c_char) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyObject_SelfIter")] pub fn PyObject_SelfIter(arg1: *mut PyObject) -> *mut PyObject; - #[cfg_attr(PyPy, link_name = "PyPyObject_GenericGetAttr")] pub fn PyObject_GenericGetAttr(arg1: *mut PyObject, arg2: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_GenericSetAttr")] @@ -350,7 +470,9 @@ extern "C" { arg3: *mut PyObject, ) -> c_int; #[cfg(not(all(Py_LIMITED_API, not(Py_3_10))))] + #[cfg_attr(PyPy, link_name = "PyPyObject_GenericGetDict")] pub fn PyObject_GenericGetDict(arg1: *mut PyObject, arg2: *mut c_void) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyObject_GenericSetDict")] pub fn PyObject_GenericSetDict( arg1: *mut PyObject, arg2: *mut PyObject, @@ -378,8 +500,8 @@ extern "C" { // Flag bits for printing: pub const Py_PRINT_RAW: c_int = 1; // No string quotes etc. -#[cfg(all(Py_3_12, not(Py_LIMITED_API)))] -pub const _Py_TPFLAGS_STATIC_BUILTIN: c_ulong = 1 << 1; +// skipped because is a private API +// const _Py_TPFLAGS_STATIC_BUILTIN: c_ulong = 1 << 1; #[cfg(all(Py_3_12, not(Py_LIMITED_API)))] pub const Py_TPFLAGS_MANAGED_WEAKREF: c_ulong = 1 << 3; @@ -408,7 +530,7 @@ pub const Py_TPFLAGS_BASETYPE: c_ulong = 1 << 10; /// Set if the type implements the vectorcall protocol (PEP 590) #[cfg(any(Py_3_12, all(Py_3_8, not(Py_LIMITED_API))))] pub const Py_TPFLAGS_HAVE_VECTORCALL: c_ulong = 1 << 11; -// skipped non-limited _Py_TPFLAGS_HAVE_VECTORCALL +// skipped backwards-compatibility alias _Py_TPFLAGS_HAVE_VECTORCALL /// Set if the type is 'ready' -- fully initialized pub const Py_TPFLAGS_READY: c_ulong = 1 << 12; @@ -452,230 +574,56 @@ pub const Py_TPFLAGS_DEFAULT: c_ulong = if cfg!(Py_3_10) { pub const Py_TPFLAGS_HAVE_FINALIZE: c_ulong = 1; pub const Py_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; -extern "C" { - #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - pub fn _Py_NegativeRefcount(filename: *const c_char, lineno: c_int, op: *mut PyObject); - #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - fn _Py_INCREF_IncRefTotal(); - #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - fn _Py_DECREF_DecRefTotal(); - - #[cfg_attr(PyPy, link_name = "_PyPy_Dealloc")] - pub fn _Py_Dealloc(arg1: *mut PyObject); - - #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] - pub fn Py_IncRef(o: *mut PyObject); - #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] - pub fn Py_DecRef(o: *mut PyObject); - - #[cfg(Py_3_10)] - #[cfg_attr(PyPy, link_name = "_PyPy_IncRef")] - pub fn _Py_IncRef(o: *mut PyObject); - #[cfg(Py_3_10)] - #[cfg_attr(PyPy, link_name = "_PyPy_DecRef")] - pub fn _Py_DecRef(o: *mut PyObject); -} - -#[inline(always)] -pub unsafe fn Py_INCREF(op: *mut PyObject) { - #[cfg(any( - all(Py_LIMITED_API, Py_3_12), - all( - py_sys_config = "Py_REF_DEBUG", - Py_3_10, - not(all(Py_3_12, not(Py_LIMITED_API))) - ) - ))] - { - _Py_IncRef(op); - } - - #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_3_10)))] - { - return Py_IncRef(op); - } - - #[cfg(any( - all(Py_LIMITED_API, not(Py_3_12)), - all( - not(Py_LIMITED_API), - any( - not(py_sys_config = "Py_REF_DEBUG"), - all(py_sys_config = "Py_REF_DEBUG", Py_3_12), - ) - ), - ))] - { - #[cfg(all(Py_3_12, target_pointer_width = "64"))] - { - let cur_refcnt = (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN]; - let new_refcnt = cur_refcnt.wrapping_add(1); - if new_refcnt == 0 { - return; - } - (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN] = new_refcnt; - } - - #[cfg(all(Py_3_12, target_pointer_width = "32"))] - { - if _Py_IsImmortal(op) != 0 { - return; - } - (*op).ob_refcnt.ob_refcnt += 1 - } - - #[cfg(not(Py_3_12))] - { - (*op).ob_refcnt += 1 - } - - // Skipped _Py_INCREF_STAT_INC - if anyone wants this, please file an issue - // or submit a PR supporting Py_STATS build option and pystats.h - - #[cfg(all(py_sys_config = "Py_REF_DEBUG", Py_3_12))] - _Py_INCREF_IncRefTotal(); - } -} - -#[inline(always)] -#[cfg_attr( - all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)), - track_caller -)] -pub unsafe fn Py_DECREF(op: *mut PyObject) { - #[cfg(any( - all(Py_LIMITED_API, Py_3_12), - all( - py_sys_config = "Py_REF_DEBUG", - Py_3_10, - not(all(Py_3_12, not(Py_LIMITED_API))) - ) - ))] - { - _Py_DecRef(op); - } - - #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_3_10)))] - { - return Py_DecRef(op); - } - - #[cfg(any( - all(Py_LIMITED_API, not(Py_3_12)), - all( - not(Py_LIMITED_API), - any( - not(py_sys_config = "Py_REF_DEBUG"), - all(py_sys_config = "Py_REF_DEBUG", Py_3_12), - ) - ), - ))] - { - #[cfg(Py_3_12)] - if _Py_IsImmortal(op) != 0 { - return; - } - - // Skipped _Py_DECREF_STAT_INC - if anyone needs this, please file an issue - // or submit a PR supporting Py_STATS build option and pystats.h - - #[cfg(all(py_sys_config = "Py_REF_DEBUG", Py_3_12))] - _Py_DECREF_DecRefTotal(); - - #[cfg(Py_3_12)] - { - (*op).ob_refcnt.ob_refcnt -= 1; - - #[cfg(py_sys_config = "Py_REF_DEBUG")] - if (*op).ob_refcnt.ob_refcnt < 0 { - let location = std::panic::Location::caller(); - let filename = std::ffi::CString::new(location.file()).unwrap(); - _Py_NegativeRefcount(filename.as_ptr(), location.line() as i32, op); - } - - if (*op).ob_refcnt.ob_refcnt == 0 { - _Py_Dealloc(op); - } - } - - #[cfg(not(Py_3_12))] - { - (*op).ob_refcnt -= 1; - - if (*op).ob_refcnt == 0 { - _Py_Dealloc(op); - } - } - } -} - -#[inline] -pub unsafe fn Py_CLEAR(op: *mut *mut PyObject) { - let tmp = *op; - if !tmp.is_null() { - *op = ptr::null_mut(); - Py_DECREF(tmp); - } -} - -#[inline] -pub unsafe fn Py_XINCREF(op: *mut PyObject) { - if !op.is_null() { - Py_INCREF(op) - } -} - -#[inline] -pub unsafe fn Py_XDECREF(op: *mut PyObject) { - if !op.is_null() { - Py_DECREF(op) - } -} +#[cfg(Py_3_13)] +pub const Py_CONSTANT_NONE: c_uint = 0; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_FALSE: c_uint = 1; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_TRUE: c_uint = 2; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_ELLIPSIS: c_uint = 3; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_NOT_IMPLEMENTED: c_uint = 4; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_ZERO: c_uint = 5; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_ONE: c_uint = 6; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_EMPTY_STR: c_uint = 7; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_EMPTY_BYTES: c_uint = 8; +#[cfg(Py_3_13)] +pub const Py_CONSTANT_EMPTY_TUPLE: c_uint = 9; extern "C" { - #[cfg(all(Py_3_10, Py_LIMITED_API))] - pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject; - #[cfg(all(Py_3_10, Py_LIMITED_API))] - pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject; -} - -// Technically these macros are only available in the C header from 3.10 and up, however their -// implementation works on all supported Python versions so we define these macros on all -// versions for simplicity. - -#[inline] -pub unsafe fn _Py_NewRef(obj: *mut PyObject) -> *mut PyObject { - Py_INCREF(obj); - obj -} - -#[inline] -pub unsafe fn _Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { - Py_XINCREF(obj); - obj -} - -#[cfg(all(Py_3_10, not(Py_LIMITED_API)))] -#[inline] -pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { - _Py_NewRef(obj) -} - -#[cfg(all(Py_3_10, not(Py_LIMITED_API)))] -#[inline] -pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { - _Py_XNewRef(obj) + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPy_GetConstant")] + pub fn Py_GetConstant(constant_id: c_uint) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPy_GetConstantBorrowed")] + pub fn Py_GetConstantBorrowed(constant_id: c_uint) -> *mut PyObject; } #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))] #[cfg_attr(PyPy, link_name = "_PyPy_NoneStruct")] static mut _Py_NoneStruct: PyObject; + + #[cfg(GraalPy)] + static mut _Py_NoneStructReference: *mut PyObject; } #[inline] pub unsafe fn Py_None() -> *mut PyObject { - ptr::addr_of_mut!(_Py_NoneStruct) + #[cfg(all(not(GraalPy), all(Py_3_13, Py_LIMITED_API)))] + return Py_GetConstantBorrowed(Py_CONSTANT_NONE); + + #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))] + return ptr::addr_of_mut!(_Py_NoneStruct); + + #[cfg(GraalPy)] + return _Py_NoneStructReference; } #[inline] @@ -687,13 +635,24 @@ pub unsafe fn Py_IsNone(x: *mut PyObject) -> c_int { #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))] #[cfg_attr(PyPy, link_name = "_PyPy_NotImplementedStruct")] static mut _Py_NotImplementedStruct: PyObject; + + #[cfg(GraalPy)] + static mut _Py_NotImplementedStructReference: *mut PyObject; } #[inline] pub unsafe fn Py_NotImplemented() -> *mut PyObject { - ptr::addr_of_mut!(_Py_NotImplementedStruct) + #[cfg(all(not(GraalPy), all(Py_3_13, Py_LIMITED_API)))] + return Py_GetConstantBorrowed(Py_CONSTANT_NOT_IMPLEMENTED); + + #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))] + return ptr::addr_of_mut!(_Py_NotImplementedStruct); + + #[cfg(GraalPy)] + return _Py_NotImplementedStructReference; } // skipped Py_RETURN_NOTIMPLEMENTED @@ -718,15 +677,17 @@ pub enum PySendResult { // skipped Py_RETURN_RICHCOMPARE #[inline] -#[cfg(Py_LIMITED_API)] -pub unsafe fn PyType_HasFeature(t: *mut PyTypeObject, f: c_ulong) -> c_int { - ((PyType_GetFlags(t) & f) != 0) as c_int -} +pub unsafe fn PyType_HasFeature(ty: *mut PyTypeObject, feature: c_ulong) -> c_int { + #[cfg(Py_LIMITED_API)] + let flags = PyType_GetFlags(ty); -#[inline] -#[cfg(not(Py_LIMITED_API))] -pub unsafe fn PyType_HasFeature(t: *mut PyTypeObject, f: c_ulong) -> c_int { - (((*t).tp_flags & f) != 0) as c_int + #[cfg(all(not(Py_LIMITED_API), Py_GIL_DISABLED))] + let flags = (*ty).tp_flags.load(std::sync::atomic::Ordering::Relaxed); + + #[cfg(all(not(Py_LIMITED_API), not(Py_GIL_DISABLED)))] + let flags = (*ty).tp_flags; + + ((flags & feature) != 0) as c_int } #[inline] @@ -739,7 +700,21 @@ pub unsafe fn PyType_Check(op: *mut PyObject) -> c_int { PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS) } +// skipped _PyType_CAST + #[inline] pub unsafe fn PyType_CheckExact(op: *mut PyObject) -> c_int { Py_IS_TYPE(op, ptr::addr_of_mut!(PyType_Type)) } + +extern "C" { + #[cfg(any(Py_3_13, all(Py_3_11, not(Py_LIMITED_API))))] + #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleByDef")] + pub fn PyType_GetModuleByDef( + arg1: *mut crate::PyTypeObject, + arg2: *mut crate::PyModuleDef, + ) -> *mut PyObject; + + #[cfg(Py_3_14)] + pub fn PyType_Freeze(tp: *mut crate::PyTypeObject) -> c_int; +} diff --git a/pyo3-ffi/src/objimpl.rs b/pyo3-ffi/src/objimpl.rs index 76835a6d158..338228a53ea 100644 --- a/pyo3-ffi/src/objimpl.rs +++ b/pyo3-ffi/src/objimpl.rs @@ -1,5 +1,5 @@ use libc::size_t; -use std::os::raw::{c_int, c_void}; +use std::ffi::{c_int, c_void}; use crate::object::*; use crate::pyport::Py_ssize_t; diff --git a/pyo3-ffi/src/pyarena.rs b/pyo3-ffi/src/pyarena.rs index 87d5f28a7a5..1200de3df48 100644 --- a/pyo3-ffi/src/pyarena.rs +++ b/pyo3-ffi/src/pyarena.rs @@ -1 +1 @@ -opaque_struct!(PyArena); +opaque_struct!(pub PyArena); diff --git a/pyo3-ffi/src/pybuffer.rs b/pyo3-ffi/src/pybuffer.rs index a414f333ce6..44ae47bdc30 100644 --- a/pyo3-ffi/src/pybuffer.rs +++ b/pyo3-ffi/src/pybuffer.rs @@ -1,6 +1,6 @@ use crate::object::PyObject; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int, c_void}; +use std::ffi::{c_char, c_int, c_void}; use std::ptr; #[repr(C)] @@ -27,6 +27,7 @@ pub struct Py_buffer { } impl Py_buffer { + #[allow(clippy::new_without_default)] pub const fn new() -> Self { Py_buffer { buf: ptr::null_mut(), @@ -102,7 +103,7 @@ extern "C" { } /// Maximum number of dimensions -pub const PyBUF_MAX_NDIM: c_int = if cfg!(PyPy) { 36 } else { 64 }; +pub const PyBUF_MAX_NDIM: usize = 64; /* Flags for getting buffers */ pub const PyBUF_SIMPLE: c_int = 0; diff --git a/pyo3-ffi/src/pycapsule.rs b/pyo3-ffi/src/pycapsule.rs index 5b77841c23c..65df8837063 100644 --- a/pyo3-ffi/src/pycapsule.rs +++ b/pyo3-ffi/src/pycapsule.rs @@ -1,5 +1,5 @@ use crate::object::*; -use std::os::raw::{c_char, c_int, c_void}; +use std::ffi::{c_char, c_int, c_void}; use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] diff --git a/pyo3-ffi/src/pyerrors.rs b/pyo3-ffi/src/pyerrors.rs index 9da00ea390e..557f314c7cb 100644 --- a/pyo3-ffi/src/pyerrors.rs +++ b/pyo3-ffi/src/pyerrors.rs @@ -1,6 +1,6 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyErr_SetNone")] @@ -101,7 +101,7 @@ pub unsafe fn PyUnicodeDecodeError_Create( ) -> *mut PyObject { crate::_PyObject_CallFunction_SizeT( PyExc_UnicodeDecodeError, - b"sy#nns\0".as_ptr().cast::(), + c"sy#nns".as_ptr(), encoding, object, length, @@ -116,6 +116,7 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyExc_BaseException")] pub static mut PyExc_BaseException: *mut PyObject; #[cfg(Py_3_11)] + #[cfg_attr(PyPy, link_name = "PyPyExc_BaseExceptionGroup")] pub static mut PyExc_BaseExceptionGroup: *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyExc_Exception")] pub static mut PyExc_Exception: *mut PyObject; diff --git a/pyo3-ffi/src/pyframe.rs b/pyo3-ffi/src/pyframe.rs index 43a9d1f6777..f91069929ed 100644 --- a/pyo3-ffi/src/pyframe.rs +++ b/pyo3-ffi/src/pyframe.rs @@ -1,14 +1,15 @@ +#[allow(unused_imports)] +use crate::object::PyObject; +#[cfg(not(GraalPy))] #[cfg(any(Py_3_10, all(Py_3_9, not(Py_LIMITED_API))))] use crate::PyCodeObject; -#[cfg(not(Py_LIMITED_API))] use crate::PyFrameObject; -use std::os::raw::c_int; - -#[cfg(Py_LIMITED_API)] -opaque_struct!(PyFrameObject); +use std::ffi::c_int; extern "C" { - pub fn PyFrame_GetLineNumber(f: *mut PyFrameObject) -> c_int; + pub fn PyFrame_GetLineNumber(frame: *mut PyFrameObject) -> c_int; + + #[cfg(not(GraalPy))] #[cfg(any(Py_3_10, all(Py_3_9, not(Py_LIMITED_API))))] - pub fn PyFrame_GetCode(f: *mut PyFrameObject) -> *mut PyCodeObject; + pub fn PyFrame_GetCode(frame: *mut PyFrameObject) -> *mut PyCodeObject; } diff --git a/pyo3-ffi/src/pyhash.rs b/pyo3-ffi/src/pyhash.rs index f8072d85108..4fab07d00d4 100644 --- a/pyo3-ffi/src/pyhash.rs +++ b/pyo3-ffi/src/pyhash.rs @@ -1,9 +1,9 @@ #[cfg(not(any(Py_LIMITED_API, PyPy)))] use crate::pyport::{Py_hash_t, Py_ssize_t}; #[cfg(not(any(Py_LIMITED_API, PyPy)))] -use std::os::raw::{c_char, c_void}; +use std::ffi::c_void; -use std::os::raw::{c_int, c_ulong}; +use std::ffi::{c_int, c_ulong}; extern "C" { // skipped non-limited _Py_HashDouble @@ -20,29 +20,6 @@ pub const _PyHASH_MULTIPLIER: c_ulong = 1000003; // skipped non-limited _Py_HashSecret_t -#[cfg(not(any(Py_LIMITED_API, PyPy)))] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct PyHash_FuncDef { - pub hash: Option Py_hash_t>, - pub name: *const c_char, - pub hash_bits: c_int, - pub seed_bits: c_int, -} - -#[cfg(not(any(Py_LIMITED_API, PyPy)))] -impl Default for PyHash_FuncDef { - #[inline] - fn default() -> Self { - unsafe { std::mem::zeroed() } - } -} - -extern "C" { - #[cfg(not(any(Py_LIMITED_API, PyPy)))] - pub fn PyHash_GetFuncDef() -> *mut PyHash_FuncDef; -} - // skipped Py_HASH_CUTOFF pub const Py_HASH_EXTERNAL: c_int = 0; diff --git a/pyo3-ffi/src/pylifecycle.rs b/pyo3-ffi/src/pylifecycle.rs index 7f73e3f0e9b..e822da09f98 100644 --- a/pyo3-ffi/src/pylifecycle.rs +++ b/pyo3-ffi/src/pylifecycle.rs @@ -1,7 +1,7 @@ use crate::pystate::PyThreadState; use libc::wchar_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; extern "C" { pub fn Py_Initialize(); @@ -18,23 +18,60 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPy_AtExit")] pub fn Py_AtExit(func: Option) -> c_int; - pub fn Py_Exit(arg1: c_int); + pub fn Py_Exit(arg1: c_int) -> !; pub fn Py_Main(argc: c_int, argv: *mut *mut wchar_t) -> c_int; pub fn Py_BytesMain(argc: c_int, argv: *mut *mut c_char) -> c_int; + #[cfg_attr( + Py_3_11, + deprecated(note = "Deprecated since Python 3.11. Use `PyConfig.program_name` instead.") + )] pub fn Py_SetProgramName(arg1: *const wchar_t); #[cfg_attr(PyPy, link_name = "PyPy_GetProgramName")] + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.executable` instead.") + )] pub fn Py_GetProgramName() -> *mut wchar_t; + #[cfg_attr( + Py_3_11, + deprecated(note = "Deprecated since Python 3.11. Use `PyConfig.home` instead.") + )] pub fn Py_SetPythonHome(arg1: *const wchar_t); + #[cfg_attr( + Py_3_13, + deprecated( + note = "Deprecated since Python 3.13. Use `PyConfig.home` or the value of the `PYTHONHOME` environment variable instead." + ) + )] pub fn Py_GetPythonHome() -> *mut wchar_t; - + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.executable` instead.") + )] pub fn Py_GetProgramFullPath() -> *mut wchar_t; - + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.prefix` instead.") + )] pub fn Py_GetPrefix() -> *mut wchar_t; + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.exec_prefix` instead.") + )] pub fn Py_GetExecPrefix() -> *mut wchar_t; + #[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `sys.path` instead.") + )] pub fn Py_GetPath() -> *mut wchar_t; + #[cfg(not(Py_3_13))] + #[cfg_attr( + Py_3_11, + deprecated(note = "Deprecated since Python 3.11. Use `sys.path` instead.") + )] pub fn Py_SetPath(arg1: *const wchar_t); // skipped _Py_CheckPython3 @@ -52,4 +89,10 @@ type PyOS_sighandler_t = unsafe extern "C" fn(arg1: c_int); extern "C" { pub fn PyOS_getsig(arg1: c_int) -> PyOS_sighandler_t; pub fn PyOS_setsig(arg1: c_int, arg2: PyOS_sighandler_t) -> PyOS_sighandler_t; + + #[cfg(Py_3_11)] + pub static Py_Version: std::ffi::c_ulong; + + #[cfg(Py_3_13)] + pub fn Py_IsFinalizing() -> c_int; } diff --git a/pyo3-ffi/src/pymem.rs b/pyo3-ffi/src/pymem.rs index ab0c3d74af9..aaa5a4f6e8d 100644 --- a/pyo3-ffi/src/pymem.rs +++ b/pyo3-ffi/src/pymem.rs @@ -1,5 +1,5 @@ use libc::size_t; -use std::os::raw::c_void; +use std::ffi::c_void; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyMem_Malloc")] diff --git a/pyo3-ffi/src/pyport.rs b/pyo3-ffi/src/pyport.rs index 741b0db7bf8..3a066353ca4 100644 --- a/pyo3-ffi/src/pyport.rs +++ b/pyo3-ffi/src/pyport.rs @@ -1,3 +1,8 @@ +// NB libc does not define this constant on all platforms, so we hard code it +// like CPython does. +// https://github.com/python/cpython/blob/d8b9011702443bb57579f8834f3effe58e290dfc/Include/pyport.h#L372 +pub const INT_MAX: std::ffi::c_int = 2147483647; + pub type PY_UINT32_T = u32; pub type PY_UINT64_T = u64; @@ -11,8 +16,8 @@ pub type Py_ssize_t = ::libc::ssize_t; pub type Py_hash_t = Py_ssize_t; pub type Py_uhash_t = ::libc::size_t; -pub const PY_SSIZE_T_MIN: Py_ssize_t = std::isize::MIN as Py_ssize_t; -pub const PY_SSIZE_T_MAX: Py_ssize_t = std::isize::MAX as Py_ssize_t; +pub const PY_SSIZE_T_MIN: Py_ssize_t = Py_ssize_t::MIN; +pub const PY_SSIZE_T_MAX: Py_ssize_t = Py_ssize_t::MAX; #[cfg(target_endian = "big")] pub const PY_BIG_ENDIAN: usize = 1; diff --git a/pyo3-ffi/src/pystate.rs b/pyo3-ffi/src/pystate.rs index d2fd39e497d..8b8cf13ce70 100644 --- a/pyo3-ffi/src/pystate.rs +++ b/pyo3-ffi/src/pystate.rs @@ -1,15 +1,17 @@ -#[cfg(any(not(PyPy), Py_3_9))] use crate::moduleobject::PyModuleDef; use crate::object::PyObject; -use std::os::raw::c_int; +use std::ffi::c_int; + +#[cfg(all(Py_3_10, not(PyPy), not(Py_LIMITED_API)))] +use crate::PyFrameObject; #[cfg(not(PyPy))] -use std::os::raw::c_long; +use std::ffi::c_long; pub const MAX_CO_EXTRA_USERS: c_int = 255; -opaque_struct!(PyThreadState); -opaque_struct!(PyInterpreterState); +opaque_struct!(pub PyThreadState); +opaque_struct!(pub PyInterpreterState); extern "C" { #[cfg(not(PyPy))] @@ -28,17 +30,13 @@ extern "C" { #[cfg(not(PyPy))] pub fn PyInterpreterState_GetID(arg1: *mut PyInterpreterState) -> i64; - #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 #[cfg_attr(PyPy, link_name = "PyPyState_AddModule")] pub fn PyState_AddModule(arg1: *mut PyObject, arg2: *mut PyModuleDef) -> c_int; - #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 #[cfg_attr(PyPy, link_name = "PyPyState_RemoveModule")] pub fn PyState_RemoveModule(arg1: *mut PyModuleDef) -> c_int; - #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 - // only has PyPy prefix since 3.10 - #[cfg_attr(all(PyPy, Py_3_10), link_name = "PyPyState_FindModule")] + #[cfg_attr(PyPy, link_name = "PyPyState_FindModule")] pub fn PyState_FindModule(arg1: *mut PyModuleDef) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyThreadState_New")] @@ -67,9 +65,14 @@ extern "C" { } // skipped non-limited / 3.9 PyThreadState_GetInterpreter -// skipped non-limited / 3.9 PyThreadState_GetFrame // skipped non-limited / 3.9 PyThreadState_GetID +extern "C" { + // PyThreadState_GetFrame + #[cfg(all(Py_3_10, not(PyPy), not(Py_LIMITED_API)))] + pub fn PyThreadState_GetFrame(arg1: *mut PyThreadState) -> *mut PyFrameObject; +} + #[repr(C)] #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum PyGILState_STATE { @@ -77,9 +80,70 @@ pub enum PyGILState_STATE { PyGILState_UNLOCKED, } +#[cfg(not(any(Py_3_14, target_arch = "wasm32")))] +struct HangThread; + +#[cfg(not(any(Py_3_14, target_arch = "wasm32")))] +impl Drop for HangThread { + fn drop(&mut self) { + loop { + std::thread::park(); // Block forever. + } + } +} + +// The PyGILState_Ensure function will call pthread_exit during interpreter shutdown, +// which causes undefined behavior. Redirect to the "safe" version that hangs instead, +// as Python 3.14 does. +// +// See https://github.com/rust-lang/rust/issues/135929 + +// C-unwind only supported (and necessary) since 1.71. Python 3.14+ does not do +// pthread_exit from PyGILState_Ensure (https://github.com/python/cpython/issues/87135). +mod raw { + #[cfg(not(any(Py_3_14, target_arch = "wasm32")))] + extern "C-unwind" { + #[cfg_attr(PyPy, link_name = "PyPyGILState_Ensure")] + pub fn PyGILState_Ensure() -> super::PyGILState_STATE; + } + + #[cfg(any(Py_3_14, target_arch = "wasm32"))] + extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyGILState_Ensure")] + pub fn PyGILState_Ensure() -> super::PyGILState_STATE; + } +} + +#[cfg(not(any(Py_3_14, target_arch = "wasm32")))] +pub unsafe extern "C" fn PyGILState_Ensure() -> PyGILState_STATE { + let guard = HangThread; + // If `PyGILState_Ensure` calls `pthread_exit`, which it does on Python < 3.14 + // when the interpreter is shutting down, this will cause a forced unwind. + // doing a forced unwind through a function with a Rust destructor is unspecified + // behavior. + // + // However, currently it runs the destructor, which will cause the thread to + // hang as it should. + // + // And if we don't catch the unwinding here, then one of our callers probably has a destructor, + // so it's unspecified behavior anyway, and on many configurations causes the process to abort. + // + // The alternative is for pyo3 to contain custom C or C++ code that catches the `pthread_exit`, + // but that's also annoying from a portability point of view. + // + // On Windows, `PyGILState_Ensure` calls `_endthreadex` instead, which AFAICT can't be caught + // and therefore will cause unsafety if there are pinned objects on the stack. AFAICT there's + // nothing we can do it other than waiting for Python 3.14 or not using Windows. At least, + // if there is nothing pinned on the stack, it won't cause the process to crash. + let ret: PyGILState_STATE = raw::PyGILState_Ensure(); + std::mem::forget(guard); + ret +} + +#[cfg(any(Py_3_14, target_arch = "wasm32"))] +pub use self::raw::PyGILState_Ensure; + extern "C" { - #[cfg_attr(PyPy, link_name = "PyPyGILState_Ensure")] - pub fn PyGILState_Ensure() -> PyGILState_STATE; #[cfg_attr(PyPy, link_name = "PyPyGILState_Release")] pub fn PyGILState_Release(arg1: PyGILState_STATE); #[cfg(not(PyPy))] diff --git a/pyo3-ffi/src/pystrtod.rs b/pyo3-ffi/src/pystrtod.rs index 1f027686bd9..20d5a24387b 100644 --- a/pyo3-ffi/src/pystrtod.rs +++ b/pyo3-ffi/src/pystrtod.rs @@ -1,5 +1,5 @@ use crate::object::PyObject; -use std::os::raw::{c_char, c_double, c_int}; +use std::ffi::{c_char, c_double, c_int}; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyOS_string_to_double")] diff --git a/pyo3-ffi/src/pythonrun.rs b/pyo3-ffi/src/pythonrun.rs index e5f20de0058..090c964d665 100644 --- a/pyo3-ffi/src/pythonrun.rs +++ b/pyo3-ffi/src/pythonrun.rs @@ -1,12 +1,12 @@ use crate::object::*; #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] use libc::FILE; -#[cfg(all(not(PyPy), any(Py_LIMITED_API, not(Py_3_10))))] -use std::os::raw::c_char; -use std::os::raw::c_int; +#[cfg(any(Py_LIMITED_API, not(Py_3_10), PyPy, GraalPy))] +use std::ffi::c_char; +use std::ffi::c_int; extern "C" { - #[cfg(all(Py_LIMITED_API, not(PyPy)))] + #[cfg(any(all(Py_LIMITED_API, not(PyPy)), GraalPy))] pub fn Py_CompileString(string: *const c_char, p: *const c_char, s: c_int) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyErr_Print")] @@ -20,6 +20,28 @@ extern "C" { pub fn PyErr_DisplayException(exc: *mut PyObject); } +#[inline] +#[cfg(PyPy)] +pub unsafe fn Py_CompileString(string: *const c_char, p: *const c_char, s: c_int) -> *mut PyObject { + // PyPy's implementation of Py_CompileString always forwards to Py_CompileStringFlags; this + // is only available in the non-limited API and has a real definition for all versions in + // the cpython/ subdirectory. + #[cfg(Py_LIMITED_API)] + extern "C" { + #[link_name = "PyPy_CompileStringFlags"] + pub fn Py_CompileStringFlags( + string: *const c_char, + p: *const c_char, + s: c_int, + f: *mut std::ffi::c_void, // Actually *mut Py_CompilerFlags in the real definition + ) -> *mut PyObject; + } + #[cfg(not(Py_LIMITED_API))] + use crate::Py_CompileStringFlags; + + Py_CompileStringFlags(string, p, s, std::ptr::null_mut()) +} + // skipped PyOS_InputHook pub const PYOS_STACK_MARGIN: c_int = 2048; @@ -27,12 +49,12 @@ pub const PYOS_STACK_MARGIN: c_int = 2048; // skipped PyOS_CheckStack under Microsoft C #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] -opaque_struct!(_mod); +opaque_struct!(pub _mod); #[cfg(not(any(PyPy, Py_3_10)))] -opaque_struct!(symtable); +opaque_struct!(pub symtable); #[cfg(not(any(PyPy, Py_3_10)))] -opaque_struct!(_node); +opaque_struct!(pub _node); #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] diff --git a/pyo3-ffi/src/pytypedefs.rs b/pyo3-ffi/src/pytypedefs.rs new file mode 100644 index 00000000000..7172724ce1f --- /dev/null +++ b/pyo3-ffi/src/pytypedefs.rs @@ -0,0 +1,4 @@ +// TODO: Created this file as part of fixing pyframe.rs and cpython/pyframe.rs +// TODO: Finish defining or moving declarations now in Include/pytypedefs.h + +opaque_struct!(pub PyFrameObject); diff --git a/pyo3-ffi/src/rangeobject.rs b/pyo3-ffi/src/rangeobject.rs index 408b5cdc5a4..35667fed8c2 100644 --- a/pyo3-ffi/src/rangeobject.rs +++ b/pyo3-ffi/src/rangeobject.rs @@ -1,5 +1,5 @@ use crate::object::*; -use std::os::raw::c_int; +use std::ffi::c_int; use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] diff --git a/pyo3-ffi/src/refcount.rs b/pyo3-ffi/src/refcount.rs new file mode 100644 index 00000000000..22a6268c729 --- /dev/null +++ b/pyo3-ffi/src/refcount.rs @@ -0,0 +1,369 @@ +use crate::pyport::Py_ssize_t; +use crate::PyObject; +#[cfg(py_sys_config = "Py_REF_DEBUG")] +use std::ffi::c_char; +#[cfg(Py_3_12)] +use std::ffi::c_int; +#[cfg(all(Py_3_14, any(not(Py_GIL_DISABLED), target_pointer_width = "32")))] +use std::ffi::c_long; +#[cfg(any(Py_GIL_DISABLED, all(Py_3_12, not(Py_3_14))))] +use std::ffi::c_uint; +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED)))] +use std::ffi::c_ulong; +use std::ptr; +#[cfg(Py_GIL_DISABLED)] +use std::sync::atomic::Ordering::Relaxed; + +#[cfg(Py_3_14)] +const _Py_STATICALLY_ALLOCATED_FLAG: c_int = 1 << 7; + +#[cfg(all(Py_3_12, not(Py_3_14)))] +const _Py_IMMORTAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + c_uint::MAX as Py_ssize_t + } else { + // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h) + (c_uint::MAX >> 2) as Py_ssize_t + } +}; + +// comments in Python.h about the choices for these constants + +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED)))] +const _Py_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + ((3 as c_ulong) << (30 as c_ulong)) as Py_ssize_t + } else { + ((5 as c_long) << (28 as c_long)) as Py_ssize_t + } +}; + +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED)))] +const _Py_STATIC_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + _Py_IMMORTAL_INITIAL_REFCNT + | ((_Py_STATICALLY_ALLOCATED_FLAG as Py_ssize_t) << (32 as Py_ssize_t)) + } else { + ((7 as c_long) << (28 as c_long)) as Py_ssize_t + } +}; + +#[cfg(all(Py_3_14, target_pointer_width = "32"))] +const _Py_IMMORTAL_MINIMUM_REFCNT: Py_ssize_t = ((1 as c_long) << (30 as c_long)) as Py_ssize_t; + +#[cfg(all(Py_3_14, target_pointer_width = "32"))] +const _Py_STATIC_IMMORTAL_MINIMUM_REFCNT: Py_ssize_t = + ((6 as c_long) << (28 as c_long)) as Py_ssize_t; + +#[cfg(all(Py_3_14, Py_GIL_DISABLED))] +const _Py_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = c_uint::MAX as Py_ssize_t; + +#[cfg(Py_GIL_DISABLED)] +pub(crate) const _Py_IMMORTAL_REFCNT_LOCAL: u32 = u32::MAX; + +#[cfg(Py_GIL_DISABLED)] +const _Py_REF_SHARED_SHIFT: isize = 2; +// skipped private _Py_REF_SHARED_FLAG_MASK + +// skipped private _Py_REF_SHARED_INIT +// skipped private _Py_REF_MAYBE_WEAKREF +// skipped private _Py_REF_QUEUED +// skipped private _Py_REF_MERGED + +// skipped private _Py_REF_SHARED + +extern "C" { + #[cfg(all(Py_3_14, Py_LIMITED_API))] + pub fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t; +} + +#[cfg(not(all(Py_3_14, Py_LIMITED_API)))] +#[inline] +pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { + #[cfg(Py_GIL_DISABLED)] + { + let local = (*ob).ob_ref_local.load(Relaxed); + if local == _Py_IMMORTAL_REFCNT_LOCAL { + #[cfg(not(Py_3_14))] + return _Py_IMMORTAL_REFCNT; + #[cfg(Py_3_14)] + return _Py_IMMORTAL_INITIAL_REFCNT; + } + let shared = (*ob).ob_ref_shared.load(Relaxed); + local as Py_ssize_t + Py_ssize_t::from(shared >> _Py_REF_SHARED_SHIFT) + } + + #[cfg(all(Py_LIMITED_API, Py_3_14))] + { + Py_REFCNT(ob) + } + + #[cfg(all(not(Py_GIL_DISABLED), not(all(Py_LIMITED_API, Py_3_14)), Py_3_12))] + { + (*ob).ob_refcnt.ob_refcnt + } + + #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), not(GraalPy)))] + { + (*ob).ob_refcnt + } + + #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), GraalPy))] + { + _Py_REFCNT(ob) + } +} + +#[cfg(Py_3_12)] +#[inline(always)] +unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { + #[cfg(all(target_pointer_width = "64", not(Py_GIL_DISABLED)))] + { + (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int + } + + #[cfg(all(target_pointer_width = "32", not(Py_GIL_DISABLED)))] + { + #[cfg(not(Py_3_14))] + { + ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int + } + + #[cfg(Py_3_14)] + { + ((*op).ob_refcnt.ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT) as c_int + } + } + + #[cfg(Py_GIL_DISABLED)] + { + ((*op).ob_ref_local.load(Relaxed) == _Py_IMMORTAL_REFCNT_LOCAL) as c_int + } +} + +// skipped _Py_IsStaticImmortal + +// TODO: Py_SET_REFCNT + +extern "C" { + #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] + fn _Py_NegativeRefcount(filename: *const c_char, lineno: c_int, op: *mut PyObject); + #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] + fn _Py_INCREF_IncRefTotal(); + #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] + fn _Py_DECREF_DecRefTotal(); + + #[cfg_attr(PyPy, link_name = "_PyPy_Dealloc")] + fn _Py_Dealloc(arg1: *mut PyObject); + + #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] + #[cfg_attr(GraalPy, link_name = "_Py_IncRef")] + pub fn Py_IncRef(o: *mut PyObject); + #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] + #[cfg_attr(GraalPy, link_name = "_Py_DecRef")] + pub fn Py_DecRef(o: *mut PyObject); + + #[cfg(all(Py_3_10, not(PyPy)))] + fn _Py_IncRef(o: *mut PyObject); + #[cfg(all(Py_3_10, not(PyPy)))] + fn _Py_DecRef(o: *mut PyObject); + + #[cfg(GraalPy)] + fn _Py_REFCNT(arg1: *const PyObject) -> Py_ssize_t; +} + +#[inline(always)] +pub unsafe fn Py_INCREF(op: *mut PyObject) { + // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting + // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. + #[cfg(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + py_sys_config = "Py_REF_DEBUG", + GraalPy + ))] + { + // _Py_IncRef was added to the ABI in 3.10; skips null checks + #[cfg(all(Py_3_10, not(PyPy)))] + { + _Py_IncRef(op); + } + + #[cfg(any(not(Py_3_10), PyPy))] + { + Py_IncRef(op); + } + } + + // version-specific builds are allowed to directly manipulate the reference count + #[cfg(not(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + py_sys_config = "Py_REF_DEBUG", + GraalPy + )))] + { + #[cfg(all(Py_3_14, target_pointer_width = "64"))] + { + let cur_refcnt = (*op).ob_refcnt.ob_refcnt; + if (cur_refcnt as i32) < 0 { + return; + } + (*op).ob_refcnt.ob_refcnt = cur_refcnt.wrapping_add(1); + } + + #[cfg(all(Py_3_12, not(Py_3_14), target_pointer_width = "64"))] + { + let cur_refcnt = (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN]; + let new_refcnt = cur_refcnt.wrapping_add(1); + if new_refcnt == 0 { + return; + } + (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN] = new_refcnt; + } + + #[cfg(all(Py_3_12, target_pointer_width = "32"))] + { + if _Py_IsImmortal(op) != 0 { + return; + } + (*op).ob_refcnt.ob_refcnt += 1 + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt += 1 + } + + // Skipped _Py_INCREF_STAT_INC - if anyone wants this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + } +} + +// skipped _Py_DecRefShared +// skipped _Py_DecRefSharedDebug +// skipped _Py_MergeZeroLocalRefcount + +#[inline(always)] +#[cfg_attr( + all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)), + track_caller +)] +pub unsafe fn Py_DECREF(op: *mut PyObject) { + // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting + // On 3.12+ we implement refcount debugging to get better assertion locations on negative refcounts + // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. + #[cfg(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), + GraalPy + ))] + { + // _Py_DecRef was added to the ABI in 3.10; skips null checks + #[cfg(all(Py_3_10, not(PyPy)))] + { + _Py_DecRef(op); + } + + #[cfg(any(not(Py_3_10), PyPy))] + { + Py_DecRef(op); + } + } + + #[cfg(not(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), + GraalPy + )))] + { + #[cfg(Py_3_12)] + if _Py_IsImmortal(op) != 0 { + return; + } + + // Skipped _Py_DECREF_STAT_INC - if anyone needs this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + + #[cfg(py_sys_config = "Py_REF_DEBUG")] + _Py_DECREF_DecRefTotal(); + + #[cfg(Py_3_12)] + { + (*op).ob_refcnt.ob_refcnt -= 1; + + #[cfg(py_sys_config = "Py_REF_DEBUG")] + if (*op).ob_refcnt.ob_refcnt < 0 { + let location = std::panic::Location::caller(); + let filename = std::ffi::CString::new(location.file()).unwrap(); + _Py_NegativeRefcount(filename.as_ptr(), location.line() as i32, op); + } + + if (*op).ob_refcnt.ob_refcnt == 0 { + _Py_Dealloc(op); + } + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt -= 1; + + if (*op).ob_refcnt == 0 { + _Py_Dealloc(op); + } + } + } +} + +#[inline] +pub unsafe fn Py_CLEAR(op: *mut *mut PyObject) { + let tmp = *op; + if !tmp.is_null() { + *op = ptr::null_mut(); + Py_DECREF(tmp); + } +} + +#[inline] +pub unsafe fn Py_XINCREF(op: *mut PyObject) { + if !op.is_null() { + Py_INCREF(op) + } +} + +#[inline] +pub unsafe fn Py_XDECREF(op: *mut PyObject) { + if !op.is_null() { + Py_DECREF(op) + } +} + +extern "C" { + #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] + #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] + pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject; + #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] + #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] + pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject; +} + +// macro _Py_NewRef not public; reimplemented directly inside Py_NewRef here +// macro _Py_XNewRef not public; reimplemented directly inside Py_XNewRef here + +#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] +#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] +#[inline] +pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { + Py_INCREF(obj); + obj +} + +#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] +#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] +#[inline] +pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { + Py_XINCREF(obj); + obj +} diff --git a/pyo3-ffi/src/setobject.rs b/pyo3-ffi/src/setobject.rs index 84a368a7f27..4c32952238c 100644 --- a/pyo3-ffi/src/setobject.rs +++ b/pyo3-ffi/src/setobject.rs @@ -1,13 +1,13 @@ use crate::object::*; -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] use crate::pyport::Py_hash_t; use crate::pyport::Py_ssize_t; -use std::os::raw::c_int; +use std::ffi::c_int; use std::ptr::addr_of_mut; pub const PySet_MINSIZE: usize = 8; -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct setentry { @@ -15,7 +15,7 @@ pub struct setentry { pub hash: Py_hash_t, } -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PySetObject { @@ -32,18 +32,14 @@ pub struct PySetObject { // skipped #[inline] -#[cfg(all(not(PyPy), not(Py_LIMITED_API)))] +#[cfg(all(not(any(PyPy, GraalPy)), not(Py_LIMITED_API)))] pub unsafe fn PySet_GET_SIZE(so: *mut PyObject) -> Py_ssize_t { debug_assert_eq!(PyAnySet_Check(so), 1); let so = so.cast::(); (*so).used } -#[cfg(not(Py_LIMITED_API))] -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _PySet_Dummy: *mut PyObject; -} +// skipped _PySet_Dummy extern "C" { #[cfg(not(Py_LIMITED_API))] @@ -92,7 +88,7 @@ extern "C" { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyFrozenSet_CheckExact(ob: *mut PyObject) -> c_int { (Py_TYPE(ob) == addr_of_mut!(PyFrozenSet_Type)) as c_int } diff --git a/pyo3-ffi/src/sliceobject.rs b/pyo3-ffi/src/sliceobject.rs index 6f09906fcc4..7f90289e010 100644 --- a/pyo3-ffi/src/sliceobject.rs +++ b/pyo3-ffi/src/sliceobject.rs @@ -1,25 +1,35 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::c_int; +use std::ffi::c_int; use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "_PyPy_EllipsisObject")] static mut _Py_EllipsisObject: PyObject; + + #[cfg(GraalPy)] + static mut _Py_EllipsisObjectReference: *mut PyObject; } #[inline] pub unsafe fn Py_Ellipsis() -> *mut PyObject { - addr_of_mut!(_Py_EllipsisObject) + #[cfg(not(GraalPy))] + return addr_of_mut!(_Py_EllipsisObject); + #[cfg(GraalPy)] + return _Py_EllipsisObjectReference; } #[cfg(not(Py_LIMITED_API))] #[repr(C)] pub struct PySliceObject { pub ob_base: PyObject, + #[cfg(not(GraalPy))] pub start: *mut PyObject, + #[cfg(not(GraalPy))] pub stop: *mut PyObject, + #[cfg(not(GraalPy))] pub step: *mut PyObject, } @@ -83,7 +93,7 @@ extern "C" { step: *mut Py_ssize_t, ) -> c_int; - #[cfg_attr(all(PyPy, Py_3_10), link_name = "PyPySlice_AdjustIndices")] + #[cfg_attr(PyPy, link_name = "PyPySlice_AdjustIndices")] pub fn PySlice_AdjustIndices( length: Py_ssize_t, start: *mut Py_ssize_t, diff --git a/pyo3-ffi/src/structmember.rs b/pyo3-ffi/src/structmember.rs index afae165c5b3..2d9dd5a4a1f 100644 --- a/pyo3-ffi/src/structmember.rs +++ b/pyo3-ffi/src/structmember.rs @@ -1,4 +1,4 @@ -use std::os::raw::c_int; +use std::ffi::c_int; pub use crate::PyMemberDef; diff --git a/pyo3-ffi/src/structseq.rs b/pyo3-ffi/src/structseq.rs index 6dfc1daf158..e5cfee1a876 100644 --- a/pyo3-ffi/src/structseq.rs +++ b/pyo3-ffi/src/structseq.rs @@ -1,7 +1,7 @@ use crate::object::{PyObject, PyTypeObject}; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; #[repr(C)] #[derive(Copy, Clone)] @@ -42,13 +42,13 @@ extern "C" { #[cfg(not(Py_LIMITED_API))] pub type PyStructSequence = crate::PyTupleObject; -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[inline] pub unsafe fn PyStructSequence_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) { crate::PyTuple_SET_ITEM(op, i, v) } -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[inline] pub unsafe fn PyStructSequence_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject { crate::PyTuple_GET_ITEM(op, i) diff --git a/pyo3-ffi/src/sysmodule.rs b/pyo3-ffi/src/sysmodule.rs index 3c552254244..c0a3f176228 100644 --- a/pyo3-ffi/src/sysmodule.rs +++ b/pyo3-ffi/src/sysmodule.rs @@ -1,6 +1,6 @@ use crate::object::PyObject; use libc::wchar_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; extern "C" { #[cfg_attr(PyPy, link_name = "PyPySys_GetObject")] @@ -8,7 +8,19 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPySys_SetObject")] pub fn PySys_SetObject(arg1: *const c_char, arg2: *mut PyObject) -> c_int; + #[cfg_attr( + Py_3_11, + deprecated( + note = "Deprecated in Python 3.11, use `PyConfig.argv` and `PyConfig.parse_argv` instead" + ) + )] pub fn PySys_SetArgv(arg1: c_int, arg2: *mut *mut wchar_t); + #[cfg_attr( + Py_3_11, + deprecated( + note = "Deprecated in Python 3.11, use `PyConfig.argv` and `PyConfig.parse_argv` instead" + ) + )] pub fn PySys_SetArgvEx(arg1: c_int, arg2: *mut *mut wchar_t, arg3: c_int); pub fn PySys_SetPath(arg1: *const wchar_t); @@ -19,6 +31,12 @@ extern "C" { pub fn PySys_FormatStdout(format: *const c_char, ...); pub fn PySys_FormatStderr(format: *const c_char, ...); + #[cfg_attr( + Py_3_13, + deprecated( + note = "Deprecated since Python 3.13. Clear sys.warnoptions and warnings.filters instead." + ) + )] pub fn PySys_ResetWarnOptions(); #[cfg_attr(Py_3_11, deprecated(note = "Python 3.11"))] pub fn PySys_AddWarnOption(arg1: *const wchar_t); diff --git a/pyo3-ffi/src/traceback.rs b/pyo3-ffi/src/traceback.rs index 432b6980ef5..8b1ac216bea 100644 --- a/pyo3-ffi/src/traceback.rs +++ b/pyo3-ffi/src/traceback.rs @@ -1,5 +1,5 @@ use crate::object::*; -use std::os::raw::c_int; +use std::ffi::c_int; #[cfg(not(PyPy))] use std::ptr::addr_of_mut; diff --git a/pyo3-ffi/src/tupleobject.rs b/pyo3-ffi/src/tupleobject.rs index d265c91a4b1..1bd34081bce 100644 --- a/pyo3-ffi/src/tupleobject.rs +++ b/pyo3-ffi/src/tupleobject.rs @@ -1,6 +1,6 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -use std::os::raw::c_int; +use std::ffi::c_int; use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] diff --git a/pyo3-ffi/src/typeslots.rs b/pyo3-ffi/src/typeslots.rs index da7c60b310c..fd0c3b77a74 100644 --- a/pyo3-ffi/src/typeslots.rs +++ b/pyo3-ffi/src/typeslots.rs @@ -1,4 +1,4 @@ -use std::os::raw::c_int; +use std::ffi::c_int; pub const Py_bf_getbuffer: c_int = 1; pub const Py_bf_releasebuffer: c_int = 2; diff --git a/pyo3-ffi/src/unicodeobject.rs b/pyo3-ffi/src/unicodeobject.rs index 087160a1efc..8ba645bfa05 100644 --- a/pyo3-ffi/src/unicodeobject.rs +++ b/pyo3-ffi/src/unicodeobject.rs @@ -1,11 +1,15 @@ use crate::object::*; use crate::pyport::Py_ssize_t; use libc::wchar_t; -use std::os::raw::{c_char, c_int, c_void}; +use std::ffi::{c_char, c_int, c_void}; #[cfg(not(PyPy))] use std::ptr::addr_of_mut; #[cfg(not(Py_LIMITED_API))] +#[cfg_attr( + Py_3_13, + deprecated(note = "Deprecated since Python 3.13. Use `libc::wchar_t` instead.") +)] pub type Py_UNICODE = wchar_t; pub type Py_UCS4 = u32; @@ -328,6 +332,15 @@ extern "C" { pub fn PyUnicode_Compare(left: *mut PyObject, right: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyUnicode_CompareWithASCIIString")] pub fn PyUnicode_CompareWithASCIIString(left: *mut PyObject, right: *const c_char) -> c_int; + #[cfg(Py_3_13)] + pub fn PyUnicode_EqualToUTF8(unicode: *mut PyObject, string: *const c_char) -> c_int; + #[cfg(Py_3_13)] + pub fn PyUnicode_EqualToUTF8AndSize( + unicode: *mut PyObject, + string: *const c_char, + size: Py_ssize_t, + ) -> c_int; + pub fn PyUnicode_RichCompare( left: *mut PyObject, right: *mut PyObject, diff --git a/pyo3-ffi/src/warnings.rs b/pyo3-ffi/src/warnings.rs index d58f37435de..f2cd8adbf9b 100644 --- a/pyo3-ffi/src/warnings.rs +++ b/pyo3-ffi/src/warnings.rs @@ -1,6 +1,6 @@ use crate::object::PyObject; use crate::pyport::Py_ssize_t; -use std::os::raw::{c_char, c_int}; +use std::ffi::{c_char, c_int}; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyErr_WarnEx")] diff --git a/pyo3-ffi/src/weakrefobject.rs b/pyo3-ffi/src/weakrefobject.rs index d065ae23e0f..3921310cee8 100644 --- a/pyo3-ffi/src/weakrefobject.rs +++ b/pyo3-ffi/src/weakrefobject.rs @@ -1,19 +1,21 @@ use crate::object::*; -use std::os::raw::c_int; +use std::ffi::c_int; #[cfg(not(PyPy))] use std::ptr::addr_of_mut; -#[cfg(all(not(PyPy), Py_LIMITED_API))] -opaque_struct!(PyWeakReference); +#[cfg(all(not(PyPy), Py_LIMITED_API, not(GraalPy)))] +opaque_struct!(pub PyWeakReference); -#[cfg(all(not(PyPy), not(Py_LIMITED_API)))] +#[cfg(all(not(PyPy), not(Py_LIMITED_API), not(GraalPy)))] pub use crate::_PyWeakReference as PyWeakReference; #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + // TODO: PyO3 is depending on this symbol in `reference.rs`, we should change this and + // remove the export as this is a private symbol. pub static mut _PyWeakref_RefType: PyTypeObject; - pub static mut _PyWeakref_ProxyType: PyTypeObject; - pub static mut _PyWeakref_CallableProxyType: PyTypeObject; + static mut _PyWeakref_ProxyType: PyTypeObject; + static mut _PyWeakref_CallableProxyType: PyTypeObject; #[cfg(PyPy)] #[link_name = "PyPyWeakref_CheckRef"] @@ -58,5 +60,12 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyWeakref_NewProxy")] pub fn PyWeakref_NewProxy(ob: *mut PyObject, callback: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyWeakref_GetObject")] - pub fn PyWeakref_GetObject(_ref: *mut PyObject) -> *mut PyObject; + #[cfg_attr( + Py_3_13, + deprecated(note = "deprecated since Python 3.13. Use `PyWeakref_GetRef` instead.") + )] + pub fn PyWeakref_GetObject(reference: *mut PyObject) -> *mut PyObject; + #[cfg(Py_3_13)] + #[cfg_attr(PyPy, link_name = "PyPyWeakref_GetRef")] + pub fn PyWeakref_GetRef(reference: *mut PyObject, pobj: *mut *mut PyObject) -> c_int; } diff --git a/pyo3-introspection/Cargo.toml b/pyo3-introspection/Cargo.toml new file mode 100644 index 00000000000..ae78c7d5030 --- /dev/null +++ b/pyo3-introspection/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "pyo3-introspection" +version = "0.27.0" +description = "Introspect dynamic libraries built with PyO3 to get metadata about the exported Python types" +authors = ["PyO3 Project and Contributors "] +homepage = "https://github.com/pyo3/pyo3" +repository = "https://github.com/pyo3/pyo3" +license = "MIT OR Apache-2.0" +edition = "2021" + +[dependencies] +anyhow = "1" +goblin = ">=0.9, <0.11" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +tempfile = "3.12.0" + +[lints] +workspace = true diff --git a/pyo3-introspection/LICENSE-APACHE b/pyo3-introspection/LICENSE-APACHE new file mode 120000 index 00000000000..965b606f331 --- /dev/null +++ b/pyo3-introspection/LICENSE-APACHE @@ -0,0 +1 @@ +../LICENSE-APACHE \ No newline at end of file diff --git a/pyo3-introspection/LICENSE-MIT b/pyo3-introspection/LICENSE-MIT new file mode 120000 index 00000000000..76219eb72e8 --- /dev/null +++ b/pyo3-introspection/LICENSE-MIT @@ -0,0 +1 @@ +../LICENSE-MIT \ No newline at end of file diff --git a/pyo3-introspection/src/introspection.rs b/pyo3-introspection/src/introspection.rs new file mode 100644 index 00000000000..ea57fdaf814 --- /dev/null +++ b/pyo3-introspection/src/introspection.rs @@ -0,0 +1,522 @@ +use crate::model::{ + Argument, Arguments, Attribute, Class, Function, Module, TypeHint, TypeHintExpr, + VariableLengthArgument, +}; +use anyhow::{anyhow, bail, ensure, Context, Result}; +use goblin::elf::section_header::SHN_XINDEX; +use goblin::elf::Elf; +use goblin::mach::load_command::CommandVariant; +use goblin::mach::symbols::{NO_SECT, N_SECT}; +use goblin::mach::{Mach, MachO, SingleArch}; +use goblin::pe::PE; +use goblin::Object; +use serde::de::value::MapAccessDeserializer; +use serde::de::{Error, MapAccess, Visitor}; +use serde::{Deserialize, Deserializer}; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::path::Path; +use std::{fmt, fs, str}; + +/// Introspect a cdylib built with PyO3 and returns the definition of a Python module. +/// +/// This function currently supports the ELF (most *nix including Linux), Match-O (macOS) and PE (Windows) formats. +pub fn introspect_cdylib(library_path: impl AsRef, main_module_name: &str) -> Result { + let chunks = find_introspection_chunks_in_binary_object(library_path.as_ref())?; + parse_chunks(&chunks, main_module_name) +} + +/// Parses the introspection chunks found in the binary +fn parse_chunks(chunks: &[Chunk], main_module_name: &str) -> Result { + let mut chunks_by_id = HashMap::<&str, &Chunk>::new(); + let mut chunks_by_parent = HashMap::<&str, Vec<&Chunk>>::new(); + for chunk in chunks { + let (id, parent) = match chunk { + Chunk::Module { id, .. } | Chunk::Class { id, .. } => (Some(id.as_str()), None), + Chunk::Function { id, parent, .. } | Chunk::Attribute { id, parent, .. } => { + (id.as_deref(), parent.as_deref()) + } + }; + if let Some(id) = id { + chunks_by_id.insert(id, chunk); + } + if let Some(parent) = parent { + chunks_by_parent.entry(parent).or_default().push(chunk); + } + } + // We look for the root chunk + for chunk in chunks { + if let Chunk::Module { + id, + name, + members, + incomplete, + } = chunk + { + if name == main_module_name { + return convert_module( + id, + name, + members, + *incomplete, + &chunks_by_id, + &chunks_by_parent, + ); + } + } + } + bail!("No module named {main_module_name} found") +} + +fn convert_module( + id: &str, + name: &str, + members: &[String], + incomplete: bool, + chunks_by_id: &HashMap<&str, &Chunk>, + chunks_by_parent: &HashMap<&str, Vec<&Chunk>>, +) -> Result { + let (modules, classes, functions, attributes) = convert_members( + members + .iter() + .filter_map(|id| chunks_by_id.get(id.as_str()).copied()) + .chain(chunks_by_parent.get(&id).into_iter().flatten().copied()), + chunks_by_id, + chunks_by_parent, + )?; + + Ok(Module { + name: name.into(), + modules, + classes, + functions, + attributes, + incomplete, + }) +} + +type Members = (Vec, Vec, Vec, Vec); + +/// Convert a list of members of a module or a class +fn convert_members<'a>( + chunks: impl IntoIterator, + chunks_by_id: &HashMap<&str, &Chunk>, + chunks_by_parent: &HashMap<&str, Vec<&Chunk>>, +) -> Result { + let mut modules = Vec::new(); + let mut classes = Vec::new(); + let mut functions = Vec::new(); + let mut attributes = Vec::new(); + for chunk in chunks { + match chunk { + Chunk::Module { + name, + id, + members, + incomplete, + } => { + modules.push(convert_module( + id, + name, + members, + *incomplete, + chunks_by_id, + chunks_by_parent, + )?); + } + Chunk::Class { name, id } => { + classes.push(convert_class(id, name, chunks_by_id, chunks_by_parent)?) + } + Chunk::Function { + name, + id: _, + arguments, + parent: _, + decorators, + returns, + } => functions.push(convert_function(name, arguments, decorators, returns)), + Chunk::Attribute { + name, + id: _, + parent: _, + value, + annotation, + } => attributes.push(convert_attribute(name, value, annotation)), + } + } + // We sort elements to get a stable output + modules.sort_by(|l, r| l.name.cmp(&r.name)); + classes.sort_by(|l, r| l.name.cmp(&r.name)); + functions.sort_by(|l, r| match l.name.cmp(&r.name) { + Ordering::Equal => { + // We put the getter before the setter + if l.decorators.iter().any(|d| d == "property") { + Ordering::Less + } else if r.decorators.iter().any(|d| d == "property") { + Ordering::Greater + } else { + // We pick an ordering based on decorators + l.decorators.cmp(&r.decorators) + } + } + o => o, + }); + attributes.sort_by(|l, r| l.name.cmp(&r.name)); + Ok((modules, classes, functions, attributes)) +} + +fn convert_class( + id: &str, + name: &str, + chunks_by_id: &HashMap<&str, &Chunk>, + chunks_by_parent: &HashMap<&str, Vec<&Chunk>>, +) -> Result { + let (nested_modules, nested_classes, methods, attributes) = convert_members( + chunks_by_parent.get(&id).into_iter().flatten().copied(), + chunks_by_id, + chunks_by_parent, + )?; + ensure!( + nested_modules.is_empty(), + "Classes cannot contain nested modules" + ); + ensure!( + nested_classes.is_empty(), + "Nested classes are not supported yet" + ); + Ok(Class { + name: name.into(), + methods, + attributes, + }) +} + +fn convert_function( + name: &str, + arguments: &ChunkArguments, + decorators: &[String], + returns: &Option, +) -> Function { + Function { + name: name.into(), + decorators: decorators.to_vec(), + arguments: Arguments { + positional_only_arguments: arguments.posonlyargs.iter().map(convert_argument).collect(), + arguments: arguments.args.iter().map(convert_argument).collect(), + vararg: arguments + .vararg + .as_ref() + .map(convert_variable_length_argument), + keyword_only_arguments: arguments.kwonlyargs.iter().map(convert_argument).collect(), + kwarg: arguments + .kwarg + .as_ref() + .map(convert_variable_length_argument), + }, + returns: returns.as_ref().map(convert_type_hint), + } +} + +fn convert_argument(arg: &ChunkArgument) -> Argument { + Argument { + name: arg.name.clone(), + default_value: arg.default.clone(), + annotation: arg.annotation.as_ref().map(convert_type_hint), + } +} + +fn convert_variable_length_argument(arg: &ChunkArgument) -> VariableLengthArgument { + VariableLengthArgument { + name: arg.name.clone(), + annotation: arg.annotation.as_ref().map(convert_type_hint), + } +} + +fn convert_attribute( + name: &str, + value: &Option, + annotation: &Option, +) -> Attribute { + Attribute { + name: name.into(), + value: value.clone(), + annotation: annotation.as_ref().map(convert_type_hint), + } +} + +fn convert_type_hint(arg: &ChunkTypeHint) -> TypeHint { + match arg { + ChunkTypeHint::Ast(expr) => TypeHint::Ast(convert_type_hint_expr(expr)), + ChunkTypeHint::Plain(t) => TypeHint::Plain(t.clone()), + } +} + +fn convert_type_hint_expr(expr: &ChunkTypeHintExpr) -> TypeHintExpr { + match expr { + ChunkTypeHintExpr::Builtin { id } => TypeHintExpr::Builtin { id: id.clone() }, + ChunkTypeHintExpr::Attribute { module, attr } => TypeHintExpr::Attribute { + module: module.clone(), + attr: attr.clone(), + }, + ChunkTypeHintExpr::Union { elts } => TypeHintExpr::Union { + elts: elts.iter().map(convert_type_hint_expr).collect(), + }, + ChunkTypeHintExpr::Subscript { value, slice } => TypeHintExpr::Subscript { + value: Box::new(convert_type_hint_expr(value)), + slice: slice.iter().map(convert_type_hint_expr).collect(), + }, + } +} + +fn find_introspection_chunks_in_binary_object(path: &Path) -> Result> { + let library_content = + fs::read(path).with_context(|| format!("Failed to read {}", path.display()))?; + match Object::parse(&library_content) + .context("The built library is not valid or not supported by our binary parser")? + { + Object::Elf(elf) => find_introspection_chunks_in_elf(&elf, &library_content), + Object::Mach(Mach::Binary(macho)) => { + find_introspection_chunks_in_macho(&macho, &library_content) + } + Object::Mach(Mach::Fat(multi_arch)) => { + for arch in &multi_arch { + match arch? { + SingleArch::MachO(macho) => { + return find_introspection_chunks_in_macho(&macho, &library_content) + } + SingleArch::Archive(_) => (), + } + } + bail!("No Mach-o chunk found in the multi-arch Mach-o container") + } + Object::PE(pe) => find_introspection_chunks_in_pe(&pe, &library_content), + _ => { + bail!("Only ELF, Mach-o and PE containers can be introspected") + } + } +} + +fn find_introspection_chunks_in_elf(elf: &Elf<'_>, library_content: &[u8]) -> Result> { + let mut chunks = Vec::new(); + for sym in &elf.syms { + if is_introspection_symbol(elf.strtab.get_at(sym.st_name).unwrap_or_default()) { + ensure!(u32::try_from(sym.st_shndx)? != SHN_XINDEX, "Section names length is greater than SHN_LORESERVE in ELF, this is not supported by PyO3 yet"); + let section_header = &elf.section_headers[sym.st_shndx]; + let data_offset = sym.st_value + section_header.sh_offset - section_header.sh_addr; + chunks.push(deserialize_chunk( + &library_content[usize::try_from(data_offset).context("File offset overflow")?..], + elf.little_endian, + )?); + } + } + Ok(chunks) +} + +fn find_introspection_chunks_in_macho( + macho: &MachO<'_>, + library_content: &[u8], +) -> Result> { + if !macho.little_endian { + bail!("Only little endian Mach-o binaries are supported"); + } + ensure!( + !macho.load_commands.iter().any(|command| { + matches!(command.command, CommandVariant::DyldChainedFixups(_)) + }), + "Mach-O binaries with fixup chains are not supported yet, to avoid using fixup chains, use `--codegen=link-arg=-no_fixup_chains` option." + ); + + let sections = macho + .segments + .sections() + .flatten() + .map(|t| t.map(|s| s.0)) + .collect::, _>>()?; + let mut chunks = Vec::new(); + for symbol in macho.symbols() { + let (name, nlist) = symbol?; + if nlist.is_global() + && nlist.get_type() == N_SECT + && nlist.n_sect != NO_SECT as usize + && is_introspection_symbol(name) + { + let section = §ions[nlist.n_sect - 1]; // Sections are counted from 1 + let data_offset = nlist.n_value + u64::from(section.offset) - section.addr; + chunks.push(deserialize_chunk( + &library_content[usize::try_from(data_offset).context("File offset overflow")?..], + macho.little_endian, + )?); + } + } + Ok(chunks) +} + +fn find_introspection_chunks_in_pe(pe: &PE<'_>, library_content: &[u8]) -> Result> { + let mut chunks = Vec::new(); + for export in &pe.exports { + if is_introspection_symbol(export.name.unwrap_or_default()) { + chunks.push(deserialize_chunk( + &library_content[export.offset.context("No symbol offset")?..], + true, + )?); + } + } + Ok(chunks) +} + +fn deserialize_chunk( + content_with_chunk_at_the_beginning: &[u8], + is_little_endian: bool, +) -> Result { + let length = content_with_chunk_at_the_beginning + .split_at(4) + .0 + .try_into() + .context("The introspection chunk must contain a length")?; + let length = if is_little_endian { + u32::from_le_bytes(length) + } else { + u32::from_be_bytes(length) + }; + let chunk = content_with_chunk_at_the_beginning + .get(4..4 + length as usize) + .ok_or_else(|| { + anyhow!("The introspection chunk length {length} is greater that the binary size") + })?; + serde_json::from_slice(chunk).with_context(|| { + format!( + "Failed to parse introspection chunk: '{}'", + String::from_utf8_lossy(chunk) + ) + }) +} + +fn is_introspection_symbol(name: &str) -> bool { + name.strip_prefix('_') + .unwrap_or(name) + .starts_with("PYO3_INTROSPECTION_1_") +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum Chunk { + Module { + id: String, + name: String, + members: Vec, + incomplete: bool, + }, + Class { + id: String, + name: String, + }, + Function { + #[serde(default)] + id: Option, + name: String, + arguments: Box, + #[serde(default)] + parent: Option, + #[serde(default)] + decorators: Vec, + #[serde(default, deserialize_with = "deserialize_type_hint")] + returns: Option, + }, + Attribute { + #[serde(default)] + id: Option, + #[serde(default)] + parent: Option, + name: String, + #[serde(default)] + value: Option, + #[serde(default, deserialize_with = "deserialize_type_hint")] + annotation: Option, + }, +} + +#[derive(Deserialize)] +struct ChunkArguments { + #[serde(default)] + posonlyargs: Vec, + #[serde(default)] + args: Vec, + #[serde(default)] + vararg: Option, + #[serde(default)] + kwonlyargs: Vec, + #[serde(default)] + kwarg: Option, +} + +#[derive(Deserialize)] +struct ChunkArgument { + name: String, + #[serde(default)] + default: Option, + #[serde(default, deserialize_with = "deserialize_type_hint")] + annotation: Option, +} + +/// Variant of [`TypeHint`] that implements deserialization. +/// +/// We keep separated type to allow them to evolve independently (this type will need to handle backward compatibility). +enum ChunkTypeHint { + Ast(ChunkTypeHintExpr), + Plain(String), +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum ChunkTypeHintExpr { + Builtin { + id: String, + }, + Attribute { + module: String, + attr: String, + }, + Union { + elts: Vec, + }, + Subscript { + value: Box, + slice: Vec, + }, +} + +fn deserialize_type_hint<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + struct AnnotationVisitor; + + impl<'de> Visitor<'de> for AnnotationVisitor { + type Value = ChunkTypeHint; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("annotation") + } + + fn visit_str(self, v: &str) -> Result + where + E: Error, + { + self.visit_string(v.into()) + } + + fn visit_string(self, v: String) -> Result + where + E: Error, + { + Ok(ChunkTypeHint::Plain(v)) + } + + fn visit_map>(self, map: M) -> Result { + Ok(ChunkTypeHint::Ast(Deserialize::deserialize( + MapAccessDeserializer::new(map), + )?)) + } + } + + Ok(Some(deserializer.deserialize_any(AnnotationVisitor)?)) +} diff --git a/pyo3-introspection/src/lib.rs b/pyo3-introspection/src/lib.rs new file mode 100644 index 00000000000..22aac933e85 --- /dev/null +++ b/pyo3-introspection/src/lib.rs @@ -0,0 +1,8 @@ +//! Utilities to introspect cdylib built using PyO3 and generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html). + +pub use crate::introspection::introspect_cdylib; +pub use crate::stubs::module_stub_files; + +mod introspection; +pub mod model; +mod stubs; diff --git a/pyo3-introspection/src/model.rs b/pyo3-introspection/src/model.rs new file mode 100644 index 00000000000..5d13d15c51b --- /dev/null +++ b/pyo3-introspection/src/model.rs @@ -0,0 +1,91 @@ +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Module { + pub name: String, + pub modules: Vec, + pub classes: Vec, + pub functions: Vec, + pub attributes: Vec, + pub incomplete: bool, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Class { + pub name: String, + pub methods: Vec, + pub attributes: Vec, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Function { + pub name: String, + /// decorator like 'property' or 'staticmethod' + pub decorators: Vec, + pub arguments: Arguments, + /// return type + pub returns: Option, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Attribute { + pub name: String, + /// Value as a Python expression if easily expressible + pub value: Option, + /// Type annotation as a Python expression + pub annotation: Option, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Arguments { + /// Arguments before / + pub positional_only_arguments: Vec, + /// Regular arguments (between / and *) + pub arguments: Vec, + /// *vararg + pub vararg: Option, + /// Arguments after * + pub keyword_only_arguments: Vec, + /// **kwarg + pub kwarg: Option, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Argument { + pub name: String, + /// Default value as a Python expression + pub default_value: Option, + /// Type annotation as a Python expression + pub annotation: Option, +} + +/// A variable length argument ie. *vararg or **kwarg +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct VariableLengthArgument { + pub name: String, + /// Type annotation as a Python expression + pub annotation: Option, +} + +/// A type hint annotation +/// +/// Might be a plain string or an AST fragment +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub enum TypeHint { + Ast(TypeHintExpr), + Plain(String), +} + +/// A type hint annotation as an AST fragment +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub enum TypeHintExpr { + /// A Python builtin like `int` + Builtin { id: String }, + /// The attribute of a python object like `{value}.{attr}` + Attribute { module: String, attr: String }, + /// A union `{left} | {right}` + Union { elts: Vec }, + /// A subscript `{value}[*slice]` + Subscript { + value: Box, + slice: Vec, + }, +} diff --git a/pyo3-introspection/src/stubs.rs b/pyo3-introspection/src/stubs.rs new file mode 100644 index 00000000000..d66d899ce00 --- /dev/null +++ b/pyo3-introspection/src/stubs.rs @@ -0,0 +1,638 @@ +use crate::model::{ + Argument, Arguments, Attribute, Class, Function, Module, TypeHint, TypeHintExpr, + VariableLengthArgument, +}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::PathBuf; +use std::str::FromStr; + +/// Generates the [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html) of a given module. +/// It returns a map between the file name and the file content. +/// The root module stubs will be in the `__init__.pyi` file and the submodules directory +/// in files with a relevant name. +pub fn module_stub_files(module: &Module) -> HashMap { + let mut output_files = HashMap::new(); + add_module_stub_files(module, &[], &mut output_files); + output_files +} + +fn add_module_stub_files( + module: &Module, + module_path: &[&str], + output_files: &mut HashMap, +) { + let mut file_path = PathBuf::new(); + for e in module_path { + file_path = file_path.join(e); + } + output_files.insert( + file_path.join("__init__.pyi"), + module_stubs(module, module_path), + ); + let mut module_path = module_path.to_vec(); + module_path.push(&module.name); + for submodule in &module.modules { + if submodule.modules.is_empty() { + output_files.insert( + file_path.join(format!("{}.pyi", submodule.name)), + module_stubs(submodule, &module_path), + ); + } else { + add_module_stub_files(submodule, &module_path, output_files); + } + } +} + +/// Generates the module stubs to a String, not including submodules +fn module_stubs(module: &Module, parents: &[&str]) -> String { + let imports = Imports::create(module, parents); + let mut elements = Vec::new(); + for attribute in &module.attributes { + elements.push(attribute_stubs(attribute, &imports)); + } + for class in &module.classes { + elements.push(class_stubs(class, &imports)); + } + for function in &module.functions { + elements.push(function_stubs(function, &imports)); + } + + // We generate a __getattr__ method to tag incomplete stubs + // See https://typing.python.org/en/latest/guides/writing_stubs.html#incomplete-stubs + if module.incomplete && !module.functions.iter().any(|f| f.name == "__getattr__") { + elements.push(function_stubs( + &Function { + name: "__getattr__".into(), + decorators: Vec::new(), + arguments: Arguments { + positional_only_arguments: Vec::new(), + arguments: vec![Argument { + name: "name".to_string(), + default_value: None, + annotation: Some(TypeHint::Ast(TypeHintExpr::Builtin { id: "str".into() })), + }], + vararg: None, + keyword_only_arguments: Vec::new(), + kwarg: None, + }, + returns: Some(TypeHint::Ast(TypeHintExpr::Attribute { + module: "_typeshed".into(), + attr: "Incomplete".into(), + })), + }, + &imports, + )); + } + + let mut final_elements = imports.imports; + final_elements.extend(elements); + + let mut output = String::new(); + + // We insert two line jumps (i.e. empty strings) only above and below multiple line elements (classes with methods, functions with decorators) + for element in final_elements { + let is_multiline = element.contains('\n'); + if is_multiline && !output.is_empty() && !output.ends_with("\n\n") { + output.push('\n'); + } + output.push_str(&element); + output.push('\n'); + if is_multiline { + output.push('\n'); + } + } + + // We remove a line jump at the end if they are two + if output.ends_with("\n\n") { + output.pop(); + } + output +} + +fn class_stubs(class: &Class, imports: &Imports) -> String { + let mut buffer = format!("class {}:", class.name); + if class.methods.is_empty() && class.attributes.is_empty() { + buffer.push_str(" ..."); + return buffer; + } + for attribute in &class.attributes { + // We do the indentation + buffer.push_str("\n "); + buffer.push_str(&attribute_stubs(attribute, imports).replace('\n', "\n ")); + } + for method in &class.methods { + // We do the indentation + buffer.push_str("\n "); + buffer.push_str(&function_stubs(method, imports).replace('\n', "\n ")); + } + buffer +} + +fn function_stubs(function: &Function, imports: &Imports) -> String { + // Signature + let mut parameters = Vec::new(); + for argument in &function.arguments.positional_only_arguments { + parameters.push(argument_stub(argument, imports)); + } + if !function.arguments.positional_only_arguments.is_empty() { + parameters.push("/".into()); + } + for argument in &function.arguments.arguments { + parameters.push(argument_stub(argument, imports)); + } + if let Some(argument) = &function.arguments.vararg { + parameters.push(format!( + "*{}", + variable_length_argument_stub(argument, imports) + )); + } else if !function.arguments.keyword_only_arguments.is_empty() { + parameters.push("*".into()); + } + for argument in &function.arguments.keyword_only_arguments { + parameters.push(argument_stub(argument, imports)); + } + if let Some(argument) = &function.arguments.kwarg { + parameters.push(format!( + "**{}", + variable_length_argument_stub(argument, imports) + )); + } + let mut buffer = String::new(); + for decorator in &function.decorators { + buffer.push('@'); + buffer.push_str(decorator); + buffer.push('\n'); + } + buffer.push_str("def "); + buffer.push_str(&function.name); + buffer.push('('); + buffer.push_str(¶meters.join(", ")); + buffer.push(')'); + if let Some(returns) = &function.returns { + buffer.push_str(" -> "); + type_hint_stub(returns, imports, &mut buffer); + } + buffer.push_str(": ..."); + buffer +} + +fn attribute_stubs(attribute: &Attribute, imports: &Imports) -> String { + let mut buffer = attribute.name.clone(); + if let Some(annotation) = &attribute.annotation { + buffer.push_str(": "); + type_hint_stub(annotation, imports, &mut buffer); + } + if let Some(value) = &attribute.value { + buffer.push_str(" = "); + buffer.push_str(value); + } + buffer +} + +fn argument_stub(argument: &Argument, imports: &Imports) -> String { + let mut buffer = argument.name.clone(); + if let Some(annotation) = &argument.annotation { + buffer.push_str(": "); + type_hint_stub(annotation, imports, &mut buffer); + } + if let Some(default_value) = &argument.default_value { + buffer.push_str(if argument.annotation.is_some() { + " = " + } else { + "=" + }); + buffer.push_str(default_value); + } + buffer +} + +fn variable_length_argument_stub(argument: &VariableLengthArgument, imports: &Imports) -> String { + let mut buffer = argument.name.clone(); + if let Some(annotation) = &argument.annotation { + buffer.push_str(": "); + type_hint_stub(annotation, imports, &mut buffer); + } + buffer +} + +fn type_hint_stub(type_hint: &TypeHint, imports: &Imports, buffer: &mut String) { + match type_hint { + TypeHint::Ast(t) => imports.serialize_type_hint(t, buffer), + TypeHint::Plain(t) => buffer.push_str(t), + } +} + +/// Datastructure to deduplicate, validate and generate imports +#[derive(Default)] +struct Imports { + /// Import lines ready to use + imports: Vec, + /// Renaming map: from module name and member name return the name to use in type hints + renaming: BTreeMap<(String, String), String>, +} + +impl Imports { + /// This generates a map from the builtin or module name to the actual alias used in the file + /// + /// For Python builtins and elements declared by the module the alias is always the actual name. + /// + /// For other elements, we can alias them using the `from X import Y as Z` syntax. + /// So, we first list all builtins and local elements, then iterate on imports + /// and create the aliases when needed. + fn create(module: &Module, module_parents: &[&str]) -> Self { + let mut elements_used_in_annotations = ElementsUsedInAnnotations::new(); + elements_used_in_annotations.walk_module(module); + + let mut imports = Vec::new(); + let mut renaming = BTreeMap::new(); + let mut local_name_to_module_and_attribute = BTreeMap::new(); + + // We first process local and built-ins elements, they are never aliased or imported + for name in module + .classes + .iter() + .map(|c| c.name.clone()) + .chain(module.functions.iter().map(|f| f.name.clone())) + .chain(module.attributes.iter().map(|a| a.name.clone())) + .chain(elements_used_in_annotations.builtins) + { + local_name_to_module_and_attribute.insert(name.clone(), (None, name.clone())); + } + + // We compute the set of ways the current module can be named + let mut possible_current_module_names = vec![module.name.clone()]; + let mut current_module_name = Some(module.name.clone()); + for parent in module_parents.iter().rev() { + let path = if let Some(current) = current_module_name { + format!("{parent}.{current}") + } else { + parent.to_string() + }; + possible_current_module_names.push(path.clone()); + current_module_name = Some(path); + } + + // We process then imports, normalizing local imports + for (module, attrs) in elements_used_in_annotations.module_members { + let normalized_module = if possible_current_module_names.contains(&module) { + None + } else { + Some(module.clone()) + }; + let mut import_for_module = Vec::new(); + for attr in attrs { + // We split nested classes A.B in "A" (the part that must be imported and can have naming conflicts) and ".B" + let (root_attr, attr_path) = attr + .split_once('.') + .map_or((attr.as_str(), None), |(root, path)| (root, Some(path))); + let mut local_name = root_attr.to_owned(); + let mut already_imported = false; + while let Some((possible_conflict_module, possible_conflict_attr)) = + local_name_to_module_and_attribute.get(&local_name) + { + if *possible_conflict_module == normalized_module + && *possible_conflict_attr == root_attr + { + // It's the same + already_imported = true; + break; + } + // We generate a new local name + // TODO: we use currently a format like Foo2. It might be nicer to use something like ModFoo + let number_of_digits_at_the_end = local_name + .bytes() + .rev() + .take_while(|b| b.is_ascii_digit()) + .count(); + let (local_name_prefix, local_name_number) = + local_name.split_at(local_name.len() - number_of_digits_at_the_end); + local_name = format!( + "{local_name_prefix}{}", + u64::from_str(local_name_number).unwrap_or(1) + 1 + ); + } + renaming.insert( + (module.clone(), attr.clone()), + if let Some(attr_path) = attr_path { + format!("{local_name}.{attr_path}") + } else { + local_name.clone() + }, + ); + if !already_imported { + local_name_to_module_and_attribute.insert( + local_name.clone(), + (normalized_module.clone(), root_attr.to_owned()), + ); + import_for_module.push(if local_name == root_attr { + local_name + } else { + format!("{root_attr} as {local_name}") + }); + } + } + if let Some(module) = normalized_module { + imports.push(format!( + "from {module} import {}", + import_for_module.join(", ") + )); + } + } + + Self { imports, renaming } + } + + fn serialize_type_hint(&self, expr: &TypeHintExpr, buffer: &mut String) { + match expr { + TypeHintExpr::Builtin { id } => buffer.push_str(id), + TypeHintExpr::Attribute { module, attr } => { + let alias = self + .renaming + .get(&(module.clone(), attr.clone())) + .expect("All type hint attributes should have been visited"); + buffer.push_str(alias) + } + TypeHintExpr::Union { elts } => { + for (i, elt) in elts.iter().enumerate() { + if i > 0 { + buffer.push_str(" | "); + } + self.serialize_type_hint(elt, buffer); + } + } + TypeHintExpr::Subscript { value, slice } => { + self.serialize_type_hint(value, buffer); + buffer.push('['); + for (i, elt) in slice.iter().enumerate() { + if i > 0 { + buffer.push_str(", "); + } + self.serialize_type_hint(elt, buffer); + } + buffer.push(']'); + } + } + } +} + +/// Lists all the elements used in annotations +struct ElementsUsedInAnnotations { + /// module -> name + module_members: BTreeMap>, + builtins: BTreeSet, +} + +impl ElementsUsedInAnnotations { + fn new() -> Self { + Self { + module_members: BTreeMap::new(), + builtins: BTreeSet::new(), + } + } + + fn walk_module(&mut self, module: &Module) { + for attr in &module.attributes { + self.walk_attribute(attr); + } + for class in &module.classes { + self.walk_class(class); + } + for function in &module.functions { + self.walk_function(function); + } + if module.incomplete { + self.builtins.insert("str".into()); + self.module_members + .entry("_typeshed".into()) + .or_default() + .insert("Incomplete".into()); + } + } + + fn walk_class(&mut self, class: &Class) { + for method in &class.methods { + self.walk_function(method); + } + for attr in &class.attributes { + self.walk_attribute(attr); + } + } + + fn walk_attribute(&mut self, attribute: &Attribute) { + if let Some(type_hint) = &attribute.annotation { + self.walk_type_hint(type_hint); + } + } + + fn walk_function(&mut self, function: &Function) { + for decorator in &function.decorators { + self.builtins.insert(decorator.clone()); + } + for arg in function + .arguments + .positional_only_arguments + .iter() + .chain(&function.arguments.arguments) + .chain(&function.arguments.keyword_only_arguments) + { + if let Some(type_hint) = &arg.annotation { + self.walk_type_hint(type_hint); + } + } + for arg in function + .arguments + .vararg + .as_ref() + .iter() + .chain(&function.arguments.kwarg.as_ref()) + { + if let Some(type_hint) = &arg.annotation { + self.walk_type_hint(type_hint); + } + } + if let Some(type_hint) = &function.returns { + self.walk_type_hint(type_hint); + } + } + + fn walk_type_hint(&mut self, type_hint: &TypeHint) { + if let TypeHint::Ast(type_hint) = type_hint { + self.walk_type_hint_expr(type_hint); + } + } + + fn walk_type_hint_expr(&mut self, expr: &TypeHintExpr) { + match expr { + TypeHintExpr::Builtin { id } => { + self.builtins.insert(id.clone()); + } + TypeHintExpr::Attribute { module, attr } => { + self.module_members + .entry(module.clone()) + .or_default() + .insert(attr.clone()); + } + TypeHintExpr::Union { elts } => { + for elt in elts { + self.walk_type_hint_expr(elt) + } + } + TypeHintExpr::Subscript { value, slice } => { + self.walk_type_hint_expr(value); + for elt in slice { + self.walk_type_hint_expr(elt); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::Arguments; + + #[test] + fn function_stubs_with_variable_length() { + let function = Function { + name: "func".into(), + decorators: Vec::new(), + arguments: Arguments { + positional_only_arguments: vec![Argument { + name: "posonly".into(), + default_value: None, + annotation: None, + }], + arguments: vec![Argument { + name: "arg".into(), + default_value: None, + annotation: None, + }], + vararg: Some(VariableLengthArgument { + name: "varargs".into(), + annotation: None, + }), + keyword_only_arguments: vec![Argument { + name: "karg".into(), + default_value: None, + annotation: Some(TypeHint::Plain("str".into())), + }], + kwarg: Some(VariableLengthArgument { + name: "kwarg".into(), + annotation: Some(TypeHint::Plain("str".into())), + }), + }, + returns: Some(TypeHint::Plain("list[str]".into())), + }; + assert_eq!( + "def func(posonly, /, arg, *varargs, karg: str, **kwarg: str) -> list[str]: ...", + function_stubs(&function, &Imports::default()) + ) + } + + #[test] + fn function_stubs_without_variable_length() { + let function = Function { + name: "afunc".into(), + decorators: Vec::new(), + arguments: Arguments { + positional_only_arguments: vec![Argument { + name: "posonly".into(), + default_value: Some("1".into()), + annotation: None, + }], + arguments: vec![Argument { + name: "arg".into(), + default_value: Some("True".into()), + annotation: None, + }], + vararg: None, + keyword_only_arguments: vec![Argument { + name: "karg".into(), + default_value: Some("\"foo\"".into()), + annotation: Some(TypeHint::Plain("str".into())), + }], + kwarg: None, + }, + returns: None, + }; + assert_eq!( + "def afunc(posonly=1, /, arg=True, *, karg: str = \"foo\"): ...", + function_stubs(&function, &Imports::default()) + ) + } + + #[test] + fn test_import() { + let big_type = TypeHintExpr::Subscript { + value: Box::new(TypeHintExpr::Builtin { id: "dict".into() }), + slice: vec![ + TypeHintExpr::Attribute { + module: "foo.bar".into(), + attr: "A".into(), + }, + TypeHintExpr::Union { + elts: vec![ + TypeHintExpr::Attribute { + module: "bar".into(), + attr: "A".into(), + }, + TypeHintExpr::Attribute { + module: "foo".into(), + attr: "A.C".into(), + }, + TypeHintExpr::Attribute { + module: "foo".into(), + attr: "A.D".into(), + }, + TypeHintExpr::Attribute { + module: "foo".into(), + attr: "B".into(), + }, + TypeHintExpr::Attribute { + module: "bat".into(), + attr: "A".into(), + }, + ], + }, + ], + }; + let imports = Imports::create( + &Module { + name: "bar".into(), + modules: Vec::new(), + classes: vec![Class { + name: "A".into(), + methods: Vec::new(), + attributes: Vec::new(), + }], + functions: vec![Function { + name: String::new(), + decorators: Vec::new(), + arguments: Arguments { + positional_only_arguments: Vec::new(), + arguments: Vec::new(), + vararg: None, + keyword_only_arguments: Vec::new(), + kwarg: None, + }, + returns: Some(TypeHint::Ast(big_type.clone())), + }], + attributes: Vec::new(), + incomplete: true, + }, + &["foo"], + ); + assert_eq!( + &imports.imports, + &[ + "from _typeshed import Incomplete", + "from bat import A as A2", + "from foo import A as A3, B" + ] + ); + let mut output = String::new(); + imports.serialize_type_hint(&big_type, &mut output); + assert_eq!(output, "dict[A, A | A3.C | A3.D | B | A2]"); + } +} diff --git a/pyo3-introspection/tests/test.rs b/pyo3-introspection/tests/test.rs new file mode 100644 index 00000000000..cf01329e9b1 --- /dev/null +++ b/pyo3-introspection/tests/test.rs @@ -0,0 +1,106 @@ +use anyhow::{ensure, Result}; +use pyo3_introspection::{introspect_cdylib, module_stub_files}; +use std::collections::HashMap; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{env, fs}; +use tempfile::NamedTempFile; + +#[test] +fn pytests_stubs() -> Result<()> { + // We run the introspection + let binary = env::var_os("PYO3_PYTEST_LIB_PATH") + .expect("The PYO3_PYTEST_LIB_PATH constant must be set and target the pyo3-pytests cdylib"); + let module = introspect_cdylib(binary, "pyo3_pytests")?; + let actual_stubs = module_stub_files(&module); + + // We read the expected stubs + let expected_subs_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("pytests") + .join("stubs"); + let mut expected_subs = HashMap::new(); + add_dir_files( + &expected_subs_dir, + &expected_subs_dir.canonicalize()?, + &mut expected_subs, + )?; + + // We ensure we do not have extra generated files + for file_name in actual_stubs.keys() { + assert!( + expected_subs.contains_key(file_name), + "The generated file {} is not in the expected stubs directory pytests/stubs", + file_name.display() + ); + } + + // We ensure the expected files are generated properly + for (file_name, expected_file_content) in &expected_subs { + let actual_file_content = actual_stubs.get(file_name).unwrap_or_else(|| { + panic!( + "The expected stub file {} has not been generated", + file_name.display() + ) + }); + + let actual_file_content = format_with_ruff(actual_file_content)?; + + // We normalize line jumps for compatibility with Windows + assert_eq!( + expected_file_content.replace('\r', ""), + actual_file_content.replace('\r', ""), + "The content of file {} is different", + file_name.display() + ) + } + + Ok(()) +} + +fn add_dir_files( + dir_path: &Path, + base_dir_path: &Path, + output: &mut HashMap, +) -> Result<()> { + for entry in fs::read_dir(dir_path)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + add_dir_files(&entry.path(), base_dir_path, output)?; + } else { + output.insert( + entry + .path() + .canonicalize()? + .strip_prefix(base_dir_path)? + .into(), + fs::read_to_string(entry.path())?, + ); + } + } + Ok(()) +} + +fn format_with_ruff(code: &str) -> Result { + let temp_file = NamedTempFile::with_suffix(".pyi")?; + // Write to file + { + let mut file = temp_file.as_file(); + file.write_all(code.as_bytes())?; + file.flush()?; + file.seek(SeekFrom::Start(0))?; + } + ensure!( + Command::new("ruff") + .arg("format") + .arg(temp_file.path()) + .status()? + .success(), + "Failed to run ruff" + ); + let mut content = String::new(); + temp_file.as_file().read_to_string(&mut content)?; + Ok(content) +} diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 458b280f881..eadee73d0c6 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.21.0-dev" +version = "0.27.0" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -9,20 +9,29 @@ repository = "https://github.com/pyo3/pyo3" categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" edition = "2021" +rust-version.workspace = true # Note: we use default-features = false for proc-macro related crates # not to depend on proc-macro itself. # See https://github.com/PyO3/pyo3/pull/810 for more. [dependencies] -heck = "0.4" -proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "0.21.0-dev", features = ["resolve-config"] } -quote = { version = "1", default-features = false } +heck = "0.5" +proc-macro2 = { version = "1.0.60", default-features = false } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.27.0", features = ["resolve-config"] } +quote = { version = "1.0.37", default-features = false } [dependencies.syn] -version = "2" +# 2.0.59 for `LitCStr` +version = "2.0.59" default-features = false -features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] +features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits", "visit-mut"] + +[build-dependencies] +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.27.0" } [lints] workspace = true + +[features] +experimental-async = [] +experimental-inspect = [] diff --git a/pyo3-macros-backend/build.rs b/pyo3-macros-backend/build.rs new file mode 100644 index 00000000000..55aa0ba03c5 --- /dev/null +++ b/pyo3-macros-backend/build.rs @@ -0,0 +1,4 @@ +fn main() { + pyo3_build_config::print_expected_cfgs(); + pyo3_build_config::print_feature_cfgs(); +} diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index e91b3b8d9a2..6e7de98e318 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -1,40 +1,179 @@ use proc_macro2::TokenStream; -use quote::ToTokens; +use quote::{quote, ToTokens}; +use syn::parse::Parser; use syn::{ + ext::IdentExt, parse::{Parse, ParseStream}, punctuated::Punctuated, spanned::Spanned, token::Comma, - Attribute, Expr, ExprPath, Ident, LitStr, Path, Result, Token, + Attribute, Expr, ExprPath, Ident, Index, LitBool, LitStr, Member, Path, Result, Token, }; +use crate::combine_errors::CombineErrors; + pub mod kw { syn::custom_keyword!(annotation); syn::custom_keyword!(attribute); syn::custom_keyword!(cancel_handle); + syn::custom_keyword!(constructor); syn::custom_keyword!(dict); + syn::custom_keyword!(eq); + syn::custom_keyword!(eq_int); syn::custom_keyword!(extends); syn::custom_keyword!(freelist); syn::custom_keyword!(from_py_with); syn::custom_keyword!(frozen); syn::custom_keyword!(get); syn::custom_keyword!(get_all); + syn::custom_keyword!(hash); + syn::custom_keyword!(into_py_with); syn::custom_keyword!(item); + syn::custom_keyword!(immutable_type); syn::custom_keyword!(from_item_all); syn::custom_keyword!(mapping); syn::custom_keyword!(module); syn::custom_keyword!(name); + syn::custom_keyword!(ord); syn::custom_keyword!(pass_module); syn::custom_keyword!(rename_all); syn::custom_keyword!(sequence); syn::custom_keyword!(set); syn::custom_keyword!(set_all); syn::custom_keyword!(signature); + syn::custom_keyword!(str); syn::custom_keyword!(subclass); + syn::custom_keyword!(submodule); syn::custom_keyword!(text_signature); syn::custom_keyword!(transparent); syn::custom_keyword!(unsendable); syn::custom_keyword!(weakref); + syn::custom_keyword!(generic); + syn::custom_keyword!(gil_used); + syn::custom_keyword!(warn); + syn::custom_keyword!(message); + syn::custom_keyword!(category); + syn::custom_keyword!(from_py_object); + syn::custom_keyword!(skip_from_py_object); +} + +fn take_int(read: &mut &str, tracker: &mut usize) -> String { + let mut int = String::new(); + for (i, ch) in read.char_indices() { + match ch { + '0'..='9' => { + *tracker += 1; + int.push(ch) + } + _ => { + *read = &read[i..]; + break; + } + } + } + int +} + +fn take_ident(read: &mut &str, tracker: &mut usize) -> Ident { + let mut ident = String::new(); + if read.starts_with("r#") { + ident.push_str("r#"); + *tracker += 2; + *read = &read[2..]; + } + for (i, ch) in read.char_indices() { + match ch { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => { + *tracker += 1; + ident.push(ch) + } + _ => { + *read = &read[i..]; + break; + } + } + } + Ident::parse_any.parse_str(&ident).unwrap() +} + +// shorthand parsing logic inspiration taken from https://github.com/dtolnay/thiserror/blob/master/impl/src/fmt.rs +fn parse_shorthand_format(fmt: LitStr) -> Result<(LitStr, Vec)> { + let span = fmt.span(); + let token = fmt.token(); + let value = fmt.value(); + let mut read = value.as_str(); + let mut out = String::new(); + let mut members = Vec::new(); + let mut tracker = 1; + while let Some(brace) = read.find('{') { + tracker += brace; + out += &read[..brace + 1]; + read = &read[brace + 1..]; + if read.starts_with('{') { + out.push('{'); + read = &read[1..]; + tracker += 2; + continue; + } + let next = match read.chars().next() { + Some(next) => next, + None => break, + }; + tracker += 1; + let member = match next { + '0'..='9' => { + let start = tracker; + let index = take_int(&mut read, &mut tracker).parse::().unwrap(); + let end = tracker; + let subspan = token.subspan(start..end).unwrap_or(span); + let idx = Index { + index, + span: subspan, + }; + Member::Unnamed(idx) + } + 'a'..='z' | 'A'..='Z' | '_' => { + let start = tracker; + let mut ident = take_ident(&mut read, &mut tracker); + let end = tracker; + let subspan = token.subspan(start..end).unwrap_or(span); + ident.set_span(subspan); + Member::Named(ident) + } + '}' | ':' => { + let start = tracker; + tracker += 1; + let end = tracker; + let subspan = token.subspan(start..end).unwrap_or(span); + // we found a closing bracket or formatting ':' without finding a member, we assume the user wants the instance formatted here + bail_spanned!(subspan.span() => "No member found, you must provide a named or positionally specified member.") + } + _ => continue, + }; + members.push(member); + } + out += read; + Ok((LitStr::new(&out, span), members)) +} + +#[derive(Clone, Debug)] +pub struct StringFormatter { + pub fmt: LitStr, + pub args: Vec, +} + +impl Parse for crate::attributes::StringFormatter { + fn parse(input: ParseStream<'_>) -> Result { + let (fmt, args) = parse_shorthand_format(input.parse()?)?; + Ok(Self { fmt, args }) + } +} + +impl ToTokens for crate::attributes::StringFormatter { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.fmt.to_tokens(tokens); + tokens.extend(quote! {self.args}) + } } #[derive(Clone, Debug)] @@ -43,6 +182,12 @@ pub struct KeywordAttribute { pub value: V, } +#[derive(Clone, Debug)] +pub struct OptionalKeywordAttribute { + pub kw: K, + pub value: Option, +} + /// A helper type which parses the inner type via a literal string /// e.g. `LitStrValue` -> parses "some::path" in quotes. #[derive(Clone, Debug, PartialEq, Eq)] @@ -68,7 +213,7 @@ pub struct NameLitStr(pub Ident); impl Parse for NameLitStr { fn parse(input: ParseStream<'_>) -> Result { let string_literal: LitStr = input.parse()?; - if let Ok(ident) = string_literal.parse() { + if let Ok(ident) = string_literal.parse_with(Ident::parse_any) { Ok(NameLitStr(ident)) } else { bail_spanned!(string_literal.span() => "expected a single identifier in double quotes") @@ -131,7 +276,7 @@ impl ToTokens for RenamingRuleLitStr { } } -/// Text signatue can be either a literal string or opt-in/out +/// Text signature can be either a literal string or opt-in/out #[derive(Clone, Debug, PartialEq, Eq)] pub enum TextSignatureAttributeValue { Str(LitStr), @@ -171,7 +316,10 @@ pub type FreelistAttribute = KeywordAttribute>; pub type ModuleAttribute = KeywordAttribute; pub type NameAttribute = KeywordAttribute; pub type RenameAllAttribute = KeywordAttribute; +pub type StrFormatterAttribute = OptionalKeywordAttribute; pub type TextSignatureAttribute = KeywordAttribute; +pub type SubmoduleAttribute = kw::submodule; +pub type GILUsedAttribute = KeywordAttribute; impl Parse for KeywordAttribute { fn parse(input: ParseStream<'_>) -> Result { @@ -190,7 +338,31 @@ impl ToTokens for KeywordAttribute { } } -pub type FromPyWithAttribute = KeywordAttribute>; +impl Parse for OptionalKeywordAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let kw: K = input.parse()?; + let value = match input.parse::() { + Ok(_) => Some(input.parse()?), + Err(_) => None, + }; + Ok(OptionalKeywordAttribute { kw, value }) + } +} + +impl ToTokens for OptionalKeywordAttribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.kw.to_tokens(tokens); + if self.value.is_some() { + Token![=](self.kw.span()).to_tokens(tokens); + self.value.to_tokens(tokens); + } + } +} + +pub type FromPyWithAttribute = KeywordAttribute; +pub type IntoPyWithAttribute = KeywordAttribute; + +pub type DefaultAttribute = OptionalKeywordAttribute; /// For specifying the path to the pyo3 crate. pub type CrateAttribute = KeywordAttribute>; @@ -227,13 +399,23 @@ pub fn take_attributes( pub fn take_pyo3_options(attrs: &mut Vec) -> Result> { let mut out = Vec::new(); - take_attributes(attrs, |attr| { - if let Some(options) = get_pyo3_options(attr)? { - out.extend(options); + + take_attributes(attrs, |attr| match get_pyo3_options(attr) { + Ok(result) => { + if let Some(options) = result { + out.extend(options.into_iter().map(|a| Ok(a))); + Ok(true) + } else { + Ok(false) + } + } + Err(err) => { + out.push(Err(err)); Ok(true) - } else { - Ok(false) } })?; + + let out: Vec = out.into_iter().try_combine_syn_errors()?; + Ok(out) } diff --git a/pyo3-macros-backend/src/combine_errors.rs b/pyo3-macros-backend/src/combine_errors.rs new file mode 100644 index 00000000000..235831ac486 --- /dev/null +++ b/pyo3-macros-backend/src/combine_errors.rs @@ -0,0 +1,36 @@ +pub(crate) trait CombineErrors: Iterator { + type Ok; + fn try_combine_syn_errors(self) -> syn::Result>; +} + +impl CombineErrors for I +where + I: Iterator>, +{ + type Ok = T; + + fn try_combine_syn_errors(self) -> syn::Result> { + let mut oks: Vec = Vec::new(); + let mut errors: Vec = Vec::new(); + + for res in self { + match res { + Ok(val) => oks.push(val), + Err(e) => errors.push(e), + } + } + + let mut err_iter = errors.into_iter(); + let mut err = match err_iter.next() { + // There are no errors + None => return Ok(oks), + Some(e) => e, + }; + + for e in err_iter { + err.combine(e); + } + + Err(err) + } +} diff --git a/pyo3-macros-backend/src/deprecations.rs b/pyo3-macros-backend/src/deprecations.rs deleted file mode 100644 index ea2922737b9..00000000000 --- a/pyo3-macros-backend/src/deprecations.rs +++ /dev/null @@ -1,44 +0,0 @@ -use proc_macro2::{Span, TokenStream}; -use quote::{quote_spanned, ToTokens}; - -pub enum Deprecation { - PyMethodsNewDeprecatedForm, -} - -impl Deprecation { - fn ident(&self, span: Span) -> syn::Ident { - let string = match self { - Deprecation::PyMethodsNewDeprecatedForm => "PYMETHODS_NEW_DEPRECATED_FORM", - }; - syn::Ident::new(string, span) - } -} - -#[derive(Default)] -pub struct Deprecations(Vec<(Deprecation, Span)>); - -impl Deprecations { - pub fn new() -> Self { - Deprecations(Vec::new()) - } - - pub fn push(&mut self, deprecation: Deprecation, span: Span) { - self.0.push((deprecation, span)) - } -} - -impl ToTokens for Deprecations { - fn to_tokens(&self, tokens: &mut TokenStream) { - for (deprecation, span) in &self.0 { - let ident = deprecation.ident(*span); - quote_spanned!( - *span => - #[allow(clippy::let_unit_value)] - { - let _ = _pyo3::impl_::deprecations::#ident; - } - ) - .to_tokens(tokens) - } - } -} diff --git a/pyo3-macros-backend/src/derive_attributes.rs b/pyo3-macros-backend/src/derive_attributes.rs new file mode 100644 index 00000000000..6ec78e17eb0 --- /dev/null +++ b/pyo3-macros-backend/src/derive_attributes.rs @@ -0,0 +1,217 @@ +use crate::attributes::{ + self, get_pyo3_options, CrateAttribute, DefaultAttribute, FromPyWithAttribute, + IntoPyWithAttribute, RenameAllAttribute, +}; +use proc_macro2::Span; +use syn::parse::{Parse, ParseStream}; +use syn::spanned::Spanned; +use syn::{parenthesized, Attribute, LitStr, Result, Token}; + +/// Attributes for deriving `FromPyObject`/`IntoPyObject` scoped on containers. +pub enum ContainerAttribute { + /// Treat the Container as a Wrapper, operate directly on its field + Transparent(attributes::kw::transparent), + /// Force every field to be extracted from item of source Python object. + ItemAll(attributes::kw::from_item_all), + /// Change the name of an enum variant in the generated error message. + ErrorAnnotation(LitStr), + /// Change the path for the pyo3 crate + Crate(CrateAttribute), + /// Converts the field idents according to the [RenamingRule](attributes::RenamingRule) before extraction + RenameAll(RenameAllAttribute), +} + +impl Parse for ContainerAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let lookahead = input.lookahead1(); + if lookahead.peek(attributes::kw::transparent) { + let kw: attributes::kw::transparent = input.parse()?; + Ok(ContainerAttribute::Transparent(kw)) + } else if lookahead.peek(attributes::kw::from_item_all) { + let kw: attributes::kw::from_item_all = input.parse()?; + Ok(ContainerAttribute::ItemAll(kw)) + } else if lookahead.peek(attributes::kw::annotation) { + let _: attributes::kw::annotation = input.parse()?; + let _: Token![=] = input.parse()?; + input.parse().map(ContainerAttribute::ErrorAnnotation) + } else if lookahead.peek(Token![crate]) { + input.parse().map(ContainerAttribute::Crate) + } else if lookahead.peek(attributes::kw::rename_all) { + input.parse().map(ContainerAttribute::RenameAll) + } else { + Err(lookahead.error()) + } + } +} + +#[derive(Default, Clone)] +pub struct ContainerAttributes { + /// Treat the Container as a Wrapper, operate directly on its field + pub transparent: Option, + /// Force every field to be extracted from item of source Python object. + pub from_item_all: Option, + /// Change the name of an enum variant in the generated error message. + pub annotation: Option, + /// Change the path for the pyo3 crate + pub krate: Option, + /// Converts the field idents according to the [RenamingRule](attributes::RenamingRule) before extraction + pub rename_all: Option, +} + +impl ContainerAttributes { + pub fn from_attrs(attrs: &[Attribute]) -> Result { + let mut options = ContainerAttributes::default(); + + for attr in attrs { + if let Some(pyo3_attrs) = get_pyo3_options(attr)? { + pyo3_attrs + .into_iter() + .try_for_each(|opt| options.set_option(opt))?; + } + } + Ok(options) + } + + fn set_option(&mut self, option: ContainerAttribute) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + { + ensure_spanned!( + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once") + ); + self.$key = Some($key); + } + }; + } + + match option { + ContainerAttribute::Transparent(transparent) => set_option!(transparent), + ContainerAttribute::ItemAll(from_item_all) => set_option!(from_item_all), + ContainerAttribute::ErrorAnnotation(annotation) => set_option!(annotation), + ContainerAttribute::Crate(krate) => set_option!(krate), + ContainerAttribute::RenameAll(rename_all) => set_option!(rename_all), + } + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub enum FieldGetter { + GetItem(attributes::kw::item, Option), + GetAttr(attributes::kw::attribute, Option), +} + +impl FieldGetter { + pub fn span(&self) -> Span { + match self { + FieldGetter::GetItem(item, _) => item.span, + FieldGetter::GetAttr(attribute, _) => attribute.span, + } + } +} + +pub enum FieldAttribute { + Getter(FieldGetter), + FromPyWith(FromPyWithAttribute), + IntoPyWith(IntoPyWithAttribute), + Default(DefaultAttribute), +} + +impl Parse for FieldAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let lookahead = input.lookahead1(); + if lookahead.peek(attributes::kw::attribute) { + let attr_kw: attributes::kw::attribute = input.parse()?; + if input.peek(syn::token::Paren) { + let content; + let _ = parenthesized!(content in input); + let attr_name: LitStr = content.parse()?; + if !content.is_empty() { + return Err(content.error( + "expected at most one argument: `attribute` or `attribute(\"name\")`", + )); + } + ensure_spanned!( + !attr_name.value().is_empty(), + attr_name.span() => "attribute name cannot be empty" + ); + Ok(Self::Getter(FieldGetter::GetAttr(attr_kw, Some(attr_name)))) + } else { + Ok(Self::Getter(FieldGetter::GetAttr(attr_kw, None))) + } + } else if lookahead.peek(attributes::kw::item) { + let item_kw: attributes::kw::item = input.parse()?; + if input.peek(syn::token::Paren) { + let content; + let _ = parenthesized!(content in input); + let key = content.parse()?; + if !content.is_empty() { + return Err( + content.error("expected at most one argument: `item` or `item(key)`") + ); + } + Ok(Self::Getter(FieldGetter::GetItem(item_kw, Some(key)))) + } else { + Ok(Self::Getter(FieldGetter::GetItem(item_kw, None))) + } + } else if lookahead.peek(attributes::kw::from_py_with) { + input.parse().map(Self::FromPyWith) + } else if lookahead.peek(attributes::kw::into_py_with) { + input.parse().map(FieldAttribute::IntoPyWith) + } else if lookahead.peek(Token![default]) { + input.parse().map(Self::Default) + } else { + Err(lookahead.error()) + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct FieldAttributes { + pub getter: Option, + pub from_py_with: Option, + pub into_py_with: Option, + pub default: Option, +} + +impl FieldAttributes { + /// Extract the field attributes. + pub fn from_attrs(attrs: &[Attribute]) -> Result { + let mut options = FieldAttributes::default(); + + for attr in attrs { + if let Some(pyo3_attrs) = get_pyo3_options(attr)? { + pyo3_attrs + .into_iter() + .try_for_each(|opt| options.set_option(opt))?; + } + } + Ok(options) + } + + fn set_option(&mut self, option: FieldAttribute) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + set_option!($key, concat!("`", stringify!($key), "` may only be specified once")) + }; + ($key:ident, $msg: expr) => {{ + ensure_spanned!( + self.$key.is_none(), + $key.span() => $msg + ); + self.$key = Some($key); + }} + } + + match option { + FieldAttribute::Getter(getter) => { + set_option!(getter, "only one of `attribute` or `item` can be provided") + } + FieldAttribute::FromPyWith(from_py_with) => set_option!(from_py_with), + FieldAttribute::IntoPyWith(into_py_with) => set_option!(into_py_with), + FieldAttribute::Default(default) => set_option!(default), + } + Ok(()) + } +} diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 5e193bf4a24..841fcd2039f 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -1,16 +1,13 @@ -use crate::{ - attributes::{self, get_pyo3_options, CrateAttribute, FromPyWithAttribute}, - utils::get_pyo3_crate, -}; +use crate::attributes::{DefaultAttribute, FromPyWithAttribute, RenamingRule}; +use crate::derive_attributes::{ContainerAttributes, FieldAttributes, FieldGetter}; +#[cfg(feature = "experimental-inspect")] +use crate::introspection::elide_lifetimes; +use crate::utils::{self, Ctx}; use proc_macro2::TokenStream; -use quote::{format_ident, quote}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ - parenthesized, - parse::{Parse, ParseStream}, - parse_quote, - punctuated::Punctuated, - spanned::Spanned, - Attribute, DataEnum, DeriveInput, Fields, Ident, LitStr, Result, Token, + ext::IdentExt, parse_quote, punctuated::Punctuated, spanned::Spanned, DataEnum, DeriveInput, + Fields, Ident, Result, Token, }; /// Describes derivation input of an enum. @@ -24,7 +21,11 @@ impl<'a> Enum<'a> { /// /// `data_enum` is the `syn` representation of the input enum, `ident` is the /// `Identifier` of the enum. - fn new(data_enum: &'a DataEnum, ident: &'a Ident) -> Result { + fn new( + data_enum: &'a DataEnum, + ident: &'a Ident, + options: ContainerAttributes, + ) -> Result { ensure_spanned!( !data_enum.variants.is_empty(), ident.span() => "cannot derive FromPyObject for empty enum" @@ -33,9 +34,21 @@ impl<'a> Enum<'a> { .variants .iter() .map(|variant| { - let attrs = ContainerOptions::from_attrs(&variant.attrs)?; + let mut variant_options = ContainerAttributes::from_attrs(&variant.attrs)?; + if let Some(rename_all) = &options.rename_all { + ensure_spanned!( + variant_options.rename_all.is_none(), + variant_options.rename_all.span() => "Useless variant `rename_all` - enum is already annotated with `rename_all" + ); + variant_options.rename_all = Some(rename_all.clone()); + + } let var_ident = &variant.ident; - Container::new(&variant.fields, parse_quote!(#ident::#var_ident), attrs) + Container::new( + &variant.fields, + parse_quote!(#ident::#var_ident), + variant_options, + ) }) .collect::>>()?; @@ -46,14 +59,16 @@ impl<'a> Enum<'a> { } /// Build derivation body for enums. - fn build(&self) -> TokenStream { + fn build(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let mut var_extracts = Vec::new(); let mut variant_names = Vec::new(); let mut error_names = Vec::new(); + for var in &self.variants { - let struct_derive = var.build(); + let struct_derive = var.build(ctx); let ext = quote!({ - let maybe_ret = || -> _pyo3::PyResult { + let maybe_ret = || -> #pyo3_path::PyResult { #struct_derive }(); @@ -73,7 +88,7 @@ impl<'a> Enum<'a> { #(#var_extracts),* ]; ::std::result::Result::Err( - _pyo3::impl_::frompyobject::failed_to_extract_enum( + #pyo3_path::impl_::frompyobject::failed_to_extract_enum( obj.py(), #ty_name, &[#(#variant_names),*], @@ -83,16 +98,28 @@ impl<'a> Enum<'a> { ) ) } + + #[cfg(feature = "experimental-inspect")] + fn input_type(&self, ctx: &Ctx) -> TokenStream { + let pyo3_crate_path = &ctx.pyo3_path; + let variants = self.variants.iter().map(|var| var.input_type(ctx)); + quote! { + #pyo3_crate_path::inspect::TypeHint::union(&[#(#variants),*]) + } + } } struct NamedStructField<'a> { ident: &'a syn::Ident, getter: Option, from_py_with: Option, + default: Option, + ty: &'a syn::Type, } struct TupleStructField { from_py_with: Option, + ty: syn::Type, } /// Container Style @@ -106,7 +133,8 @@ enum ContainerType<'a> { /// Newtype struct container, e.g. `#[transparent] struct Foo { a: String }` /// /// The field specified by the identifier is extracted directly from the object. - StructNewtype(&'a syn::Ident, Option), + #[cfg_attr(not(feature = "experimental-inspect"), allow(unused))] + StructNewtype(&'a syn::Ident, Option, &'a syn::Type), /// Tuple struct, e.g. `struct Foo(String)`. /// /// Variant contains a list of conversion methods for each of the fields that are directly @@ -115,7 +143,8 @@ enum ContainerType<'a> { /// Tuple newtype, e.g. `#[transparent] struct Foo(String)` /// /// The wrapped field is directly extracted from the object. - TupleNewtype(Option), + #[cfg_attr(not(feature = "experimental-inspect"), allow(unused))] + TupleNewtype(Option, Box), } /// Data container @@ -125,26 +154,36 @@ struct Container<'a> { path: syn::Path, ty: ContainerType<'a>, err_name: String, + rename_rule: Option, } impl<'a> Container<'a> { /// Construct a container based on fields, identifier and attributes. /// /// Fails if the variant has no fields or incompatible attributes. - fn new(fields: &'a Fields, path: syn::Path, options: ContainerOptions) -> Result { + fn new(fields: &'a Fields, path: syn::Path, options: ContainerAttributes) -> Result { let style = match fields { Fields::Unnamed(unnamed) if !unnamed.unnamed.is_empty() => { + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is useless on tuple structs and variants." + ); let mut tuple_fields = unnamed .unnamed .iter() .map(|field| { - let attrs = FieldPyO3Attributes::from_attrs(&field.attrs)?; + let attrs = FieldAttributes::from_attrs(&field.attrs)?; ensure_spanned!( attrs.getter.is_none(), field.span() => "`getter` is not permitted on tuple struct elements." ); + ensure_spanned!( + attrs.default.is_none(), + field.span() => "`default` is not permitted on tuple struct elements." + ); Ok(TupleStructField { from_py_with: attrs.from_py_with, + ty: field.ty.clone(), }) }) .collect::>>()?; @@ -153,8 +192,8 @@ impl<'a> Container<'a> { // Always treat a 1-length tuple struct as "transparent", even without the // explicit annotation. let field = tuple_fields.pop().unwrap(); - ContainerType::TupleNewtype(field.from_py_with) - } else if options.transparent { + ContainerType::TupleNewtype(field.from_py_with, Box::new(field.ty)) + } else if options.transparent.is_some() { bail_spanned!( fields.span() => "transparent structs and variants can only have 1 field" ); @@ -171,17 +210,17 @@ impl<'a> Container<'a> { .ident .as_ref() .expect("Named fields should have identifiers"); - let mut attrs = FieldPyO3Attributes::from_attrs(&field.attrs)?; + let mut attrs = FieldAttributes::from_attrs(&field.attrs)?; if let Some(ref from_item_all) = options.from_item_all { - if let Some(replaced) = attrs.getter.replace(FieldGetter::GetItem(None)) + if let Some(replaced) = attrs.getter.replace(FieldGetter::GetItem(parse_quote!(item), None)) { match replaced { - FieldGetter::GetItem(Some(item_name)) => { - attrs.getter = Some(FieldGetter::GetItem(Some(item_name))); + FieldGetter::GetItem(item, Some(item_name)) => { + attrs.getter = Some(FieldGetter::GetItem(item, Some(item_name))); } - FieldGetter::GetItem(None) => bail_spanned!(from_item_all.span() => "Useless `item` - the struct is already annotated with `from_item_all`"), - FieldGetter::GetAttr(_) => bail_spanned!( + FieldGetter::GetItem(_, None) => bail_spanned!(from_item_all.span() => "Useless `item` - the struct is already annotated with `from_item_all`"), + FieldGetter::GetAttr(_, _) => bail_spanned!( from_item_all.span() => "The struct is already annotated with `from_item_all`, `attribute` is not allowed" ), } @@ -192,20 +231,30 @@ impl<'a> Container<'a> { ident, getter: attrs.getter, from_py_with: attrs.from_py_with, + default: attrs.default, + ty: &field.ty, }) }) .collect::>>()?; - if options.transparent { + if struct_fields.iter().all(|field| field.default.is_some()) { + bail_spanned!( + fields.span() => "cannot derive FromPyObject for structs and variants with only default values" + ) + } else if options.transparent.is_some() { ensure_spanned!( struct_fields.len() == 1, fields.span() => "transparent structs and variants can only have 1 field" ); + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is not permitted on `transparent` structs and variants" + ); let field = struct_fields.pop().unwrap(); ensure_spanned!( field.getter.is_none(), field.ident.span() => "`transparent` structs may not have a `getter` for the inner field" ); - ContainerType::StructNewtype(field.ident, field.from_py_with) + ContainerType::StructNewtype(field.ident, field.from_py_with, field.ty) } else { ContainerType::Struct(struct_fields) } @@ -223,6 +272,7 @@ impl<'a> Container<'a> { path, ty: style, err_name, + rename_rule: options.rename_all.map(|v| v.value.rule), }; Ok(v) } @@ -239,16 +289,16 @@ impl<'a> Container<'a> { } /// Build derivation body for a struct. - fn build(&self) -> TokenStream { + fn build(&self, ctx: &Ctx) -> TokenStream { match &self.ty { - ContainerType::StructNewtype(ident, from_py_with) => { - self.build_newtype_struct(Some(ident), from_py_with) + ContainerType::StructNewtype(ident, from_py_with, _) => { + self.build_newtype_struct(Some(ident), from_py_with, ctx) } - ContainerType::TupleNewtype(from_py_with) => { - self.build_newtype_struct(None, from_py_with) + ContainerType::TupleNewtype(from_py_with, _) => { + self.build_newtype_struct(None, from_py_with, ctx) } - ContainerType::Tuple(tups) => self.build_tuple_struct(tups), - ContainerType::Struct(tups) => self.build_struct(tups), + ContainerType::Tuple(tups) => self.build_tuple_struct(tups, ctx), + ContainerType::Struct(tups) => self.build_struct(tups, ctx), } } @@ -256,291 +306,194 @@ impl<'a> Container<'a> { &self, field_ident: Option<&Ident>, from_py_with: &Option, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let self_ty = &self.path; let struct_name = self.name(); if let Some(ident) = field_ident { let field_name = ident.to_string(); - match from_py_with { - None => quote! { + if let Some(FromPyWithAttribute { + kw, + value: expr_path, + }) = from_py_with + { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } + }; + quote! { Ok(#self_ty { - #ident: _pyo3::impl_::frompyobject::extract_struct_field(obj, #struct_name, #field_name)? + #ident: #pyo3_path::impl_::frompyobject::extract_struct_field_with(#extractor, obj, #struct_name, #field_name)? }) - }, - Some(FromPyWithAttribute { - value: expr_path, .. - }) => quote! { + } + } else { + quote! { Ok(#self_ty { - #ident: _pyo3::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, #field_name)? + #ident: #pyo3_path::impl_::frompyobject::extract_struct_field(obj, #struct_name, #field_name)? }) - }, + } + } + } else if let Some(FromPyWithAttribute { + kw, + value: expr_path, + }) = from_py_with + { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } + }; + quote! { + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#extractor, obj, #struct_name, 0).map(#self_ty) } } else { - match from_py_with { - None => quote!( - _pyo3::impl_::frompyobject::extract_tuple_struct_field(obj, #struct_name, 0).map(#self_ty) - ), - Some(FromPyWithAttribute { - value: expr_path, .. - }) => quote! ( - _pyo3::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, 0).map(#self_ty) - ), + quote! { + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field(obj, #struct_name, 0).map(#self_ty) } } } - fn build_tuple_struct(&self, struct_fields: &[TupleStructField]) -> TokenStream { + fn build_tuple_struct(&self, struct_fields: &[TupleStructField], ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let self_ty = &self.path; let struct_name = &self.name(); let field_idents: Vec<_> = (0..struct_fields.len()) .map(|i| format_ident!("arg{}", i)) .collect(); let fields = struct_fields.iter().zip(&field_idents).enumerate().map(|(index, (field, ident))| { - match &field.from_py_with { - None => quote!( - _pyo3::impl_::frompyobject::extract_tuple_struct_field(&#ident, #struct_name, #index)? - ), - Some(FromPyWithAttribute { - value: expr_path, .. - }) => quote! ( - _pyo3::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, &#ident, #struct_name, #index)? - ), - } + if let Some(FromPyWithAttribute { + kw, + value: expr_path, .. + }) = &field.from_py_with { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } + }; + quote! { + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#extractor, &#ident, #struct_name, #index)? + } + } else { + quote!{ + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field(&#ident, #struct_name, #index)? + }} }); + quote!( - match obj.extract() { + match #pyo3_path::types::PyAnyMethods::extract(obj) { ::std::result::Result::Ok((#(#field_idents),*)) => ::std::result::Result::Ok(#self_ty(#(#fields),*)), ::std::result::Result::Err(err) => ::std::result::Result::Err(err), } ) } - fn build_struct(&self, struct_fields: &[NamedStructField<'_>]) -> TokenStream { + fn build_struct(&self, struct_fields: &[NamedStructField<'_>], ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let self_ty = &self.path; - let struct_name = &self.name(); - let mut fields: Punctuated = Punctuated::new(); + let struct_name = self.name(); + let mut fields: Punctuated = Punctuated::new(); for field in struct_fields { - let ident = &field.ident; - let field_name = ident.to_string(); - let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { - FieldGetter::GetAttr(Some(name)) => { - quote!(getattr(_pyo3::intern!(obj.py(), #name))) + let ident = field.ident; + let field_name = ident.unraw().to_string(); + let getter = match field + .getter + .as_ref() + .unwrap_or(&FieldGetter::GetAttr(parse_quote!(attribute), None)) + { + FieldGetter::GetAttr(_, Some(name)) => { + quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) } - FieldGetter::GetAttr(None) => { - quote!(getattr(_pyo3::intern!(obj.py(), #field_name))) + FieldGetter::GetAttr(_, None) => { + let name = self + .rename_rule + .map(|rule| utils::apply_renaming_rule(rule, &field_name)); + let name = name.as_deref().unwrap_or(&field_name); + quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) } - FieldGetter::GetItem(Some(syn::Lit::Str(key))) => { - quote!(get_item(_pyo3::intern!(obj.py(), #key))) + FieldGetter::GetItem(_, Some(syn::Lit::Str(key))) => { + quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #pyo3_path::intern!(obj.py(), #key))) } - FieldGetter::GetItem(Some(key)) => quote!(get_item(#key)), - FieldGetter::GetItem(None) => { - quote!(get_item(_pyo3::intern!(obj.py(), #field_name))) + FieldGetter::GetItem(_, Some(key)) => { + quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #key)) } - }; - let extractor = match &field.from_py_with { - None => { - quote!(_pyo3::impl_::frompyobject::extract_struct_field(&obj.#getter?, #struct_name, #field_name)?) - } - Some(FromPyWithAttribute { - value: expr_path, .. - }) => { - quote! (_pyo3::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, &obj.#getter?, #struct_name, #field_name)?) + FieldGetter::GetItem(_, None) => { + let name = self + .rename_rule + .map(|rule| utils::apply_renaming_rule(rule, &field_name)); + let name = name.as_deref().unwrap_or(&field_name); + quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #pyo3_path::intern!(obj.py(), #name))) } }; + let extractor = if let Some(FromPyWithAttribute { + kw, + value: expr_path, + }) = &field.from_py_with + { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } + }; + quote! (#pyo3_path::impl_::frompyobject::extract_struct_field_with(#extractor, &#getter?, #struct_name, #field_name)?) + } else { + quote!(#pyo3_path::impl_::frompyobject::extract_struct_field(&value, #struct_name, #field_name)?) + }; + let extracted = if let Some(default) = &field.default { + let default_expr = if let Some(default_expr) = &default.value { + default_expr.to_token_stream() + } else { + quote!(::std::default::Default::default()) + }; + quote!(if let ::std::result::Result::Ok(value) = #getter { + #extractor + } else { + #default_expr + }) + } else { + quote!({ + let value = #getter?; + #extractor + }) + }; - fields.push(quote!(#ident: #extractor)); + fields.push(quote!(#ident: #extracted)); } - quote!(::std::result::Result::Ok(#self_ty{#fields})) - } -} - -#[derive(Default)] -struct ContainerOptions { - /// Treat the Container as a Wrapper, directly extract its fields from the input object. - transparent: bool, - /// Force every field to be extracted from item of source Python object. - from_item_all: Option, - /// Change the name of an enum variant in the generated error message. - annotation: Option, - /// Change the path for the pyo3 crate - krate: Option, -} - -/// Attributes for deriving FromPyObject scoped on containers. -enum ContainerPyO3Attribute { - /// Treat the Container as a Wrapper, directly extract its fields from the input object. - Transparent(attributes::kw::transparent), - /// Force every field to be extracted from item of source Python object. - ItemAll(attributes::kw::from_item_all), - /// Change the name of an enum variant in the generated error message. - ErrorAnnotation(LitStr), - /// Change the path for the pyo3 crate - Crate(CrateAttribute), -} -impl Parse for ContainerPyO3Attribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::transparent) { - let kw: attributes::kw::transparent = input.parse()?; - Ok(ContainerPyO3Attribute::Transparent(kw)) - } else if lookahead.peek(attributes::kw::from_item_all) { - let kw: attributes::kw::from_item_all = input.parse()?; - Ok(ContainerPyO3Attribute::ItemAll(kw)) - } else if lookahead.peek(attributes::kw::annotation) { - let _: attributes::kw::annotation = input.parse()?; - let _: Token![=] = input.parse()?; - input.parse().map(ContainerPyO3Attribute::ErrorAnnotation) - } else if lookahead.peek(Token![crate]) { - input.parse().map(ContainerPyO3Attribute::Crate) - } else { - Err(lookahead.error()) - } + quote!(::std::result::Result::Ok(#self_ty{#fields})) } -} -impl ContainerOptions { - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut options = ContainerOptions::default(); - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - for pyo3_attr in pyo3_attrs { - match pyo3_attr { - ContainerPyO3Attribute::Transparent(kw) => { - ensure_spanned!( - !options.transparent, - kw.span() => "`transparent` may only be provided once" - ); - options.transparent = true; - } - ContainerPyO3Attribute::ItemAll(kw) => { - ensure_spanned!( - options.from_item_all.is_none(), - kw.span() => "`from_item_all` may only be provided once" - ); - options.from_item_all = Some(kw); - } - ContainerPyO3Attribute::ErrorAnnotation(lit_str) => { - ensure_spanned!( - options.annotation.is_none(), - lit_str.span() => "`annotation` may only be provided once" - ); - options.annotation = Some(lit_str); - } - ContainerPyO3Attribute::Crate(path) => { - ensure_spanned!( - options.krate.is_none(), - path.span() => "`crate` may only be provided once" - ); - options.krate = Some(path); - } - } - } + #[cfg(feature = "experimental-inspect")] + fn input_type(&self, ctx: &Ctx) -> TokenStream { + let pyo3_crate_path = &ctx.pyo3_path; + match &self.ty { + ContainerType::StructNewtype(_, from_py_with, ty) => { + Self::field_input_type(from_py_with, ty, ctx) } - } - Ok(options) - } -} - -/// Attributes for deriving FromPyObject scoped on fields. -#[derive(Clone, Debug)] -struct FieldPyO3Attributes { - getter: Option, - from_py_with: Option, -} - -#[derive(Clone, Debug)] -enum FieldGetter { - GetItem(Option), - GetAttr(Option), -} - -enum FieldPyO3Attribute { - Getter(FieldGetter), - FromPyWith(FromPyWithAttribute), -} - -impl Parse for FieldPyO3Attribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::attribute) { - let _: attributes::kw::attribute = input.parse()?; - if input.peek(syn::token::Paren) { - let content; - let _ = parenthesized!(content in input); - let attr_name: LitStr = content.parse()?; - if !content.is_empty() { - return Err(content.error( - "expected at most one argument: `attribute` or `attribute(\"name\")`", - )); - } - ensure_spanned!( - !attr_name.value().is_empty(), - attr_name.span() => "attribute name cannot be empty" - ); - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetAttr(Some( - attr_name, - )))) - } else { - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetAttr(None))) + ContainerType::TupleNewtype(from_py_with, ty) => { + Self::field_input_type(from_py_with, ty, ctx) } - } else if lookahead.peek(attributes::kw::item) { - let _: attributes::kw::item = input.parse()?; - if input.peek(syn::token::Paren) { - let content; - let _ = parenthesized!(content in input); - let key = content.parse()?; - if !content.is_empty() { - return Err( - content.error("expected at most one argument: `item` or `item(key)`") - ); - } - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetItem(Some(key)))) - } else { - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetItem(None))) + ContainerType::Tuple(tups) => { + let elements = tups.iter().map(|TupleStructField { from_py_with, ty }| { + Self::field_input_type(from_py_with, ty, ctx) + }); + quote! { #pyo3_crate_path::inspect::TypeHint::subscript(&#pyo3_crate_path::inspect::TypeHint::builtin("tuple"), &[#(#elements),*]) } + } + ContainerType::Struct(_) => { + // TODO: implement using a Protocol? + quote! { #pyo3_crate_path::inspect::TypeHint::module_attr("_typeshed", "Incomplete") } } - } else if lookahead.peek(attributes::kw::from_py_with) { - input.parse().map(FieldPyO3Attribute::FromPyWith) - } else { - Err(lookahead.error()) } } -} -impl FieldPyO3Attributes { - /// Extract the field attributes. - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut getter = None; - let mut from_py_with = None; - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - for pyo3_attr in pyo3_attrs { - match pyo3_attr { - FieldPyO3Attribute::Getter(field_getter) => { - ensure_spanned!( - getter.is_none(), - attr.span() => "only one of `attribute` or `item` can be provided" - ); - getter = Some(field_getter); - } - FieldPyO3Attribute::FromPyWith(from_py_with_attr) => { - ensure_spanned!( - from_py_with.is_none(), - attr.span() => "`from_py_with` may only be provided once" - ); - from_py_with = Some(from_py_with_attr); - } - } - } - } + #[cfg(feature = "experimental-inspect")] + fn field_input_type( + from_py_with: &Option, + ty: &syn::Type, + ctx: &Ctx, + ) -> TokenStream { + let pyo3_crate_path = &ctx.pyo3_path; + if from_py_with.is_some() { + // We don't know what from_py_with is doing + quote! { #pyo3_crate_path::inspect::TypeHint::module_attr("_typeshed", "Incomplete") } + } else { + let mut ty = ty.clone(); + elide_lifetimes(&mut ty); + quote! { <#ty as #pyo3_crate_path::FromPyObject<'_, '_>>::INPUT_TYPE } } - - Ok(FieldPyO3Attributes { - getter, - from_py_with, - }) } } @@ -563,57 +516,91 @@ fn verify_and_get_lifetime(generics: &syn::Generics) -> Result Foo(T)` /// adds `T: FromPyObject` on the derived implementation. pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { + let options = ContainerAttributes::from_attrs(&tokens.attrs)?; + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = &ctx; + + let (_, ty_generics, _) = tokens.generics.split_for_impl(); let mut trait_generics = tokens.generics.clone(); - let generics = &tokens.generics; - let lt_param = if let Some(lt) = verify_and_get_lifetime(generics)? { + let lt_param = if let Some(lt) = verify_and_get_lifetime(&trait_generics)? { lt.clone() } else { trait_generics.params.push(parse_quote!('py)); parse_quote!('py) }; - let mut where_clause: syn::WhereClause = parse_quote!(where); - for param in generics.type_params() { + let (impl_generics, _, where_clause) = trait_generics.split_for_impl(); + + let mut where_clause = where_clause.cloned().unwrap_or_else(|| parse_quote!(where)); + for param in trait_generics.type_params() { let gen_ident = ¶m.ident; where_clause .predicates - .push(parse_quote!(#gen_ident: FromPyObject<#lt_param>)) + .push(parse_quote!(#gen_ident: #pyo3_path::conversion::FromPyObjectOwned<#lt_param>)) } - let options = ContainerOptions::from_attrs(&tokens.attrs)?; - let krate = get_pyo3_crate(&options.krate); + let derives = match &tokens.data { syn::Data::Enum(en) => { - if options.transparent || options.annotation.is_some() { + if options.transparent.is_some() || options.annotation.is_some() { bail_spanned!(tokens.span() => "`transparent` or `annotation` is not supported \ at top level for enums"); } - let en = Enum::new(en, &tokens.ident)?; - en.build() + let en = Enum::new(en, &tokens.ident, options.clone())?; + en.build(ctx) } syn::Data::Struct(st) => { if let Some(lit_str) = &options.annotation { bail_spanned!(lit_str.span() => "`annotation` is unsupported for structs"); } let ident = &tokens.ident; - let st = Container::new(&st.fields, parse_quote!(#ident), options)?; - st.build() + let st = Container::new(&st.fields, parse_quote!(#ident), options.clone())?; + st.build(ctx) } syn::Data::Union(_) => bail_spanned!( tokens.span() => "#[derive(FromPyObject)] is not supported for unions" ), }; - let ident = &tokens.ident; - Ok(quote!( - const _: () = { - use #krate as _pyo3; - use _pyo3::prelude::PyAnyMethods; - - #[automatically_derived] - impl #trait_generics _pyo3::FromPyObject<#lt_param> for #ident #generics #where_clause { - fn extract_bound(obj: &_pyo3::Bound<#lt_param, _pyo3::PyAny>) -> _pyo3::PyResult { - #derives + #[cfg(feature = "experimental-inspect")] + let input_type = { + let pyo3_crate_path = &ctx.pyo3_path; + let input_type = if tokens + .generics + .params + .iter() + .all(|p| matches!(p, syn::GenericParam::Lifetime(_))) + { + match &tokens.data { + syn::Data::Enum(en) => Enum::new(en, &tokens.ident, options)?.input_type(ctx), + syn::Data::Struct(st) => { + let ident = &tokens.ident; + Container::new(&st.fields, parse_quote!(#ident), options.clone())? + .input_type(ctx) + } + syn::Data::Union(_) => { + // Not supported at this point + quote! { #pyo3_crate_path::inspect::TypeHint::module_attr("_typeshed", "Incomplete") } } } + } else { + // We don't know how to deal with generic parameters + // Blocked by https://github.com/rust-lang/rust/issues/76560 + quote! { #pyo3_crate_path::inspect::TypeHint::module_attr("_typeshed", "Incomplete") } }; + quote! { const INPUT_TYPE: #pyo3_crate_path::inspect::TypeHint = #input_type; } + }; + #[cfg(not(feature = "experimental-inspect"))] + let input_type = quote! {}; + + let ident = &tokens.ident; + Ok(quote!( + #[automatically_derived] + impl #impl_generics #pyo3_path::FromPyObject<'_, #lt_param> for #ident #ty_generics #where_clause { + type Error = #pyo3_path::PyErr; + fn extract(obj: #pyo3_path::Borrowed<'_, #lt_param, #pyo3_path::PyAny>) -> ::std::result::Result { + let obj: &#pyo3_path::Bound<'_, _> = &*obj; + #derives + } + #input_type + } )) } diff --git a/pyo3-macros-backend/src/intopyobject.rs b/pyo3-macros-backend/src/intopyobject.rs new file mode 100644 index 00000000000..75dfe054fa7 --- /dev/null +++ b/pyo3-macros-backend/src/intopyobject.rs @@ -0,0 +1,631 @@ +use crate::attributes::{IntoPyWithAttribute, RenamingRule}; +use crate::derive_attributes::{ContainerAttributes, FieldAttributes}; +#[cfg(feature = "experimental-inspect")] +use crate::introspection::elide_lifetimes; +use crate::utils::{self, Ctx}; +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; +use syn::ext::IdentExt; +use syn::spanned::Spanned as _; +use syn::{parse_quote, DataEnum, DeriveInput, Fields, Ident, Index, Result}; + +struct ItemOption(Option); + +enum IntoPyObjectTypes { + Transparent(syn::Type), + Opaque { + target: TokenStream, + output: TokenStream, + error: TokenStream, + }, +} + +struct IntoPyObjectImpl { + types: IntoPyObjectTypes, + body: TokenStream, +} + +struct NamedStructField<'a> { + ident: &'a syn::Ident, + field: &'a syn::Field, + item: Option, + into_py_with: Option, +} + +struct TupleStructField<'a> { + field: &'a syn::Field, + into_py_with: Option, +} + +/// Container Style +/// +/// Covers Structs, Tuplestructs and corresponding Newtypes. +enum ContainerType<'a> { + /// Struct Container, e.g. `struct Foo { a: String }` + /// + /// Variant contains the list of field identifiers and the corresponding extraction call. + Struct(Vec>), + /// Newtype struct container, e.g. `#[transparent] struct Foo { a: String }` + /// + /// The field specified by the identifier is extracted directly from the object. + StructNewtype(&'a syn::Field), + /// Tuple struct, e.g. `struct Foo(String)`. + /// + /// Variant contains a list of conversion methods for each of the fields that are directly + /// extracted from the tuple. + Tuple(Vec>), + /// Tuple newtype, e.g. `#[transparent] struct Foo(String)` + /// + /// The wrapped field is directly extracted from the object. + TupleNewtype(&'a syn::Field), +} + +/// Data container +/// +/// Either describes a struct or an enum variant. +struct Container<'a, const REF: bool> { + path: syn::Path, + receiver: Option, + ty: ContainerType<'a>, + rename_rule: Option, +} + +/// Construct a container based on fields, identifier and attributes. +impl<'a, const REF: bool> Container<'a, REF> { + /// + /// Fails if the variant has no fields or incompatible attributes. + fn new( + receiver: Option, + fields: &'a Fields, + path: syn::Path, + options: ContainerAttributes, + ) -> Result { + let style = match fields { + Fields::Unnamed(unnamed) if !unnamed.unnamed.is_empty() => { + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is useless on tuple structs and variants." + ); + let mut tuple_fields = unnamed + .unnamed + .iter() + .map(|field| { + let attrs = FieldAttributes::from_attrs(&field.attrs)?; + ensure_spanned!( + attrs.getter.is_none(), + attrs.getter.unwrap().span() => "`item` and `attribute` are not permitted on tuple struct elements." + ); + Ok(TupleStructField { + field, + into_py_with: attrs.into_py_with, + }) + }) + .collect::>>()?; + if tuple_fields.len() == 1 { + // Always treat a 1-length tuple struct as "transparent", even without the + // explicit annotation. + let TupleStructField { + field, + into_py_with, + } = tuple_fields.pop().unwrap(); + ensure_spanned!( + into_py_with.is_none(), + into_py_with.span() => "`into_py_with` is not permitted on `transparent` structs" + ); + ContainerType::TupleNewtype(field) + } else if options.transparent.is_some() { + bail_spanned!( + fields.span() => "transparent structs and variants can only have 1 field" + ); + } else { + ContainerType::Tuple(tuple_fields) + } + } + Fields::Named(named) if !named.named.is_empty() => { + if options.transparent.is_some() { + ensure_spanned!( + named.named.iter().count() == 1, + fields.span() => "transparent structs and variants can only have 1 field" + ); + + let field = named.named.iter().next().unwrap(); + let attrs = FieldAttributes::from_attrs(&field.attrs)?; + ensure_spanned!( + attrs.getter.is_none(), + attrs.getter.unwrap().span() => "`transparent` structs may not have `item` nor `attribute` for the inner field" + ); + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is not permitted on `transparent` structs and variants" + ); + ensure_spanned!( + attrs.into_py_with.is_none(), + attrs.into_py_with.span() => "`into_py_with` is not permitted on `transparent` structs or variants" + ); + ContainerType::StructNewtype(field) + } else { + let struct_fields = named + .named + .iter() + .map(|field| { + let ident = field + .ident + .as_ref() + .expect("Named fields should have identifiers"); + + let attrs = FieldAttributes::from_attrs(&field.attrs)?; + + Ok(NamedStructField { + ident, + field, + item: attrs.getter.and_then(|getter| match getter { + crate::derive_attributes::FieldGetter::GetItem(_, lit) => { + Some(ItemOption(lit)) + } + crate::derive_attributes::FieldGetter::GetAttr(_, _) => None, + }), + into_py_with: attrs.into_py_with, + }) + }) + .collect::>>()?; + ContainerType::Struct(struct_fields) + } + } + _ => bail_spanned!( + fields.span() => "cannot derive `IntoPyObject` for empty structs" + ), + }; + + let v = Container { + path, + receiver, + ty: style, + rename_rule: options.rename_all.map(|v| v.value.rule), + }; + Ok(v) + } + + fn match_pattern(&self) -> TokenStream { + let path = &self.path; + let pattern = match &self.ty { + ContainerType::Struct(fields) => fields + .iter() + .enumerate() + .map(|(i, f)| { + let ident = f.ident; + let new_ident = format_ident!("arg{i}"); + quote! {#ident: #new_ident,} + }) + .collect::(), + ContainerType::StructNewtype(field) => { + let ident = field.ident.as_ref().unwrap(); + quote!(#ident: arg0) + } + ContainerType::Tuple(fields) => { + let i = (0..fields.len()).map(Index::from); + let idents = (0..fields.len()).map(|i| format_ident!("arg{i}")); + quote! { #(#i: #idents,)* } + } + ContainerType::TupleNewtype(_) => quote!(0: arg0), + }; + + quote! { #path{ #pattern } } + } + + /// Build derivation body for a struct. + fn build(&self, ctx: &Ctx) -> IntoPyObjectImpl { + match &self.ty { + ContainerType::StructNewtype(field) | ContainerType::TupleNewtype(field) => { + self.build_newtype_struct(field, ctx) + } + ContainerType::Tuple(fields) => self.build_tuple_struct(fields, ctx), + ContainerType::Struct(fields) => self.build_struct(fields, ctx), + } + } + + fn build_newtype_struct(&self, field: &syn::Field, ctx: &Ctx) -> IntoPyObjectImpl { + let Ctx { pyo3_path, .. } = ctx; + let ty = &field.ty; + + let unpack = self + .receiver + .as_ref() + .map(|i| { + let pattern = self.match_pattern(); + quote! { let #pattern = #i;} + }) + .unwrap_or_default(); + + IntoPyObjectImpl { + types: IntoPyObjectTypes::Transparent(ty.clone()), + body: quote_spanned! { ty.span() => + #unpack + #pyo3_path::conversion::IntoPyObject::into_pyobject(arg0, py) + }, + } + } + + fn build_struct(&self, fields: &[NamedStructField<'_>], ctx: &Ctx) -> IntoPyObjectImpl { + let Ctx { pyo3_path, .. } = ctx; + + let unpack = self + .receiver + .as_ref() + .map(|i| { + let pattern = self.match_pattern(); + quote! { let #pattern = #i;} + }) + .unwrap_or_default(); + + let setter = fields + .iter() + .enumerate() + .map(|(i, f)| { + let key = f + .item + .as_ref() + .and_then(|item| item.0.as_ref()) + .map(|item| item.into_token_stream()) + .unwrap_or_else(|| { + let name = f.ident.unraw().to_string(); + self.rename_rule.map(|rule| utils::apply_renaming_rule(rule, &name)).unwrap_or(name).into_token_stream() + }); + let value = Ident::new(&format!("arg{i}"), f.field.ty.span()); + + if let Some(expr_path) = f.into_py_with.as_ref().map(|i|&i.value) { + let cow = if REF { + quote!(::std::borrow::Cow::Borrowed(#value)) + } else { + quote!(::std::borrow::Cow::Owned(#value)) + }; + quote! { + let into_py_with: fn(::std::borrow::Cow<'_, _>, #pyo3_path::Python<'py>) -> #pyo3_path::PyResult<#pyo3_path::Bound<'py, #pyo3_path::PyAny>> = #expr_path; + #pyo3_path::types::PyDictMethods::set_item(&dict, #key, into_py_with(#cow, py)?)?; + } + } else { + quote! { + #pyo3_path::types::PyDictMethods::set_item(&dict, #key, #value)?; + } + } + }) + .collect::(); + + IntoPyObjectImpl { + types: IntoPyObjectTypes::Opaque { + target: quote!(#pyo3_path::types::PyDict), + output: quote!(#pyo3_path::Bound<'py, Self::Target>), + error: quote!(#pyo3_path::PyErr), + }, + body: quote! { + #unpack + let dict = #pyo3_path::types::PyDict::new(py); + #setter + ::std::result::Result::Ok::<_, Self::Error>(dict) + }, + } + } + + fn build_tuple_struct(&self, fields: &[TupleStructField<'_>], ctx: &Ctx) -> IntoPyObjectImpl { + let Ctx { pyo3_path, .. } = ctx; + + let unpack = self + .receiver + .as_ref() + .map(|i| { + let pattern = self.match_pattern(); + quote! { let #pattern = #i;} + }) + .unwrap_or_default(); + + let setter = fields + .iter() + .enumerate() + .map(|(i, f)| { + let ty = &f.field.ty; + let value = Ident::new(&format!("arg{i}"), f.field.ty.span()); + + if let Some(expr_path) = f.into_py_with.as_ref().map(|i|&i.value) { + let cow = if REF { + quote!(::std::borrow::Cow::Borrowed(#value)) + } else { + quote!(::std::borrow::Cow::Owned(#value)) + }; + quote_spanned! { ty.span() => + { + let into_py_with: fn(::std::borrow::Cow<'_, _>, #pyo3_path::Python<'py>) -> #pyo3_path::PyResult<#pyo3_path::Bound<'py, #pyo3_path::PyAny>> = #expr_path; + into_py_with(#cow, py)? + }, + } + } else { + quote_spanned! { ty.span() => + #pyo3_path::conversion::IntoPyObject::into_pyobject(#value, py) + .map(#pyo3_path::BoundObject::into_any) + .map(#pyo3_path::BoundObject::into_bound)?, + } + } + }) + .collect::(); + + IntoPyObjectImpl { + types: IntoPyObjectTypes::Opaque { + target: quote!(#pyo3_path::types::PyTuple), + output: quote!(#pyo3_path::Bound<'py, Self::Target>), + error: quote!(#pyo3_path::PyErr), + }, + body: quote! { + #unpack + #pyo3_path::types::PyTuple::new(py, [#setter]) + }, + } + } + + #[cfg(feature = "experimental-inspect")] + fn output_type(&self, ctx: &Ctx) -> TokenStream { + let pyo3_crate_path = &ctx.pyo3_path; + match &self.ty { + ContainerType::StructNewtype(field) | ContainerType::TupleNewtype(field) => { + Self::field_output_type(&None, &field.ty, ctx) + } + ContainerType::Tuple(tups) => { + let elements = tups.iter().map( + |TupleStructField { + into_py_with, + field, + }| { + Self::field_output_type(into_py_with, &field.ty, ctx) + }, + ); + quote! { #pyo3_crate_path::inspect::TypeHint::subscript(&#pyo3_crate_path::inspect::TypeHint::builtin("tuple"), &[#(#elements),*]) } + } + ContainerType::Struct(_) => { + // TODO: implement using a Protocol? + quote! { #pyo3_crate_path::inspect::TypeHint::module_attr("_typeshed", "Incomplete") } + } + } + } + + #[cfg(feature = "experimental-inspect")] + fn field_output_type( + into_py_with: &Option, + ty: &syn::Type, + ctx: &Ctx, + ) -> TokenStream { + let pyo3_crate_path = &ctx.pyo3_path; + if into_py_with.is_some() { + // We don't know what into_py_with is doing + quote! { #pyo3_crate_path::inspect::TypeHint::module_attr("_typeshed", "Incomplete") } + } else { + let mut ty = ty.clone(); + elide_lifetimes(&mut ty); + quote! { <#ty as #pyo3_crate_path::IntoPyObject<'_>>::OUTPUT_TYPE } + } + } +} + +/// Describes derivation input of an enum. +struct Enum<'a, const REF: bool> { + variants: Vec>, +} + +impl<'a, const REF: bool> Enum<'a, REF> { + /// Construct a new enum representation. + /// + /// `data_enum` is the `syn` representation of the input enum, `ident` is the + /// `Identifier` of the enum. + fn new(data_enum: &'a DataEnum, ident: &'a Ident) -> Result { + ensure_spanned!( + !data_enum.variants.is_empty(), + ident.span() => "cannot derive `IntoPyObject` for empty enum" + ); + let variants = data_enum + .variants + .iter() + .map(|variant| { + let attrs = ContainerAttributes::from_attrs(&variant.attrs)?; + let var_ident = &variant.ident; + + ensure_spanned!( + !variant.fields.is_empty(), + variant.ident.span() => "cannot derive `IntoPyObject` for empty variants" + ); + + Container::new( + None, + &variant.fields, + parse_quote!(#ident::#var_ident), + attrs, + ) + }) + .collect::>>()?; + + Ok(Enum { variants }) + } + + /// Build derivation body for enums. + fn build(&self, ctx: &Ctx) -> IntoPyObjectImpl { + let Ctx { pyo3_path, .. } = ctx; + + let variants = self + .variants + .iter() + .map(|v| { + let IntoPyObjectImpl { body, .. } = v.build(ctx); + let pattern = v.match_pattern(); + quote! { + #pattern => { + {#body} + .map(#pyo3_path::BoundObject::into_any) + .map(#pyo3_path::BoundObject::into_bound) + .map_err(::std::convert::Into::<#pyo3_path::PyErr>::into) + } + } + }) + .collect::(); + + IntoPyObjectImpl { + types: IntoPyObjectTypes::Opaque { + target: quote!(#pyo3_path::types::PyAny), + output: quote!(#pyo3_path::Bound<'py, >::Target>), + error: quote!(#pyo3_path::PyErr), + }, + body: quote! { + match self { + #variants + } + }, + } + } + + #[cfg(feature = "experimental-inspect")] + fn output_type(&self, ctx: &Ctx) -> TokenStream { + let pyo3_crate_path = &ctx.pyo3_path; + let variants = self.variants.iter().map(|var| var.output_type(ctx)); + quote! { + #pyo3_crate_path::inspect::TypeHint::union(&[#(#variants),*]) + } + } +} + +// if there is a `'py` lifetime, we treat it as the `Python<'py>` lifetime +fn verify_and_get_lifetime(generics: &syn::Generics) -> Option<&syn::LifetimeParam> { + let mut lifetimes = generics.lifetimes(); + lifetimes.find(|l| l.lifetime.ident == "py") +} + +pub fn build_derive_into_pyobject(tokens: &DeriveInput) -> Result { + let options = ContainerAttributes::from_attrs(&tokens.attrs)?; + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = &ctx; + + let (_, ty_generics, _) = tokens.generics.split_for_impl(); + let mut trait_generics = tokens.generics.clone(); + if REF { + trait_generics.params.push(parse_quote!('_a)); + } + let lt_param = if let Some(lt) = verify_and_get_lifetime(&trait_generics) { + lt.clone() + } else { + trait_generics.params.push(parse_quote!('py)); + parse_quote!('py) + }; + let (impl_generics, _, where_clause) = trait_generics.split_for_impl(); + + let mut where_clause = where_clause.cloned().unwrap_or_else(|| parse_quote!(where)); + for param in trait_generics.type_params() { + let gen_ident = ¶m.ident; + where_clause.predicates.push(if REF { + parse_quote!(&'_a #gen_ident: #pyo3_path::conversion::IntoPyObject<'py>) + } else { + parse_quote!(#gen_ident: #pyo3_path::conversion::IntoPyObject<'py>) + }) + } + + let IntoPyObjectImpl { types, body } = match &tokens.data { + syn::Data::Enum(en) => { + if options.transparent.is_some() { + bail_spanned!(tokens.span() => "`transparent` is not supported at top level for enums"); + } + if let Some(rename_all) = options.rename_all { + bail_spanned!(rename_all.span() => "`rename_all` is not supported at top level for enums"); + } + let en = Enum::::new(en, &tokens.ident)?; + en.build(ctx) + } + syn::Data::Struct(st) => { + let ident = &tokens.ident; + let st = Container::::new( + Some(Ident::new("self", Span::call_site())), + &st.fields, + parse_quote!(#ident), + options.clone(), + )?; + st.build(ctx) + } + syn::Data::Union(_) => bail_spanned!( + tokens.span() => "#[derive(`IntoPyObject`)] is not supported for unions" + ), + }; + + let (target, output, error) = match types { + IntoPyObjectTypes::Transparent(ty) => { + if REF { + ( + quote! { <&'_a #ty as #pyo3_path::IntoPyObject<'py>>::Target }, + quote! { <&'_a #ty as #pyo3_path::IntoPyObject<'py>>::Output }, + quote! { <&'_a #ty as #pyo3_path::IntoPyObject<'py>>::Error }, + ) + } else { + ( + quote! { <#ty as #pyo3_path::IntoPyObject<'py>>::Target }, + quote! { <#ty as #pyo3_path::IntoPyObject<'py>>::Output }, + quote! { <#ty as #pyo3_path::IntoPyObject<'py>>::Error }, + ) + } + } + IntoPyObjectTypes::Opaque { + target, + output, + error, + } => (target, output, error), + }; + + let ident = &tokens.ident; + let ident = if REF { + quote! { &'_a #ident} + } else { + quote! { #ident } + }; + + #[cfg(feature = "experimental-inspect")] + let output_type = { + let pyo3_crate_path = &ctx.pyo3_path; + let output_type = if tokens + .generics + .params + .iter() + .all(|p| matches!(p, syn::GenericParam::Lifetime(_))) + { + match &tokens.data { + syn::Data::Enum(en) => Enum::::new(en, &tokens.ident)?.output_type(ctx), + syn::Data::Struct(st) => { + let ident = &tokens.ident; + Container::::new( + Some(Ident::new("self", Span::call_site())), + &st.fields, + parse_quote!(#ident), + options, + )? + .output_type(ctx) + } + syn::Data::Union(_) => { + // Not supported at this point + quote! { #pyo3_crate_path::inspect::TypeHint::module_attr("_typeshed", "Incomplete") } + } + } + } else { + // We don't know how to deal with generic parameters + // Blocked by https://github.com/rust-lang/rust/issues/76560 + quote! { #pyo3_crate_path::inspect::TypeHint::module_attr("_typeshed", "Incomplete") } + }; + quote! { const OUTPUT_TYPE: #pyo3_path::inspect::TypeHint = #output_type; } + }; + #[cfg(not(feature = "experimental-inspect"))] + let output_type = quote! {}; + + Ok(quote!( + #[automatically_derived] + impl #impl_generics #pyo3_path::conversion::IntoPyObject<#lt_param> for #ident #ty_generics #where_clause { + type Target = #target; + type Output = #output; + type Error = #error; + #output_type + + fn into_pyobject(self, py: #pyo3_path::Python<#lt_param>) -> ::std::result::Result< + >::Output, + >::Error, + > { + #body + } + } + )) +} diff --git a/pyo3-macros-backend/src/introspection.rs b/pyo3-macros-backend/src/introspection.rs new file mode 100644 index 00000000000..1aa53e516e0 --- /dev/null +++ b/pyo3-macros-backend/src/introspection.rs @@ -0,0 +1,666 @@ +//! Generates introspection data i.e. JSON strings in the .pyo3i0 section. +//! +//! There is a JSON per PyO3 proc macro (pyclass, pymodule, pyfunction...). +//! +//! These JSON blobs can refer to each others via the _PYO3_INTROSPECTION_ID constants +//! providing unique ids for each element. +//! +//! The JSON blobs format must be synchronized with the `pyo3_introspection::introspection.rs::Chunk` +//! type that is used to parse them. + +use crate::method::{FnArg, RegularArg}; +use crate::pyfunction::FunctionSignature; +use crate::utils::PyO3CratePath; +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote, ToTokens}; +use std::borrow::Cow; +use std::collections::hash_map::DefaultHasher; +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; +use std::mem::take; +use std::sync::atomic::{AtomicUsize, Ordering}; +use syn::visit_mut::{visit_type_mut, VisitMut}; +use syn::{Attribute, Ident, Lifetime, ReturnType, Type, TypePath}; + +static GLOBAL_COUNTER_FOR_UNIQUE_NAMES: AtomicUsize = AtomicUsize::new(0); + +#[allow(clippy::too_many_arguments)] +pub fn module_introspection_code<'a>( + pyo3_crate_path: &PyO3CratePath, + name: &str, + members: impl IntoIterator, + members_cfg_attrs: impl IntoIterator>, + incomplete: bool, +) -> TokenStream { + IntrospectionNode::Map( + [ + ("type", IntrospectionNode::String("module".into())), + ("id", IntrospectionNode::IntrospectionId(None)), + ("name", IntrospectionNode::String(name.into())), + ( + "members", + IntrospectionNode::List( + members + .into_iter() + .zip(members_cfg_attrs) + .map(|(member, attributes)| AttributedIntrospectionNode { + node: IntrospectionNode::IntrospectionId(Some(ident_to_type(member))), + attributes, + }) + .collect(), + ), + ), + ("incomplete", IntrospectionNode::Bool(incomplete)), + ] + .into(), + ) + .emit(pyo3_crate_path) +} + +pub fn class_introspection_code( + pyo3_crate_path: &PyO3CratePath, + ident: &Ident, + name: &str, +) -> TokenStream { + IntrospectionNode::Map( + [ + ("type", IntrospectionNode::String("class".into())), + ( + "id", + IntrospectionNode::IntrospectionId(Some(ident_to_type(ident))), + ), + ("name", IntrospectionNode::String(name.into())), + ] + .into(), + ) + .emit(pyo3_crate_path) +} + +#[allow(clippy::too_many_arguments)] +pub fn function_introspection_code( + pyo3_crate_path: &PyO3CratePath, + ident: Option<&Ident>, + name: &str, + signature: &FunctionSignature<'_>, + first_argument: Option<&'static str>, + returns: ReturnType, + decorators: impl IntoIterator, + parent: Option<&Type>, +) -> TokenStream { + let mut desc = HashMap::from([ + ("type", IntrospectionNode::String("function".into())), + ("name", IntrospectionNode::String(name.into())), + ( + "arguments", + arguments_introspection_data(signature, first_argument, parent), + ), + ( + "returns", + if let Some((_, returns)) = signature + .attribute + .as_ref() + .and_then(|attribute| attribute.value.returns.as_ref()) + { + IntrospectionNode::String(returns.to_python().into()) + } else { + match returns { + ReturnType::Default => IntrospectionNode::ConstantType { + name: "None", + module: None, + }, + ReturnType::Type(_, ty) => match *ty { + Type::Tuple(t) if t.elems.is_empty() => { + // () is converted to None in return types + IntrospectionNode::ConstantType { + name: "None", + module: None, + } + } + mut ty => { + if let Some(class_type) = parent { + replace_self(&mut ty, class_type); + } + elide_lifetimes(&mut ty); + IntrospectionNode::OutputType { + rust_type: ty, + is_final: false, + } + } + }, + } + }, + ), + ]); + if let Some(ident) = ident { + desc.insert( + "id", + IntrospectionNode::IntrospectionId(Some(ident_to_type(ident))), + ); + } + let decorators = decorators + .into_iter() + .map(|d| IntrospectionNode::String(d.into()).into()) + .collect::>(); + if !decorators.is_empty() { + desc.insert("decorators", IntrospectionNode::List(decorators)); + } + if let Some(parent) = parent { + desc.insert( + "parent", + IntrospectionNode::IntrospectionId(Some(Cow::Borrowed(parent))), + ); + } + IntrospectionNode::Map(desc).emit(pyo3_crate_path) +} + +pub fn attribute_introspection_code( + pyo3_crate_path: &PyO3CratePath, + parent: Option<&Type>, + name: String, + value: String, + mut rust_type: Type, + is_final: bool, +) -> TokenStream { + let mut desc = HashMap::from([ + ("type", IntrospectionNode::String("attribute".into())), + ("name", IntrospectionNode::String(name.into())), + ( + "parent", + IntrospectionNode::IntrospectionId(parent.map(Cow::Borrowed)), + ), + ]); + if value == "..." { + // We need to set a type, but not need to set the value to ..., all attributes have a value + if let Some(parent) = parent { + replace_self(&mut rust_type, parent); + } + elide_lifetimes(&mut rust_type); + desc.insert( + "annotation", + IntrospectionNode::OutputType { + rust_type, + is_final, + }, + ); + } else { + desc.insert( + "annotation", + if is_final { + // Type checkers can infer the type from the value because it's typing.Literal[value] + // So, following stubs best practices, we only write typing.Final and not + // typing.Final[typing.literal[value]] + IntrospectionNode::ConstantType { + name: "Final", + module: Some("typing"), + } + } else { + IntrospectionNode::OutputType { + rust_type, + is_final, + } + }, + ); + desc.insert("value", IntrospectionNode::String(value.into())); + } + IntrospectionNode::Map(desc).emit(pyo3_crate_path) +} + +fn arguments_introspection_data<'a>( + signature: &'a FunctionSignature<'a>, + first_argument: Option<&'a str>, + class_type: Option<&Type>, +) -> IntrospectionNode<'a> { + let mut argument_desc = signature.arguments.iter().filter(|arg| { + matches!( + arg, + FnArg::Regular(_) | FnArg::VarArgs(_) | FnArg::KwArgs(_) + ) + }); + + let mut posonlyargs = Vec::new(); + let mut args = Vec::new(); + let mut vararg = None; + let mut kwonlyargs = Vec::new(); + let mut kwarg = None; + + if let Some(first_argument) = first_argument { + posonlyargs.push( + IntrospectionNode::Map( + [("name", IntrospectionNode::String(first_argument.into()))].into(), + ) + .into(), + ); + } + + for (i, param) in signature + .python_signature + .positional_parameters + .iter() + .enumerate() + { + let arg_desc = if let Some(FnArg::Regular(arg_desc)) = argument_desc.next() { + arg_desc + } else { + panic!("Less arguments than in python signature"); + }; + let arg = argument_introspection_data(param, arg_desc, class_type); + if i < signature.python_signature.positional_only_parameters { + posonlyargs.push(arg); + } else { + args.push(arg) + } + } + + if let Some(param) = &signature.python_signature.varargs { + let Some(FnArg::VarArgs(arg_desc)) = argument_desc.next() else { + panic!("Fewer arguments than in python signature"); + }; + let mut params = HashMap::from([("name", IntrospectionNode::String(param.into()))]); + if let Some(annotation) = &arg_desc.annotation { + params.insert("annotation", IntrospectionNode::String(annotation.into())); + } + vararg = Some(IntrospectionNode::Map(params)); + } + + for (param, _) in &signature.python_signature.keyword_only_parameters { + let Some(FnArg::Regular(arg_desc)) = argument_desc.next() else { + panic!("Less arguments than in python signature"); + }; + kwonlyargs.push(argument_introspection_data(param, arg_desc, class_type)); + } + + if let Some(param) = &signature.python_signature.kwargs { + let Some(FnArg::KwArgs(arg_desc)) = argument_desc.next() else { + panic!("Less arguments than in python signature"); + }; + let mut params = HashMap::from([("name", IntrospectionNode::String(param.into()))]); + if let Some(annotation) = &arg_desc.annotation { + params.insert("annotation", IntrospectionNode::String(annotation.into())); + } + kwarg = Some(IntrospectionNode::Map(params)); + } + + let mut map = HashMap::new(); + if !posonlyargs.is_empty() { + map.insert("posonlyargs", IntrospectionNode::List(posonlyargs)); + } + if !args.is_empty() { + map.insert("args", IntrospectionNode::List(args)); + } + if let Some(vararg) = vararg { + map.insert("vararg", vararg); + } + if !kwonlyargs.is_empty() { + map.insert("kwonlyargs", IntrospectionNode::List(kwonlyargs)); + } + if let Some(kwarg) = kwarg { + map.insert("kwarg", kwarg); + } + IntrospectionNode::Map(map) +} + +fn argument_introspection_data<'a>( + name: &'a str, + desc: &'a RegularArg<'_>, + class_type: Option<&Type>, +) -> AttributedIntrospectionNode<'a> { + let mut params: HashMap<_, _> = [("name", IntrospectionNode::String(name.into()))].into(); + if desc.default_value.is_some() { + params.insert( + "default", + IntrospectionNode::String(desc.default_value().into()), + ); + } + + if let Some(annotation) = &desc.annotation { + params.insert("annotation", IntrospectionNode::String(annotation.into())); + } else if desc.from_py_with.is_none() { + // If from_py_with is set we don't know anything on the input type + if let Some(ty) = desc.option_wrapped_type { + // Special case to properly generate a `T | None` annotation + let mut ty = ty.clone(); + if let Some(class_type) = class_type { + replace_self(&mut ty, class_type); + } + elide_lifetimes(&mut ty); + params.insert( + "annotation", + IntrospectionNode::InputType { + rust_type: ty, + nullable: true, + }, + ); + } else { + let mut ty = desc.ty.clone(); + if let Some(class_type) = class_type { + replace_self(&mut ty, class_type); + } + elide_lifetimes(&mut ty); + params.insert( + "annotation", + IntrospectionNode::InputType { + rust_type: ty, + nullable: false, + }, + ); + } + } + IntrospectionNode::Map(params).into() +} + +enum IntrospectionNode<'a> { + String(Cow<'a, str>), + Bool(bool), + IntrospectionId(Option>), + InputType { + rust_type: Type, + nullable: bool, + }, + OutputType { + rust_type: Type, + is_final: bool, + }, + ConstantType { + name: &'static str, + module: Option<&'static str>, + }, + Map(HashMap<&'static str, IntrospectionNode<'a>>), + List(Vec>), +} + +impl IntrospectionNode<'_> { + fn emit(self, pyo3_crate_path: &PyO3CratePath) -> TokenStream { + let mut content = ConcatenationBuilder::default(); + self.add_to_serialization(&mut content, pyo3_crate_path); + content.into_static( + pyo3_crate_path, + format_ident!("PYO3_INTROSPECTION_1_{}", unique_element_id()), + ) + } + + fn add_to_serialization( + self, + content: &mut ConcatenationBuilder, + pyo3_crate_path: &PyO3CratePath, + ) { + match self { + Self::String(string) => { + content.push_str_to_escape(&string); + } + Self::Bool(value) => content.push_str(if value { "true" } else { "false" }), + Self::IntrospectionId(ident) => { + content.push_str("\""); + content.push_tokens(if let Some(ident) = ident { + quote! { #ident::_PYO3_INTROSPECTION_ID.as_bytes() } + } else { + quote! { _PYO3_INTROSPECTION_ID.as_bytes() } + }); + content.push_str("\""); + } + Self::InputType { + rust_type, + nullable, + } => { + let mut annotation = quote! { + <#rust_type as #pyo3_crate_path::impl_::extract_argument::PyFunctionArgument< + { + #[allow(unused_imports)] + use #pyo3_crate_path::impl_::pyclass::Probe as _; + #pyo3_crate_path::impl_::pyclass::IsFromPyObject::<#rust_type>::VALUE + } + >>::INPUT_TYPE + }; + if nullable { + annotation = quote! { #pyo3_crate_path::inspect::TypeHint::union(&[#annotation, #pyo3_crate_path::inspect::TypeHint::builtin("None")]) }; + } + content.push_tokens(serialize_type_hint(annotation, pyo3_crate_path)); + } + Self::OutputType { + rust_type, + is_final, + } => { + let mut annotation = quote! { <#rust_type as #pyo3_crate_path::impl_::introspection::PyReturnType>::OUTPUT_TYPE }; + if is_final { + annotation = quote! { #pyo3_crate_path::inspect::TypeHint::subscript(&#pyo3_crate_path::inspect::TypeHint::module_attr("typing", "Final"), &[#annotation]) }; + } + content.push_tokens(serialize_type_hint(annotation, pyo3_crate_path)); + } + Self::ConstantType { name, module } => { + let annotation = if let Some(module) = module { + quote! { #pyo3_crate_path::inspect::TypeHint::module_attr(#module, #name) } + } else { + quote! { #pyo3_crate_path::inspect::TypeHint::builtin(#name) } + }; + content.push_tokens(serialize_type_hint(annotation, pyo3_crate_path)); + } + Self::Map(map) => { + content.push_str("{"); + for (i, (key, value)) in map.into_iter().enumerate() { + if i > 0 { + content.push_str(","); + } + content.push_str_to_escape(key); + content.push_str(":"); + value.add_to_serialization(content, pyo3_crate_path); + } + content.push_str("}"); + } + Self::List(list) => { + content.push_str("["); + for (i, AttributedIntrospectionNode { node, attributes }) in + list.into_iter().enumerate() + { + if attributes.is_empty() { + if i > 0 { + content.push_str(","); + } + node.add_to_serialization(content, pyo3_crate_path); + } else { + // We serialize the element to easily gate it behind the attributes + let mut nested_builder = ConcatenationBuilder::default(); + if i > 0 { + nested_builder.push_str(","); + } + node.add_to_serialization(&mut nested_builder, pyo3_crate_path); + let nested_content = nested_builder.into_token_stream(pyo3_crate_path); + content.push_tokens(quote! { #(#attributes)* #nested_content }); + } + } + content.push_str("]"); + } + } + } +} + +fn serialize_type_hint(hint: TokenStream, pyo3_crate_path: &PyO3CratePath) -> TokenStream { + quote! {{ + const TYPE_HINT: #pyo3_crate_path::inspect::TypeHint = #hint; + const TYPE_HINT_LEN: usize = #pyo3_crate_path::inspect::serialized_len_for_introspection(&TYPE_HINT); + const TYPE_HINT_SER: [u8; TYPE_HINT_LEN] = { + let mut result: [u8; TYPE_HINT_LEN] = [0; TYPE_HINT_LEN]; + #pyo3_crate_path::inspect::serialize_for_introspection(&TYPE_HINT, &mut result); + result + }; + &TYPE_HINT_SER + }} +} + +struct AttributedIntrospectionNode<'a> { + node: IntrospectionNode<'a>, + attributes: &'a [Attribute], +} + +impl<'a> From> for AttributedIntrospectionNode<'a> { + fn from(node: IntrospectionNode<'a>) -> Self { + Self { + node, + attributes: &[], + } + } +} + +#[derive(Default)] +pub struct ConcatenationBuilder { + elements: Vec, + current_string: String, +} + +impl ConcatenationBuilder { + pub fn push_tokens(&mut self, token_stream: TokenStream) { + if !self.current_string.is_empty() { + self.elements.push(ConcatenationBuilderElement::String(take( + &mut self.current_string, + ))); + } + self.elements + .push(ConcatenationBuilderElement::TokenStream(token_stream)); + } + + pub fn push_str(&mut self, value: &str) { + self.current_string.push_str(value); + } + + fn push_str_to_escape(&mut self, value: &str) { + self.current_string.push('"'); + for c in value.chars() { + match c { + '\\' => self.current_string.push_str("\\\\"), + '"' => self.current_string.push_str("\\\""), + c => { + if c < char::from(32) { + panic!("ASCII chars below 32 are not allowed") + } else { + self.current_string.push(c); + } + } + } + } + self.current_string.push('"'); + } + + pub fn into_token_stream(self, pyo3_crate_path: &PyO3CratePath) -> TokenStream { + let mut elements = self.elements; + if !self.current_string.is_empty() { + elements.push(ConcatenationBuilderElement::String(self.current_string)); + } + + if let [ConcatenationBuilderElement::String(string)] = elements.as_slice() { + // We avoid the const_concat! macro if there is only a single string + return quote! { #string.as_bytes() }; + } + + quote! { + { + const PIECES: &[&[u8]] = &[#(#elements , )*]; + &#pyo3_crate_path::impl_::concat::combine_to_array::<{ + #pyo3_crate_path::impl_::concat::combined_len(PIECES) + }>(PIECES) + } + } + } + + fn into_static(self, pyo3_crate_path: &PyO3CratePath, ident: Ident) -> TokenStream { + let mut elements = self.elements; + if !self.current_string.is_empty() { + elements.push(ConcatenationBuilderElement::String(self.current_string)); + } + + // #[no_mangle] is required to make sure some linkers like Linux ones do not mangle the section name too. + quote! { + const _: () = { + const PIECES: &[&[u8]] = &[#(#elements , )*]; + const PIECES_LEN: usize = #pyo3_crate_path::impl_::concat::combined_len(PIECES); + #[used] + #[no_mangle] + static #ident: #pyo3_crate_path::impl_::introspection::SerializedIntrospectionFragment = #pyo3_crate_path::impl_::introspection::SerializedIntrospectionFragment { + length: PIECES_LEN as u32, + fragment: #pyo3_crate_path::impl_::concat::combine_to_array::(PIECES) + }; + }; + } + } +} + +enum ConcatenationBuilderElement { + String(String), + TokenStream(TokenStream), +} + +impl ToTokens for ConcatenationBuilderElement { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::String(s) => quote! { #s.as_bytes() }.to_tokens(tokens), + Self::TokenStream(ts) => ts.to_tokens(tokens), + } + } +} + +/// Generates a new unique identifier for linking introspection objects together +pub fn introspection_id_const() -> TokenStream { + let id = unique_element_id().to_string(); + quote! { + #[doc(hidden)] + pub const _PYO3_INTROSPECTION_ID: &'static str = #id; + } +} + +pub fn unique_element_id() -> u64 { + let mut hasher = DefaultHasher::new(); + format!("{:?}", Span::call_site()).hash(&mut hasher); // Distinguishes between call sites + GLOBAL_COUNTER_FOR_UNIQUE_NAMES + .fetch_add(1, Ordering::Relaxed) + .hash(&mut hasher); // If there are multiple elements in the same call site + hasher.finish() +} + +fn ident_to_type(ident: &Ident) -> Cow<'static, Type> { + Cow::Owned( + TypePath { + path: ident.clone().into(), + qself: None, + } + .into(), + ) +} + +/// Replaces all explicit lifetimes in `self` with elided (`'_`) lifetimes +/// +/// This is useful if `Self` is used in `const` context, where explicit +/// lifetimes are not allowed (yet). +pub fn elide_lifetimes(ty: &mut Type) { + struct ElideLifetimesVisitor; + + impl VisitMut for ElideLifetimesVisitor { + fn visit_lifetime_mut(&mut self, l: &mut syn::Lifetime) { + *l = Lifetime::new("'_", l.span()); + } + } + + ElideLifetimesVisitor.visit_type_mut(ty); +} + +// Replace Self in types with the given type +fn replace_self(ty: &mut Type, self_target: &Type) { + struct SelfReplacementVisitor<'a> { + self_target: &'a Type, + } + + impl VisitMut for SelfReplacementVisitor<'_> { + fn visit_type_mut(&mut self, ty: &mut Type) { + if let Type::Path(type_path) = ty { + if type_path.qself.is_none() + && type_path.path.segments.len() == 1 + && type_path.path.segments[0].ident == "Self" + && type_path.path.segments[0].arguments.is_empty() + { + // It is Self + *ty = self.self_target.clone(); + return; + } + } + visit_type_mut(self, ty); + } + } + + SelfReplacementVisitor { self_target }.visit_type_mut(ty); +} diff --git a/pyo3-macros-backend/src/konst.rs b/pyo3-macros-backend/src/konst.rs index 935c9d4a302..e56afd0516d 100644 --- a/pyo3-macros-backend/src/konst.rs +++ b/pyo3-macros-backend/src/konst.rs @@ -1,11 +1,9 @@ use std::borrow::Cow; +use std::ffi::CString; -use crate::{ - attributes::{self, get_pyo3_options, take_attributes, NameAttribute}, - deprecations::Deprecations, -}; -use proc_macro2::{Ident, TokenStream}; -use quote::quote; +use crate::attributes::{self, get_pyo3_options, take_attributes, NameAttribute}; +use proc_macro2::{Ident, Span}; +use syn::LitCStr; use syn::{ ext::IdentExt, parse::{Parse, ParseStream}, @@ -28,16 +26,15 @@ impl ConstSpec { } /// Null-terminated Python name - pub fn null_terminated_python_name(&self) -> TokenStream { - let name = format!("{}\0", self.python_name()); - quote!({#name}) + pub fn null_terminated_python_name(&self) -> LitCStr { + let name = self.python_name().to_string(); + LitCStr::new(&CString::new(name).unwrap(), Span::call_site()) } } pub struct ConstAttributes { pub is_class_attr: bool, pub name: Option, - pub deprecations: Deprecations, } pub enum PyO3ConstAttribute { @@ -60,7 +57,6 @@ impl ConstAttributes { let mut attributes = ConstAttributes { is_class_attr: false, name: None, - deprecations: Deprecations::new(), }; take_attributes(attrs, |attr| { diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index 745a8471c2b..5a99f499f19 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -1,7 +1,7 @@ //! This crate contains the implementation of the proc macro attributes #![warn(elided_lifetimes_in_paths, unused_lifetimes)] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![recursion_limit = "1024"] // Listed first so that macros in this module are available in the rest of the crate. @@ -9,8 +9,12 @@ mod utils; mod attributes; -mod deprecations; +mod combine_errors; +mod derive_attributes; mod frompyobject; +mod intopyobject; +#[cfg(feature = "experimental-inspect")] +mod introspection; mod konst; mod method; mod module; @@ -19,10 +23,12 @@ mod pyclass; mod pyfunction; mod pyimpl; mod pymethod; +mod pyversions; mod quotes; pub use frompyobject::build_derive_from_pyobject; -pub use module::{process_functions_in_module, pymodule_impl, PyModuleOptions}; +pub use intopyobject::build_derive_into_pyobject; +pub use module::{pymodule_function_impl, pymodule_module_impl, PyModuleOptions}; pub use pyclass::{build_py_class, build_py_enum, PyClassArgs}; pub use pyfunction::{build_py_function, PyFunctionOptions}; pub use pyimpl::{build_py_methods, PyClassMethodsType}; diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index f492a330c92..5484b8d5d0c 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -1,34 +1,172 @@ +use std::borrow::Cow; +use std::ffi::CString; use std::fmt::Display; use proc_macro2::{Span, TokenStream}; -use quote::{quote, quote_spanned, ToTokens}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; +use syn::LitCStr; use syn::{ext::IdentExt, spanned::Spanned, Ident, Result}; +use crate::pyfunction::{PyFunctionWarning, WarningFactory}; +use crate::pyversions::is_abi3_before; +use crate::utils::{expr_to_python, Ctx}; use crate::{ - attributes::{TextSignatureAttribute, TextSignatureAttributeValue}, - deprecations::{Deprecation, Deprecations}, - params::impl_arg_params, + attributes::{FromPyWithAttribute, TextSignatureAttribute, TextSignatureAttributeValue}, + params::{impl_arg_params, Holders}, pyfunction::{ FunctionSignature, PyFunctionArgPyO3Attributes, PyFunctionOptions, SignatureAttribute, }, quotes, - utils::{self, is_abi3, PythonDoc}, + utils::{self, PythonDoc}, }; #[derive(Clone, Debug)] -pub struct FnArg<'a> { +pub struct RegularArg<'a> { + pub name: Cow<'a, syn::Ident>, + pub ty: &'a syn::Type, + pub from_py_with: Option, + pub default_value: Option, + pub option_wrapped_type: Option<&'a syn::Type>, + #[cfg(feature = "experimental-inspect")] + pub annotation: Option, +} + +impl RegularArg<'_> { + pub fn default_value(&self) -> String { + if let Self { + default_value: Some(arg_default), + .. + } = self + { + expr_to_python(arg_default) + } else if let RegularArg { + option_wrapped_type: Some(..), + .. + } = self + { + // functions without a `#[pyo3(signature = (...))]` option + // will treat trailing `Option` arguments as having a default of `None` + "None".to_string() + } else { + "...".to_string() + } + } +} + +/// Pythons *args argument +#[derive(Clone, Debug)] +pub struct VarargsArg<'a> { + pub name: Cow<'a, syn::Ident>, + pub ty: &'a syn::Type, + #[cfg(feature = "experimental-inspect")] + pub annotation: Option, +} + +/// Pythons **kwarg argument +#[derive(Clone, Debug)] +pub struct KwargsArg<'a> { + pub name: Cow<'a, syn::Ident>, + pub ty: &'a syn::Type, + #[cfg(feature = "experimental-inspect")] + pub annotation: Option, +} + +#[derive(Clone, Debug)] +pub struct CancelHandleArg<'a> { + pub name: &'a syn::Ident, + pub ty: &'a syn::Type, +} + +#[derive(Clone, Debug)] +pub struct PyArg<'a> { pub name: &'a syn::Ident, pub ty: &'a syn::Type, - pub optional: Option<&'a syn::Type>, - pub default: Option, - pub py: bool, - pub attrs: PyFunctionArgPyO3Attributes, - pub is_varargs: bool, - pub is_kwargs: bool, - pub is_cancel_handle: bool, +} + +#[allow(clippy::large_enum_variant)] // See #5039 +#[derive(Clone, Debug)] +pub enum FnArg<'a> { + Regular(RegularArg<'a>), + VarArgs(VarargsArg<'a>), + KwArgs(KwargsArg<'a>), + Py(PyArg<'a>), + CancelHandle(CancelHandleArg<'a>), } impl<'a> FnArg<'a> { + pub fn name(&self) -> &syn::Ident { + match self { + FnArg::Regular(RegularArg { name, .. }) => name, + FnArg::VarArgs(VarargsArg { name, .. }) => name, + FnArg::KwArgs(KwargsArg { name, .. }) => name, + FnArg::Py(PyArg { name, .. }) => name, + FnArg::CancelHandle(CancelHandleArg { name, .. }) => name, + } + } + + pub fn ty(&self) -> &'a syn::Type { + match self { + FnArg::Regular(RegularArg { ty, .. }) => ty, + FnArg::VarArgs(VarargsArg { ty, .. }) => ty, + FnArg::KwArgs(KwargsArg { ty, .. }) => ty, + FnArg::Py(PyArg { ty, .. }) => ty, + FnArg::CancelHandle(CancelHandleArg { ty, .. }) => ty, + } + } + + #[allow(clippy::wrong_self_convention)] + pub fn from_py_with(&self) -> Option<&FromPyWithAttribute> { + if let FnArg::Regular(RegularArg { from_py_with, .. }) = self { + from_py_with.as_ref() + } else { + None + } + } + + pub fn to_varargs_mut(&mut self) -> Result<&mut Self> { + if let Self::Regular(RegularArg { + name, + ty, + option_wrapped_type: None, + #[cfg(feature = "experimental-inspect")] + annotation, + .. + }) = self + { + *self = Self::VarArgs(VarargsArg { + name: name.clone(), + ty, + #[cfg(feature = "experimental-inspect")] + annotation: annotation.clone(), + }); + Ok(self) + } else { + bail_spanned!(self.name().span() => "args cannot be optional") + } + } + + pub fn to_kwargs_mut(&mut self) -> Result<&mut Self> { + if let Self::Regular(RegularArg { + name, + ty, + option_wrapped_type: Some(..), + #[cfg(feature = "experimental-inspect")] + annotation, + .. + }) = self + { + *self = Self::KwArgs(KwargsArg { + name: name.clone(), + ty, + #[cfg(feature = "experimental-inspect")] + annotation: annotation.clone(), + }); + Ok(self) + } else { + bail_spanned!(self.name().span() => "kwargs must be Option<_>") + } + } + /// Transforms a rust fn arg parsed with syn into a method::FnArg pub fn parse(arg: &'a mut syn::FnArg) -> Result { match arg { @@ -40,28 +178,53 @@ impl<'a> FnArg<'a> { bail_spanned!(cap.ty.span() => IMPL_TRAIT_ERR); } - let arg_attrs = PyFunctionArgPyO3Attributes::from_attrs(&mut cap.attrs)?; + let PyFunctionArgPyO3Attributes { + from_py_with, + cancel_handle, + } = PyFunctionArgPyO3Attributes::from_attrs(&mut cap.attrs)?; let ident = match &*cap.pat { syn::Pat::Ident(syn::PatIdent { ident, .. }) => ident, other => return Err(handle_argument_error(other)), }; - let is_cancel_handle = arg_attrs.cancel_handle.is_some(); + if utils::is_python(&cap.ty) { + return Ok(Self::Py(PyArg { + name: ident, + ty: &cap.ty, + })); + } - Ok(FnArg { - name: ident, + if cancel_handle.is_some() { + // `PyFunctionArgPyO3Attributes::from_attrs` validates that + // only compatible attributes are specified, either + // `cancel_handle` or `from_py_with`, duplicates and any + // combination of the two are already rejected. + return Ok(Self::CancelHandle(CancelHandleArg { + name: ident, + ty: &cap.ty, + })); + } + + Ok(Self::Regular(RegularArg { + name: Cow::Borrowed(ident), ty: &cap.ty, - optional: utils::option_type_argument(&cap.ty), - default: None, - py: utils::is_python(&cap.ty), - attrs: arg_attrs, - is_varargs: false, - is_kwargs: false, - is_cancel_handle, - }) + from_py_with, + default_value: None, + option_wrapped_type: utils::option_type_argument(&cap.ty), + #[cfg(feature = "experimental-inspect")] + annotation: None, + })) } } } + + pub fn default_value(&self) -> String { + if let Self::Regular(args) = self { + args.default_value() + } else { + "...".to_string() + } + } } fn handle_argument_error(pat: &syn::Pat) -> syn::Error { @@ -77,16 +240,26 @@ fn handle_argument_error(pat: &syn::Pat) -> syn::Error { syn::Error::new(span, msg) } +/// Represents what kind of a function a pyfunction or pymethod is #[derive(Clone, Debug)] pub enum FnType { + /// Represents a pymethod annotated with `#[getter]` Getter(SelfType), + /// Represents a pymethod annotated with `#[setter]` Setter(SelfType), + /// Represents a regular pymethod Fn(SelfType), + /// Represents a pymethod annotated with `#[new]`, i.e. the `__new__` dunder. FnNew, + /// Represents a pymethod annotated with both `#[new]` and `#[classmethod]` (in either order) FnNewClass(Span), + /// Represents a pymethod annotated with `#[classmethod]`, like a `@classmethod` FnClass(Span), + /// Represents a pyfunction or a pymethod annotated with `#[staticmethod]`, like a `@staticmethod` FnStatic, + /// Represents a pyfunction annotated with `#[pyo3(pass_module)] FnModule(Span), + /// Represents a pymethod or associated constant annotated with `#[classattr]` ClassAttribute, } @@ -103,47 +276,66 @@ impl FnType { } } + pub fn signature_attribute_allowed(&self) -> bool { + match self { + FnType::Fn(_) + | FnType::FnNew + | FnType::FnStatic + | FnType::FnClass(_) + | FnType::FnNewClass(_) + | FnType::FnModule(_) => true, + // Setter, Getter and ClassAttribute all have fixed signatures (either take 0 or 1 + // arguments) so cannot have a `signature = (...)` attribute. + FnType::Getter(_) | FnType::Setter(_) | FnType::ClassAttribute => false, + } + } + pub fn self_arg( &self, cls: Option<&syn::Type>, error_mode: ExtractErrorMode, - holders: &mut Vec, - ) -> TokenStream { + holders: &mut Holders, + ctx: &Ctx, + ) -> Option { + let Ctx { pyo3_path, .. } = ctx; match self { FnType::Getter(st) | FnType::Setter(st) | FnType::Fn(st) => { let mut receiver = st.receiver( cls.expect("no class given for Fn with a \"self\" receiver"), error_mode, holders, + ctx, ); syn::Token![,](Span::call_site()).to_tokens(&mut receiver); - receiver - } - FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => { - quote!() + Some(receiver) } FnType::FnClass(span) | FnType::FnNewClass(span) => { let py = syn::Ident::new("py", Span::call_site()); let slf: Ident = syn::Ident::new("_slf", Span::call_site()); - quote_spanned! { *span => + let pyo3_path = pyo3_path.to_tokens_spanned(*span); + let ret = quote_spanned! { *span => #[allow(clippy::useless_conversion)] ::std::convert::Into::into( - _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) - .downcast_unchecked::<_pyo3::types::PyType>() - ), - } + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &*(&#slf as *const _ as *const *mut _)) + .cast_unchecked::<#pyo3_path::types::PyType>() + ) + }; + Some(quote! { unsafe { #ret }, }) } FnType::FnModule(span) => { let py = syn::Ident::new("py", Span::call_site()); let slf: Ident = syn::Ident::new("_slf", Span::call_site()); - quote_spanned! { *span => + let pyo3_path = pyo3_path.to_tokens_spanned(*span); + let ret = quote_spanned! { *span => #[allow(clippy::useless_conversion)] ::std::convert::Into::into( - _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) - .downcast_unchecked::<_pyo3::types::PyModule>() - ), - } + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &*(&#slf as *const _ as *const *mut _)) + .cast_unchecked::<#pyo3_path::types::PyModule>() + ) + }; + Some(quote! { unsafe { #ret }, }) } + FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => None, } } } @@ -151,7 +343,7 @@ impl FnType { #[derive(Clone, Debug)] pub enum SelfType { Receiver { mutable: bool, span: Span }, - TryFromPyCell(Span), + TryFromBoundRef(Span), } #[derive(Clone, Copy)] @@ -161,13 +353,14 @@ pub enum ExtractErrorMode { } impl ExtractErrorMode { - pub fn handle_error(self, extract: TokenStream) -> TokenStream { + pub fn handle_error(self, extract: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; match self { ExtractErrorMode::Raise => quote! { #extract? }, ExtractErrorMode::NotImplemented => quote! { match #extract { ::std::result::Result::Ok(value) => value, - ::std::result::Result::Err(_) => { return _pyo3::callback::convert(py, py.NotImplemented()); }, + ::std::result::Result::Err(_) => { return #pyo3_path::impl_::callback::convert(py, py.NotImplemented()); }, } }, } @@ -179,12 +372,16 @@ impl SelfType { &self, cls: &syn::Type, error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, + ctx: &Ctx, ) -> TokenStream { // Due to use of quote_spanned in this function, need to bind these idents to the // main macro callsite. let py = syn::Ident::new("py", Span::call_site()); let slf = syn::Ident::new("_slf", Span::call_site()); + let Ctx { pyo3_path, .. } = ctx; + let bound_ref = + quote! { unsafe { #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf) } }; match self { SelfType::Receiver { span, mutable } => { let method = if *mutable { @@ -192,30 +389,31 @@ impl SelfType { } else { syn::Ident::new("extract_pyclass_ref", *span) }; - let holder = syn::Ident::new(&format!("holder_{}", holders.len()), *span); - holders.push(quote_spanned! { *span => - #[allow(clippy::let_unit_value)] - let mut #holder = _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT; - }); - error_mode.handle_error(quote_spanned! { *span => - _pyo3::impl_::extract_argument::#method::<#cls>( - #py.from_borrowed_ptr::<_pyo3::PyAny>(#slf), - &mut #holder, - ) - }) + let holder = holders.push_holder(*span); + let pyo3_path = pyo3_path.to_tokens_spanned(*span); + error_mode.handle_error( + quote_spanned! { *span => + #pyo3_path::impl_::extract_argument::#method::<#cls>( + #bound_ref.0, + &mut #holder, + ) + }, + ctx, + ) } - SelfType::TryFromPyCell(span) => { + SelfType::TryFromBoundRef(span) => { + let pyo3_path = pyo3_path.to_tokens_spanned(*span); error_mode.handle_error( quote_spanned! { *span => - #py.from_borrowed_ptr::<_pyo3::PyAny>(#slf).downcast::<_pyo3::PyCell<#cls>>() - .map_err(::std::convert::Into::<_pyo3::PyErr>::into) + #bound_ref.cast::<#cls>() + .map_err(::std::convert::Into::<#pyo3_path::PyErr>::into) .and_then( - #[allow(clippy::useless_conversion)] // In case slf is PyCell - #[allow(unknown_lints, clippy::unnecessary_fallible_conversions)] // In case slf is Py (unknown_lints can be removed when MSRV is 1.75+) - |cell| ::std::convert::TryFrom::try_from(cell).map_err(::std::convert::Into::into) + #[allow(clippy::unnecessary_fallible_conversions)] // In case slf is Py + |bound| ::std::convert::TryFrom::try_from(bound).map_err(::std::convert::Into::into) ) - } + }, + ctx ) } } @@ -227,7 +425,7 @@ impl SelfType { pub enum CallingConvention { Noargs, // METH_NOARGS Varargs, // METH_VARARGS | METH_KEYWORDS - Fastcall, // METH_FASTCALL | METH_KEYWORDS (not compatible with `abi3` feature) + Fastcall, // METH_FASTCALL | METH_KEYWORDS (not compatible with `abi3` feature before 3.10) TpNew, // special convention for tp_new } @@ -239,11 +437,11 @@ impl CallingConvention { pub fn from_signature(signature: &FunctionSignature<'_>) -> Self { if signature.python_signature.has_no_args() { Self::Noargs - } else if signature.python_signature.kwargs.is_some() { - // for functions that accept **kwargs, always prefer varargs - Self::Varargs - } else if !is_abi3() { - // FIXME: available in the stable ABI since 3.10 + } else if signature.python_signature.kwargs.is_none() && !is_abi3_before(3, 10) { + // For functions that accept **kwargs, always prefer varargs for now based on + // historical performance testing. + // + // FASTCALL not compatible with `abi3` before 3.10 Self::Fastcall } else { Self::Varargs @@ -251,6 +449,7 @@ impl CallingConvention { } } +#[derive(Clone)] pub struct FnSpec<'a> { pub tp: FnType, // Rust function name @@ -259,19 +458,13 @@ pub struct FnSpec<'a> { // r# can be removed by syn::ext::IdentExt::unraw() pub python_name: syn::Ident, pub signature: FunctionSignature<'a>, - pub output: syn::Type, pub convention: CallingConvention, pub text_signature: Option, pub asyncness: Option, pub unsafety: Option, - pub deprecations: Deprecations, -} - -pub fn get_return_info(output: &syn::ReturnType) -> syn::Type { - match output { - syn::ReturnType::Default => syn::Type::Infer(syn::parse_quote! {_}), - syn::ReturnType::Type(_, ty) => *ty.clone(), - } + pub warnings: Vec, + #[cfg(feature = "experimental-inspect")] + pub output: syn::ReturnType, } pub fn parse_method_receiver(arg: &syn::FnArg) -> Result { @@ -291,7 +484,7 @@ pub fn parse_method_receiver(arg: &syn::FnArg) -> Result { if let syn::Type::ImplTrait(_) = &**ty { bail_spanned!(ty.span() => IMPL_TRAIT_ERR); } - Ok(SelfType::TryFromPyCell(ty.span())) + Ok(SelfType::TryFromBoundRef(ty.span())) } } } @@ -308,17 +501,16 @@ impl<'a> FnSpec<'a> { text_signature, name, signature, + warnings, .. } = options; let mut python_name = name.map(|name| name.value.0); - let mut deprecations = Deprecations::new(); - let fn_type = Self::parse_fn_type(sig, meth_attrs, &mut python_name, &mut deprecations)?; + let fn_type = Self::parse_fn_type(sig, meth_attrs, &mut python_name)?; ensure_signatures_on_valid_method(&fn_type, signature.as_ref(), text_signature.as_ref())?; let name = &sig.ident; - let ty = get_return_info(&sig.output); let python_name = python_name.as_ref().unwrap_or(name).unraw(); let arguments: Vec<_> = sig @@ -335,7 +527,7 @@ impl<'a> FnSpec<'a> { let signature = if let Some(signature) = signature { FunctionSignature::from_arguments_and_attribute(arguments, signature)? } else { - FunctionSignature::from_arguments(arguments)? + FunctionSignature::from_arguments(arguments) }; let convention = if matches!(fn_type, FnType::FnNew | FnType::FnNewClass(_)) { @@ -350,25 +542,27 @@ impl<'a> FnSpec<'a> { convention, python_name, signature, - output: ty, text_signature, asyncness: sig.asyncness, unsafety: sig.unsafety, - deprecations, + warnings, + #[cfg(feature = "experimental-inspect")] + output: sig.output.clone(), }) } - pub fn null_terminated_python_name(&self) -> syn::LitStr { - syn::LitStr::new(&format!("{}\0", self.python_name), self.python_name.span()) + pub fn null_terminated_python_name(&self) -> LitCStr { + let name = self.python_name.to_string(); + let name = CString::new(name).unwrap(); + LitCStr::new(&name, self.python_name.span()) } fn parse_fn_type( sig: &syn::Signature, meth_attrs: &mut Vec, python_name: &mut Option, - deprecations: &mut Deprecations, ) -> Result { - let mut method_attributes = parse_method_attributes(meth_attrs, deprecations)?; + let mut method_attributes = parse_method_attributes(meth_attrs)?; let name = &sig.ident; let parse_receiver = |msg: &'static str| { @@ -455,10 +649,10 @@ impl<'a> FnSpec<'a> { .fold(first.span(), |s, next| s.join(next.span()).unwrap_or(s)); let span = span.join(last.span()).unwrap_or(span); // List all the attributes in the error message - let mut msg = format!("`{}` may not be combined with", first); + let mut msg = format!("`{first}` may not be combined with"); let mut is_first = true; for attr in &*rest { - msg.push_str(&format!(" `{}`", attr)); + msg.push_str(&format!(" `{attr}`")); if is_first { is_first = false; } else { @@ -468,7 +662,7 @@ impl<'a> FnSpec<'a> { if !rest.is_empty() { msg.push_str(" and"); } - msg.push_str(&format!(" `{}`", last)); + msg.push_str(&format!(" `{last}`")); bail_spanned!(span => msg) } }; @@ -480,22 +674,29 @@ impl<'a> FnSpec<'a> { &self, ident: &proc_macro2::Ident, cls: Option<&syn::Type>, + ctx: &Ctx, ) -> Result { + let Ctx { + pyo3_path, + output_span, + } = ctx; let mut cancel_handle_iter = self .signature .arguments .iter() - .filter(|arg| arg.is_cancel_handle); + .filter(|arg| matches!(arg, FnArg::CancelHandle(..))); let cancel_handle = cancel_handle_iter.next(); - if let Some(arg) = cancel_handle { - ensure_spanned!(self.asyncness.is_some(), arg.name.span() => "`cancel_handle` attribute can only be used with `async fn`"); - if let Some(arg2) = cancel_handle_iter.next() { - bail_spanned!(arg2.name.span() => "`cancel_handle` may only be specified once"); + if let Some(FnArg::CancelHandle(CancelHandleArg { name, .. })) = cancel_handle { + ensure_spanned!(self.asyncness.is_some(), name.span() => "`cancel_handle` attribute can only be used with `async fn`"); + if let Some(FnArg::CancelHandle(CancelHandleArg { name, .. })) = + cancel_handle_iter.next() + { + bail_spanned!(name.span() => "`cancel_handle` may only be specified once"); } } - let rust_call = |args: Vec, holders: &mut Vec| { - let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, holders); + let rust_call = |args: Vec, holders: &mut Holders| { + let mut self_arg = || self.tp.self_arg(cls, ExtractErrorMode::Raise, holders, ctx); let call = if self.asyncness.is_some() { let throw_callback = if cancel_handle.is_some() { @@ -505,49 +706,85 @@ impl<'a> FnSpec<'a> { }; let python_name = &self.python_name; let qualname_prefix = match cls { - Some(cls) => quote!(Some(<#cls as _pyo3::PyTypeInfo>::NAME)), + Some(cls) => quote!(Some(<#cls as #pyo3_path::PyTypeInfo>::NAME)), None => quote!(None), }; + let arg_names = (0..args.len()) + .map(|i| format_ident!("arg_{}", i)) + .collect::>(); let future = match self.tp { FnType::Fn(SelfType::Receiver { mutable: false, .. }) => { - holders.pop().unwrap(); // does not actually use holder created by `self_arg` - quote! {{ - let __guard = _pyo3::impl_::coroutine::RefGuard::<#cls>::new(py.from_borrowed_ptr::<_pyo3::types::PyAny>(_slf))?; - async move { function(&__guard, #(#args),*).await } + #(let #arg_names = #args;)* + let __guard = unsafe { #pyo3_path::impl_::coroutine::RefGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))? }; + async move { function(&__guard, #(#arg_names),*).await } }} } FnType::Fn(SelfType::Receiver { mutable: true, .. }) => { - holders.pop().unwrap(); // does not actually use holder created by `self_arg` - quote! {{ - let mut __guard = _pyo3::impl_::coroutine::RefMutGuard::<#cls>::new(py.from_borrowed_ptr::<_pyo3::types::PyAny>(_slf))?; - async move { function(&mut __guard, #(#args),*).await } + #(let #arg_names = #args;)* + let mut __guard = unsafe { #pyo3_path::impl_::coroutine::RefMutGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))? }; + async move { function(&mut __guard, #(#arg_names),*).await } }} } - _ => quote! { function(#self_arg #(#args),*) }, + _ => { + if let Some(self_arg) = self_arg() { + quote! { + function( + // NB #self_arg includes a comma, so none inserted here + #self_arg + #(#args),* + ) + } + } else { + quote! { function(#(#args),*) } + } + } }; let mut call = quote! {{ let future = #future; - _pyo3::impl_::coroutine::new_coroutine( - _pyo3::intern!(py, stringify!(#python_name)), + #pyo3_path::impl_::coroutine::new_coroutine( + #pyo3_path::intern!(py, stringify!(#python_name)), #qualname_prefix, #throw_callback, - async move { _pyo3::impl_::wrap::OkWrap::wrap(future.await) }, + async move { + let fut = future.await; + #pyo3_path::impl_::wrap::converter(&fut).wrap(fut) + }, ) }}; if cancel_handle.is_some() { call = quote! {{ - let __cancel_handle = _pyo3::coroutine::CancelHandle::new(); + let __cancel_handle = #pyo3_path::coroutine::CancelHandle::new(); let __throw_callback = __cancel_handle.throw_callback(); #call }}; } call + } else if let Some(self_arg) = self_arg() { + quote! { + function( + // NB #self_arg includes a comma, so none inserted here + #self_arg + #(#args),* + ) + } } else { - quote! { function(#self_arg #(#args),*) } + quote! { function(#(#args),*) } }; - quotes::map_result_into_ptr(quotes::ok_wrap(call)) + + // We must assign the output_span to the return value of the call, + // but *not* of the call itself otherwise the spans get really weird + let ret_ident = Ident::new("ret", *output_span); + let ret_expr = quote! { let #ret_ident = #call; }; + let return_conversion = + quotes::map_result_into_ptr(quotes::ok_wrap(ret_ident.to_token_stream(), ctx), ctx); + quote! { + { + #ret_expr + #return_conversion + } + } }; let func_name = &self.name; @@ -557,93 +794,104 @@ impl<'a> FnSpec<'a> { quote!(#func_name) }; + let warnings = self.warnings.build_py_warning(ctx); + Ok(match self.convention { CallingConvention::Noargs => { - let mut holders = Vec::new(); + let mut holders = Holders::new(); let args = self .signature .arguments .iter() - .map(|arg| { - if arg.py { - quote!(py) - } else if arg.is_cancel_handle { - quote!(__cancel_handle) - } else { - unreachable!() - } + .map(|arg| match arg { + FnArg::Py(..) => quote!(py), + FnArg::CancelHandle(..) => quote!(__cancel_handle), + _ => unreachable!("`CallingConvention::Noargs` should not contain any arguments (reaching Python) except for `self`, which is handled below."), }) .collect(); let call = rust_call(args, &mut holders); - + let init_holders = holders.init_holders(ctx); quote! { unsafe fn #ident<'py>( - py: _pyo3::Python<'py>, - _slf: *mut _pyo3::ffi::PyObject, - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { + py: #pyo3_path::Python<'py>, + _slf: *mut #pyo3_path::ffi::PyObject, + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 - #( #holders )* - #call + #init_holders + #warnings + let result = #call; + result } } } CallingConvention::Fastcall => { - let mut holders = Vec::new(); - let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders)?; + let mut holders = Holders::new(); + let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders, ctx); let call = rust_call(args, &mut holders); + let init_holders = holders.init_holders(ctx); + quote! { unsafe fn #ident<'py>( - py: _pyo3::Python<'py>, - _slf: *mut _pyo3::ffi::PyObject, - _args: *const *mut _pyo3::ffi::PyObject, - _nargs: _pyo3::ffi::Py_ssize_t, - _kwnames: *mut _pyo3::ffi::PyObject - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { + py: #pyo3_path::Python<'py>, + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *const *mut #pyo3_path::ffi::PyObject, + _nargs: #pyo3_path::ffi::Py_ssize_t, + _kwnames: *mut #pyo3_path::ffi::PyObject + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert - #( #holders )* - #call + #init_holders + #warnings + let result = #call; + result } } } CallingConvention::Varargs => { - let mut holders = Vec::new(); - let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders)?; + let mut holders = Holders::new(); + let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx); let call = rust_call(args, &mut holders); + let init_holders = holders.init_holders(ctx); + quote! { unsafe fn #ident<'py>( - py: _pyo3::Python<'py>, - _slf: *mut _pyo3::ffi::PyObject, - _args: *mut _pyo3::ffi::PyObject, - _kwargs: *mut _pyo3::ffi::PyObject - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { + py: #pyo3_path::Python<'py>, + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *mut #pyo3_path::ffi::PyObject, + _kwargs: *mut #pyo3_path::ffi::PyObject + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert - #( #holders )* - #call + #init_holders + #warnings + let result = #call; + result } } } CallingConvention::TpNew => { - let mut holders = Vec::new(); - let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders)?; - let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, &mut holders); - let call = quote! { #rust_name(#self_arg #(#args),*) }; + let mut holders = Holders::new(); + let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx); + let self_arg = self + .tp + .self_arg(cls, ExtractErrorMode::Raise, &mut holders, ctx); + let call = quote_spanned! {*output_span=> #rust_name(#self_arg #(#args),*) }; + let init_holders = holders.init_holders(ctx); quote! { unsafe fn #ident( - py: _pyo3::Python<'_>, - _slf: *mut _pyo3::ffi::PyTypeObject, - _args: *mut _pyo3::ffi::PyObject, - _kwargs: *mut _pyo3::ffi::PyObject - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { - use _pyo3::callback::IntoPyCallbackOutput; + py: #pyo3_path::Python<'_>, + _slf: *mut #pyo3_path::ffi::PyTypeObject, + _args: *mut #pyo3_path::ffi::PyObject, + _kwargs: *mut #pyo3_path::ffi::PyObject + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + use #pyo3_path::impl_::callback::IntoPyCallbackOutput; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert - #( #holders )* + #init_holders + #warnings let result = #call; - let initializer: _pyo3::PyClassInitializer::<#cls> = result.convert(py)?; - let cell = initializer.create_cell_from_subtype(py, _slf)?; - ::std::result::Result::Ok(cell as *mut _pyo3::ffi::PyObject) + let initializer: #pyo3_path::PyClassInitializer::<#cls> = result.convert(py)?; + #pyo3_path::impl_::pymethods::tp_new_impl(py, initializer, _slf) } } } @@ -652,41 +900,44 @@ impl<'a> FnSpec<'a> { /// Return a `PyMethodDef` constructor for this function, matching the selected /// calling convention. - pub fn get_methoddef(&self, wrapper: impl ToTokens, doc: &PythonDoc) -> TokenStream { + pub fn get_methoddef(&self, wrapper: impl ToTokens, doc: &PythonDoc, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let python_name = self.null_terminated_python_name(); match self.convention { CallingConvention::Noargs => quote! { - _pyo3::impl_::pymethods::PyMethodDef::noargs( + #pyo3_path::impl_::pymethods::PyMethodDef::noargs( #python_name, - _pyo3::impl_::pymethods::PyCFunction({ + { unsafe extern "C" fn trampoline( - _slf: *mut _pyo3::ffi::PyObject, - _args: *mut _pyo3::ffi::PyObject, - ) -> *mut _pyo3::ffi::PyObject + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *mut #pyo3_path::ffi::PyObject, + ) -> *mut #pyo3_path::ffi::PyObject { - _pyo3::impl_::trampoline::noargs( - _slf, - _args, - #wrapper - ) + unsafe { + #pyo3_path::impl_::trampoline::noargs( + _slf, + _args, + #wrapper + ) + } } trampoline - }), + }, #doc, ) }, CallingConvention::Fastcall => quote! { - _pyo3::impl_::pymethods::PyMethodDef::fastcall_cfunction_with_keywords( + #pyo3_path::impl_::pymethods::PyMethodDef::fastcall_cfunction_with_keywords( #python_name, - _pyo3::impl_::pymethods::PyCFunctionFastWithKeywords({ + { unsafe extern "C" fn trampoline( - _slf: *mut _pyo3::ffi::PyObject, - _args: *const *mut _pyo3::ffi::PyObject, - _nargs: _pyo3::ffi::Py_ssize_t, - _kwnames: *mut _pyo3::ffi::PyObject - ) -> *mut _pyo3::ffi::PyObject + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *const *mut #pyo3_path::ffi::PyObject, + _nargs: #pyo3_path::ffi::Py_ssize_t, + _kwnames: *mut #pyo3_path::ffi::PyObject + ) -> *mut #pyo3_path::ffi::PyObject { - _pyo3::impl_::trampoline::fastcall_with_keywords( + #pyo3_path::impl_::trampoline::fastcall_with_keywords( _slf, _args, _nargs, @@ -695,21 +946,21 @@ impl<'a> FnSpec<'a> { ) } trampoline - }), + }, #doc, ) }, CallingConvention::Varargs => quote! { - _pyo3::impl_::pymethods::PyMethodDef::cfunction_with_keywords( + #pyo3_path::impl_::pymethods::PyMethodDef::cfunction_with_keywords( #python_name, - _pyo3::impl_::pymethods::PyCFunctionWithKeywords({ + { unsafe extern "C" fn trampoline( - _slf: *mut _pyo3::ffi::PyObject, - _args: *mut _pyo3::ffi::PyObject, - _kwargs: *mut _pyo3::ffi::PyObject, - ) -> *mut _pyo3::ffi::PyObject + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *mut #pyo3_path::ffi::PyObject, + _kwargs: *mut #pyo3_path::ffi::PyObject, + ) -> *mut #pyo3_path::ffi::PyObject { - _pyo3::impl_::trampoline::cfunction_with_keywords( + #pyo3_path::impl_::trampoline::cfunction_with_keywords( _slf, _args, _kwargs, @@ -717,7 +968,7 @@ impl<'a> FnSpec<'a> { ) } trampoline - }), + }, #doc, ) }, @@ -726,11 +977,11 @@ impl<'a> FnSpec<'a> { } /// Forwards to [utils::get_doc] with the text signature of this spec. - pub fn get_doc(&self, attrs: &[syn::Attribute]) -> PythonDoc { + pub fn get_doc(&self, attrs: &[syn::Attribute], ctx: &Ctx) -> syn::Result { let text_signature = self .text_signature_call_signature() .map(|sig| format!("{}{}", self.python_name, sig)); - utils::get_doc(attrs, text_signature) + utils::get_doc(attrs, text_signature, ctx) } /// Creates the parenthesised arguments list for `__text_signature__` snippet based on this spec's signature @@ -779,10 +1030,7 @@ impl MethodTypeAttribute { /// If the attribute does not match one of the attribute names, returns `Ok(None)`. /// /// Otherwise will either return a parse error or the attribute. - fn parse_if_matching_attribute( - attr: &syn::Attribute, - deprecations: &mut Deprecations, - ) -> Result> { + fn parse_if_matching_attribute(attr: &syn::Attribute) -> Result> { fn ensure_no_arguments(meta: &syn::Meta, ident: &str) -> syn::Result<()> { match meta { syn::Meta::Path(_) => Ok(()), @@ -826,11 +1074,6 @@ impl MethodTypeAttribute { if path.is_ident("new") { ensure_no_arguments(meta, "new")?; Ok(Some(MethodTypeAttribute::New(path.span()))) - } else if path.is_ident("__new__") { - let span = path.span(); - deprecations.push(Deprecation::PyMethodsNewDeprecatedForm, span); - ensure_no_arguments(meta, "__new__")?; - Ok(Some(MethodTypeAttribute::New(span))) } else if path.is_ident("classmethod") { ensure_no_arguments(meta, "classmethod")?; Ok(Some(MethodTypeAttribute::ClassMethod(path.span()))) @@ -865,15 +1108,12 @@ impl Display for MethodTypeAttribute { } } -fn parse_method_attributes( - attrs: &mut Vec, - deprecations: &mut Deprecations, -) -> Result> { +fn parse_method_attributes(attrs: &mut Vec) -> Result> { let mut new_attrs = Vec::new(); let mut found_attrs = Vec::new(); for attr in attrs.drain(..) { - match MethodTypeAttribute::parse_if_matching_attribute(&attr, deprecations)? { + match MethodTypeAttribute::parse_if_matching_attribute(&attr)? { Some(attr) => found_attrs.push(attr), None => new_attrs.push(attr), } @@ -887,7 +1127,7 @@ fn parse_method_attributes( const IMPL_TRAIT_ERR: &str = "Python functions cannot have `impl Trait` arguments"; const RECEIVER_BY_VALUE_ERR: &str = "Python objects are shared, so 'self' cannot be moved out of the Python interpreter. -Try `&self`, `&mut self, `slf: PyRef<'_, Self>` or `slf: PyRefMut<'_, Self>`."; +Try `&self`, `&mut self, `slf: PyClassGuard<'_, Self>` or `slf: PyClassGuardMut<'_, Self>`."; fn ensure_signatures_on_valid_method( fn_type: &FnType, @@ -897,15 +1137,18 @@ fn ensure_signatures_on_valid_method( if let Some(signature) = signature { match fn_type { FnType::Getter(_) => { + debug_assert!(!fn_type.signature_attribute_allowed()); bail_spanned!(signature.kw.span() => "`signature` not allowed with `getter`") } FnType::Setter(_) => { + debug_assert!(!fn_type.signature_attribute_allowed()); bail_spanned!(signature.kw.span() => "`signature` not allowed with `setter`") } FnType::ClassAttribute => { + debug_assert!(!fn_type.signature_attribute_allowed()); bail_spanned!(signature.kw.span() => "`signature` not allowed with `classattr`") } - _ => {} + _ => debug_assert!(fn_type.signature_attribute_allowed()), } } if let Some(text_signature) = text_signature { diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index ccd84bb363a..831bd677b92 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -1,125 +1,579 @@ //! Code generation for the function that initializes a python module and adds classes and function. +#[cfg(feature = "experimental-inspect")] +use crate::introspection::{ + attribute_introspection_code, introspection_id_const, module_introspection_code, +}; +#[cfg(feature = "experimental-inspect")] +use crate::utils::expr_to_python; use crate::{ - attributes::{self, take_attributes, take_pyo3_options, CrateAttribute, NameAttribute}, + attributes::{ + self, kw, take_attributes, take_pyo3_options, CrateAttribute, GILUsedAttribute, + ModuleAttribute, NameAttribute, SubmoduleAttribute, + }, + combine_errors::CombineErrors, + get_doc, + pyclass::PyClassPyO3Option, pyfunction::{impl_wrap_pyfunction, PyFunctionOptions}, - utils::{get_pyo3_crate, PythonDoc}, + utils::{has_attribute, has_attribute_with_namespace, Ctx, IdentOrStr}, }; -use proc_macro2::TokenStream; +use proc_macro2::{Span, TokenStream}; use quote::quote; +use std::ffi::CString; +use syn::LitCStr; use syn::{ ext::IdentExt, parse::{Parse, ParseStream}, + parse_quote, parse_quote_spanned, + punctuated::Punctuated, spanned::Spanned, token::Comma, - Ident, Path, Result, Visibility, + Item, Meta, Path, Result, }; #[derive(Default)] pub struct PyModuleOptions { krate: Option, - name: Option, + name: Option, + module: Option, + submodule: Option, + gil_used: Option, } -impl PyModuleOptions { - pub fn from_attrs(attrs: &mut Vec) -> Result { +impl Parse for PyModuleOptions { + fn parse(input: ParseStream<'_>) -> syn::Result { let mut options: PyModuleOptions = Default::default(); - for option in take_pyo3_options(attrs)? { - match option { - PyModulePyO3Option::Name(name) => options.set_name(name.value.0)?, - PyModulePyO3Option::Crate(path) => options.set_crate(path)?, - } - } + options.add_attributes( + Punctuated::::parse_terminated(input)?, + )?; Ok(options) } +} + +impl PyModuleOptions { + fn take_pyo3_options(&mut self, attrs: &mut Vec) -> Result<()> { + self.add_attributes(take_pyo3_options(attrs)?) + } - fn set_name(&mut self, name: syn::Ident) -> Result<()> { - ensure_spanned!( - self.name.is_none(), - name.span() => "`name` may only be specified once" - ); + fn add_attributes( + &mut self, + attrs: impl IntoIterator, + ) -> Result<()> { + macro_rules! set_option { + ($key:ident $(, $extra:literal)?) => { + { + ensure_spanned!( + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once" $(, $extra)?) + ); + self.$key = Some($key); + } + }; + } + attrs + .into_iter() + .map(|attr| { + match attr { + PyModulePyO3Option::Crate(krate) => set_option!(krate), + PyModulePyO3Option::Name(name) => set_option!(name), + PyModulePyO3Option::Module(module) => set_option!(module), + PyModulePyO3Option::Submodule(submodule) => set_option!( + submodule, + " (it is implicitly always specified for nested modules)" + ), + PyModulePyO3Option::GILUsed(gil_used) => { + set_option!(gil_used) + } + } - self.name = Some(name); + Ok(()) + }) + .try_combine_syn_errors()?; Ok(()) } +} - fn set_crate(&mut self, path: CrateAttribute) -> Result<()> { - ensure_spanned!( - self.krate.is_none(), - path.span() => "`crate` may only be specified once" - ); +pub fn pymodule_module_impl( + module: &mut syn::ItemMod, + mut options: PyModuleOptions, +) -> Result { + let syn::ItemMod { + attrs, + vis, + unsafety: _, + ident, + mod_token, + content, + semi: _, + } = module; + let items = if let Some((_, items)) = content { + items + } else { + bail_spanned!(mod_token.span() => "`#[pymodule]` can only be used on inline modules") + }; + options.take_pyo3_options(attrs)?; + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = ctx; + let doc = get_doc(attrs, None, ctx)?; + let name = options + .name + .map_or_else(|| ident.unraw(), |name| name.value.0); + let full_name = if let Some(module) = &options.module { + format!("{}.{}", module.value.value(), name) + } else { + name.to_string() + }; - self.krate = Some(path); + let mut module_items = Vec::new(); + let mut module_items_cfg_attrs = Vec::new(); + #[cfg(feature = "experimental-inspect")] + let mut introspection_chunks = Vec::new(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_chunks = Vec::::new(); + + fn extract_use_items( + source: &syn::UseTree, + cfg_attrs: &[syn::Attribute], + target_items: &mut Vec, + target_cfg_attrs: &mut Vec>, + ) -> Result<()> { + match source { + syn::UseTree::Name(name) => { + target_items.push(name.ident.clone()); + target_cfg_attrs.push(cfg_attrs.to_vec()); + } + syn::UseTree::Path(path) => { + extract_use_items(&path.tree, cfg_attrs, target_items, target_cfg_attrs)? + } + syn::UseTree::Group(group) => { + for tree in &group.items { + extract_use_items(tree, cfg_attrs, target_items, target_cfg_attrs)? + } + } + syn::UseTree::Glob(glob) => { + bail_spanned!(glob.span() => "#[pymodule] cannot import glob statements") + } + syn::UseTree::Rename(rename) => { + target_items.push(rename.rename.clone()); + target_cfg_attrs.push(cfg_attrs.to_vec()); + } + } Ok(()) } + + let mut pymodule_init = None; + let mut module_consts = Vec::new(); + let mut module_consts_cfg_attrs = Vec::new(); + + let _: Vec<()> = (*items).iter_mut().map(|item|{ + match item { + Item::Use(item_use) => { + let is_pymodule_export = + find_and_remove_attribute(&mut item_use.attrs, "pymodule_export"); + if is_pymodule_export { + let cfg_attrs = get_cfg_attributes(&item_use.attrs); + extract_use_items( + &item_use.tree, + &cfg_attrs, + &mut module_items, + &mut module_items_cfg_attrs, + )?; + } + } + Item::Fn(item_fn) => { + ensure_spanned!( + !has_attribute(&item_fn.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + let is_pymodule_init = + find_and_remove_attribute(&mut item_fn.attrs, "pymodule_init"); + let ident = &item_fn.sig.ident; + if is_pymodule_init { + ensure_spanned!( + !has_attribute(&item_fn.attrs, "pyfunction"), + item_fn.span() => "`#[pyfunction]` cannot be used alongside `#[pymodule_init]`" + ); + ensure_spanned!(pymodule_init.is_none(), item_fn.span() => "only one `#[pymodule_init]` may be specified"); + pymodule_init = Some(quote! { #ident(module)?; }); + } else if has_attribute(&item_fn.attrs, "pyfunction") + || has_attribute_with_namespace( + &item_fn.attrs, + Some(pyo3_path), + &["pyfunction"], + ) + || has_attribute_with_namespace( + &item_fn.attrs, + Some(pyo3_path), + &["prelude", "pyfunction"], + ) + { + module_items.push(ident.clone()); + module_items_cfg_attrs.push(get_cfg_attributes(&item_fn.attrs)); + } + } + Item::Struct(item_struct) => { + ensure_spanned!( + !has_attribute(&item_struct.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + if has_attribute(&item_struct.attrs, "pyclass") + || has_attribute_with_namespace( + &item_struct.attrs, + Some(pyo3_path), + &["pyclass"], + ) + || has_attribute_with_namespace( + &item_struct.attrs, + Some(pyo3_path), + &["prelude", "pyclass"], + ) + { + module_items.push(item_struct.ident.clone()); + module_items_cfg_attrs.push(get_cfg_attributes(&item_struct.attrs)); + if !has_pyo3_module_declared::( + &item_struct.attrs, + "pyclass", + |option| matches!(option, PyClassPyO3Option::Module(_)), + )? { + set_module_attribute(&mut item_struct.attrs, &full_name); + } + } + } + Item::Enum(item_enum) => { + ensure_spanned!( + !has_attribute(&item_enum.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + if has_attribute(&item_enum.attrs, "pyclass") + || has_attribute_with_namespace(&item_enum.attrs, Some(pyo3_path), &["pyclass"]) + || has_attribute_with_namespace( + &item_enum.attrs, + Some(pyo3_path), + &["prelude", "pyclass"], + ) + { + module_items.push(item_enum.ident.clone()); + module_items_cfg_attrs.push(get_cfg_attributes(&item_enum.attrs)); + if !has_pyo3_module_declared::( + &item_enum.attrs, + "pyclass", + |option| matches!(option, PyClassPyO3Option::Module(_)), + )? { + set_module_attribute(&mut item_enum.attrs, &full_name); + } + } + } + Item::Mod(item_mod) => { + ensure_spanned!( + !has_attribute(&item_mod.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + if has_attribute(&item_mod.attrs, "pymodule") + || has_attribute_with_namespace(&item_mod.attrs, Some(pyo3_path), &["pymodule"]) + || has_attribute_with_namespace( + &item_mod.attrs, + Some(pyo3_path), + &["prelude", "pymodule"], + ) + { + module_items.push(item_mod.ident.clone()); + module_items_cfg_attrs.push(get_cfg_attributes(&item_mod.attrs)); + if !has_pyo3_module_declared::( + &item_mod.attrs, + "pymodule", + |option| matches!(option, PyModulePyO3Option::Module(_)), + )? { + set_module_attribute(&mut item_mod.attrs, &full_name); + } + item_mod + .attrs + .push(parse_quote_spanned!(item_mod.mod_token.span()=> #[pyo3(submodule)])); + } + } + Item::ForeignMod(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + } + Item::Trait(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + } + Item::Const(item) => { + if !find_and_remove_attribute(&mut item.attrs, "pymodule_export") { + return Ok(()); + } + module_consts.push(item.ident.clone()); + module_consts_cfg_attrs.push(get_cfg_attributes(&item.attrs)); + #[cfg(feature = "experimental-inspect")] + { + let cfg_attrs = get_cfg_attributes(&item.attrs); + let chunk = attribute_introspection_code( + pyo3_path, + None, + item.ident.unraw().to_string(), + expr_to_python(&item.expr), + (*item.ty).clone(), + true, + ); + introspection_chunks.push(quote! { + #(#cfg_attrs)* + #chunk + }); + } + } + Item::Static(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + } + Item::Macro(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + } + Item::ExternCrate(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + } + Item::Impl(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + } + Item::TraitAlias(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + } + Item::Type(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + } + Item::Union(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" + ); + } + _ => (), + } + Ok(()) + }).try_combine_syn_errors()?; + + #[cfg(feature = "experimental-inspect")] + let introspection = module_introspection_code( + pyo3_path, + &name.to_string(), + &module_items, + &module_items_cfg_attrs, + pymodule_init.is_some(), + ); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; + #[cfg(feature = "experimental-inspect")] + let introspection_id = introspection_id_const(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_id = quote! {}; + + let module_def = quote! {{ + use #pyo3_path::impl_::pymodule as impl_; + const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); + unsafe { + impl_::ModuleDef::new( + __PYO3_NAME, + #doc, + INITIALIZER + ) + } + }}; + let initialization = module_initialization( + &name, + ctx, + module_def, + options.submodule.is_some(), + options.gil_used.is_none_or(|op| op.value.value), + ); + + let module_consts_names = module_consts.iter().map(|i| i.unraw().to_string()); + + Ok(quote!( + #(#attrs)* + #vis #mod_token #ident { + #(#items)* + + #initialization + #introspection + #introspection_id + #(#introspection_chunks)* + + fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { + use #pyo3_path::impl_::pymodule::PyAddToModule; + #( + #(#module_items_cfg_attrs)* + #module_items::_PYO3_DEF.add_to_module(module)?; + )* + + #( + #(#module_consts_cfg_attrs)* + #pyo3_path::types::PyModuleMethods::add(module, #module_consts_names, #module_consts)?; + )* + + #pymodule_init + ::std::result::Result::Ok(()) + } + } + )) } /// Generates the function that is called by the python interpreter to initialize the native /// module -pub fn pymodule_impl( - fnname: &Ident, - options: PyModuleOptions, - doc: PythonDoc, - visibility: &Visibility, -) -> TokenStream { - let name = options.name.unwrap_or_else(|| fnname.unraw()); - let krate = get_pyo3_crate(&options.krate); - let pyinit_symbol = format!("PyInit_{}", name); +pub fn pymodule_function_impl( + function: &mut syn::ItemFn, + mut options: PyModuleOptions, +) -> Result { + options.take_pyo3_options(&mut function.attrs)?; + process_functions_in_module(&options, function)?; + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = ctx; + let ident = &function.sig.ident; + let name = options + .name + .map_or_else(|| ident.unraw(), |name| name.value.0); + let vis = &function.vis; + let doc = get_doc(&function.attrs, None, ctx)?; - quote! { - // Create a module with the same name as the `#[pymodule]` - this way `use ` - // will actually bring both the module and the function into scope. - #[doc(hidden)] - #visibility mod #fnname { - pub(crate) struct MakeDef; - pub static DEF: #krate::impl_::pymodule::ModuleDef = MakeDef::make_def(); - pub const NAME: &'static str = concat!(stringify!(#name), "\0"); + let initialization = module_initialization( + &name, + ctx, + quote! { MakeDef::make_def() }, + false, + options.gil_used.is_none_or(|op| op.value.value), + ); - /// This autogenerated function is called by the python interpreter when importing - /// the module. - #[export_name = #pyinit_symbol] - pub unsafe extern "C" fn init() -> *mut #krate::ffi::PyObject { - #krate::impl_::trampoline::module_init(|py| DEF.make_module(py)) - } + #[cfg(feature = "experimental-inspect")] + let introspection = + module_introspection_code(pyo3_path, &name.unraw().to_string(), &[], &[], true); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; + #[cfg(feature = "experimental-inspect")] + let introspection_id = introspection_id_const(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_id = quote! {}; + + // Module function called with optional Python<'_> marker as first arg, followed by the module. + let mut module_args = Vec::new(); + if function.sig.inputs.len() == 2 { + module_args.push(quote!(module.py())); + } + module_args + .push(quote!(::std::convert::Into::into(#pyo3_path::impl_::pymethods::BoundRef(module)))); + + Ok(quote! { + #[doc(hidden)] + #vis mod #ident { + #initialization + #introspection + #introspection_id } // Generate the definition inside an anonymous function in the same scope as the original function - // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pymodule] is // inside a function body) - const _: () = { - use #krate::impl_::pymodule as impl_; - impl #fnname::MakeDef { - const fn make_def() -> impl_::ModuleDef { - const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(#fnname); - unsafe { - impl_::ModuleDef::new(#fnname::NAME, #doc, INITIALIZER) - } + #[allow(unknown_lints, non_local_definitions)] + impl #ident::MakeDef { + const fn make_def() -> #pyo3_path::impl_::pymodule::ModuleDef { + fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { + #ident(#(#module_args),*) + } + + const INITIALIZER: #pyo3_path::impl_::pymodule::ModuleInitializer = #pyo3_path::impl_::pymodule::ModuleInitializer(__pyo3_pymodule); + unsafe { + #pyo3_path::impl_::pymodule::ModuleDef::new( + #ident::__PYO3_NAME, + #doc, + INITIALIZER + ) } } - }; + } + }) +} + +fn module_initialization( + name: &syn::Ident, + ctx: &Ctx, + module_def: TokenStream, + is_submodule: bool, + gil_used: bool, +) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + let pyinit_symbol = format!("PyInit_{name}"); + let name = name.to_string(); + let pyo3_name = LitCStr::new(&CString::new(name).unwrap(), Span::call_site()); + + let mut result = quote! { + #[doc(hidden)] + pub const __PYO3_NAME: &'static ::std::ffi::CStr = #pyo3_name; + + pub(super) struct MakeDef; + #[doc(hidden)] + pub static _PYO3_DEF: #pyo3_path::impl_::pymodule::ModuleDef = #module_def; + #[doc(hidden)] + // so wrapped submodules can see what gil_used is + pub static __PYO3_GIL_USED: bool = #gil_used; + }; + if !is_submodule { + result.extend(quote! { + /// This autogenerated function is called by the python interpreter when importing + /// the module. + #[doc(hidden)] + #[export_name = #pyinit_symbol] + pub unsafe extern "C" fn __pyo3_init() -> *mut #pyo3_path::ffi::PyObject { + unsafe { #pyo3_path::impl_::trampoline::module_init(|py| _PYO3_DEF.make_module(py, #gil_used)) } + } + }); } + result } /// Finds and takes care of the #[pyfn(...)] in `#[pymodule]` -pub fn process_functions_in_module( - options: &PyModuleOptions, - func: &mut syn::ItemFn, -) -> syn::Result<()> { +fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn) -> Result<()> { + let ctx = &Ctx::new(&options.krate, None); + let Ctx { pyo3_path, .. } = ctx; let mut stmts: Vec = Vec::new(); - let krate = get_pyo3_crate(&options.krate); for mut stmt in func.block.stmts.drain(..) { - if let syn::Stmt::Item(syn::Item::Fn(func)) = &mut stmt { - if let Some(pyfn_args) = get_pyfn_attr(&mut func.attrs)? { + if let syn::Stmt::Item(Item::Fn(func)) = &mut stmt { + if let Some((pyfn_span, pyfn_args)) = get_pyfn_attr(&mut func.attrs)? { let module_name = pyfn_args.modname; let wrapped_function = impl_wrap_pyfunction(func, pyfn_args.options)?; let name = &func.sig.ident; - let statements: Vec = syn::parse_quote! { + let statements: Vec = syn::parse_quote_spanned! { + pyfn_span => #wrapped_function - #module_name.add_function(#krate::impl_::pyfunction::_wrap_pyfunction(&#name::DEF, #module_name)?)?; + { + use #pyo3_path::types::PyModuleMethods; + #module_name.add_function(#pyo3_path::wrap_pyfunction!(#name, #module_name.as_borrowed())?)?; + #[deprecated(note = "`pyfn` will be removed in a future PyO3 version, use declarative `#[pymodule]` with `mod` instead")] + #[allow(dead_code)] + const PYFN_ATTRIBUTE: () = (); + const _: () = PYFN_ATTRIBUTE; + } }; stmts.extend(statements); } @@ -159,8 +613,8 @@ impl Parse for PyFnArgs { } /// Extracts the data from the #[pyfn(...)] attribute of a function -fn get_pyfn_attr(attrs: &mut Vec) -> syn::Result> { - let mut pyfn_args: Option = None; +fn get_pyfn_attr(attrs: &mut Vec) -> syn::Result> { + let mut pyfn_args: Option<(Span, PyFnArgs)> = None; take_attributes(attrs, |attr| { if attr.path().is_ident("pyfn") { @@ -168,14 +622,14 @@ fn get_pyfn_attr(attrs: &mut Vec) -> syn::Result "`#[pyfn] may only be specified once" ); - pyfn_args = Some(attr.parse_args()?); + pyfn_args = Some((attr.path().span(), attr.parse_args()?)); Ok(true) } else { Ok(false) } })?; - if let Some(pyfn_args) = &mut pyfn_args { + if let Some((_, pyfn_args)) = &mut pyfn_args { pyfn_args .options .add_attributes(take_pyo3_options(attrs)?)?; @@ -184,9 +638,65 @@ fn get_pyfn_attr(attrs: &mut Vec) -> syn::Result Vec { + attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + .cloned() + .collect() +} + +fn find_and_remove_attribute(attrs: &mut Vec, ident: &str) -> bool { + let mut found = false; + attrs.retain(|attr| { + if attr.path().is_ident(ident) { + found = true; + false + } else { + true + } + }); + found +} + +impl PartialEq for IdentOrStr<'_> { + fn eq(&self, other: &syn::Ident) -> bool { + match self { + IdentOrStr::Str(s) => other == s, + IdentOrStr::Ident(i) => other == i, + } + } +} + +fn set_module_attribute(attrs: &mut Vec, module_name: &str) { + attrs.push(parse_quote!(#[pyo3(module = #module_name)])); +} + +fn has_pyo3_module_declared( + attrs: &[syn::Attribute], + root_attribute_name: &str, + is_module_option: impl Fn(&T) -> bool + Copy, +) -> Result { + for attr in attrs { + if (attr.path().is_ident("pyo3") || attr.path().is_ident(root_attribute_name)) + && matches!(attr.meta, Meta::List(_)) + { + for option in &attr.parse_args_with(Punctuated::::parse_terminated)? { + if is_module_option(option) { + return Ok(true); + } + } + } + } + Ok(false) +} + enum PyModulePyO3Option { + Submodule(SubmoduleAttribute), Crate(CrateAttribute), Name(NameAttribute), + Module(ModuleAttribute), + GILUsed(GILUsedAttribute), } impl Parse for PyModulePyO3Option { @@ -196,6 +706,12 @@ impl Parse for PyModulePyO3Option { input.parse().map(PyModulePyO3Option::Name) } else if lookahead.peek(syn::Token![crate]) { input.parse().map(PyModulePyO3Option::Crate) + } else if lookahead.peek(attributes::kw::module) { + input.parse().map(PyModulePyO3Option::Module) + } else if lookahead.peek(attributes::kw::submodule) { + input.parse().map(PyModulePyO3Option::Submodule) + } else if lookahead.peek(attributes::kw::gil_used) { + input.parse().map(PyModulePyO3Option::GILUsed) } else { Err(lookahead.error()) } diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 3781b41b765..9e71630346b 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -1,27 +1,46 @@ +use crate::utils::Ctx; use crate::{ - method::{FnArg, FnSpec}, + attributes::FromPyWithAttribute, + method::{FnArg, FnSpec, RegularArg}, pyfunction::FunctionSignature, quotes::some_wrap, }; use proc_macro2::{Span, TokenStream}; -use quote::{quote, quote_spanned}; +use quote::{format_ident, quote, quote_spanned}; use syn::spanned::Spanned; -use syn::Result; + +pub struct Holders { + holders: Vec, +} + +impl Holders { + pub fn new() -> Self { + Holders { + holders: Vec::new(), + } + } + + pub fn push_holder(&mut self, span: Span) -> syn::Ident { + let holder = syn::Ident::new(&format!("holder_{}", self.holders.len()), span); + self.holders.push(holder.clone()); + holder + } + + pub fn init_holders(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + let holders = &self.holders; + quote! { + #[allow(clippy::let_unit_value)] + #(let mut #holders = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT;)* + } + } +} /// Return true if the argument list is simply (*args, **kwds). pub fn is_forwarded_args(signature: &FunctionSignature<'_>) -> bool { matches!( signature.arguments.as_slice(), - [ - FnArg { - is_varargs: true, - .. - }, - FnArg { - is_kwargs: true, - .. - }, - ] + [FnArg::VarArgs(..), FnArg::KwArgs(..),] ) } @@ -29,9 +48,25 @@ pub fn impl_arg_params( spec: &FnSpec<'_>, self_: Option<&syn::Type>, fastcall: bool, - holders: &mut Vec, -) -> Result<(TokenStream, Vec)> { + holders: &mut Holders, + ctx: &Ctx, +) -> (TokenStream, Vec) { let args_array = syn::Ident::new("output", Span::call_site()); + let Ctx { pyo3_path, .. } = ctx; + + let from_py_with = spec + .signature + .arguments + .iter() + .enumerate() + .filter_map(|(i, arg)| { + let from_py_with = &arg.from_py_with()?.value; + let from_py_with_holder = format_ident!("from_py_with_{}", i); + Some(quote_spanned! { from_py_with.span() => + let #from_py_with_holder = #from_py_with; + }) + }) + .collect::(); if !fastcall && is_forwarded_args(&spec.signature) { // In the varargs convention, we can just pass though if the signature @@ -40,15 +75,17 @@ pub fn impl_arg_params( .signature .arguments .iter() - .map(|arg| impl_arg_param(arg, &mut 0, &args_array, holders)) - .collect::>()?; - return Ok(( + .enumerate() + .map(|(i, arg)| impl_arg_param(arg, i, &mut 0, holders, ctx)) + .collect(); + return ( quote! { - let _args = py.from_borrowed_ptr::<_pyo3::types::PyTuple>(_args); - let _kwargs: ::std::option::Option<&_pyo3::types::PyDict> = py.from_borrowed_ptr_or_opt(_kwargs); + let _args = unsafe { #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_args) }; + let _kwargs = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_kwargs); + #from_py_with }, arg_convert, - )); + ); }; let positional_parameter_names = &spec.signature.python_signature.positional_parameters; @@ -64,7 +101,7 @@ pub fn impl_arg_params( .iter() .map(|(name, required)| { quote! { - _pyo3::impl_::extract_argument::KeywordOnlyParameterDescription { + #pyo3_path::impl_::extract_argument::KeywordOnlyParameterDescription { name: #name, required: #required, } @@ -73,27 +110,28 @@ pub fn impl_arg_params( let num_params = positional_parameter_names.len() + keyword_only_parameters.len(); - let mut option_pos = 0; + let mut option_pos = 0usize; let param_conversion = spec .signature .arguments .iter() - .map(|arg| impl_arg_param(arg, &mut option_pos, &args_array, holders)) - .collect::>()?; + .enumerate() + .map(|(i, arg)| impl_arg_param(arg, i, &mut option_pos, holders, ctx)) + .collect(); let args_handler = if spec.signature.python_signature.varargs.is_some() { - quote! { _pyo3::impl_::extract_argument::TupleVarargs } + quote! { #pyo3_path::impl_::extract_argument::TupleVarargs } } else { - quote! { _pyo3::impl_::extract_argument::NoVarargs } + quote! { #pyo3_path::impl_::extract_argument::NoVarargs } }; let kwargs_handler = if spec.signature.python_signature.kwargs.is_some() { - quote! { _pyo3::impl_::extract_argument::DictVarkeywords } + quote! { #pyo3_path::impl_::extract_argument::DictVarkeywords } } else { - quote! { _pyo3::impl_::extract_argument::NoVarkeywords } + quote! { #pyo3_path::impl_::extract_argument::NoVarkeywords } }; let cls_name = if let Some(cls) = self_ { - quote! { ::std::option::Option::Some(<#cls as _pyo3::type_object::PyTypeInfo>::NAME) } + quote! { ::std::option::Option::Some(<#cls as #pyo3_path::type_object::PyTypeInfo>::NAME) } } else { quote! { ::std::option::Option::None } }; @@ -121,9 +159,9 @@ pub fn impl_arg_params( }; // create array of arguments, and then parse - Ok(( + ( quote! { - const DESCRIPTION: _pyo3::impl_::extract_argument::FunctionDescription = _pyo3::impl_::extract_argument::FunctionDescription { + const DESCRIPTION: #pyo3_path::impl_::extract_argument::FunctionDescription = #pyo3_path::impl_::extract_argument::FunctionDescription { cls_name: #cls_name, func_name: stringify!(#python_name), positional_parameter_names: &[#(#positional_parameter_names),*], @@ -133,136 +171,136 @@ pub fn impl_arg_params( }; let mut #args_array = [::std::option::Option::None; #num_params]; let (_args, _kwargs) = #extract_expression; + #from_py_with }, param_conversion, - )) + ) } -/// Re option_pos: The option slice doesn't contain the py: Python argument, so the argument -/// index and the index in option diverge when using py: Python fn impl_arg_param( arg: &FnArg<'_>, + pos: usize, option_pos: &mut usize, - args_array: &syn::Ident, - holders: &mut Vec, -) -> Result { - // Use this macro inside this function, to ensure that all code generated here is associated - // with the function argument - macro_rules! quote_arg_span { - ($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => $($tokens)*) } - } - - if arg.py { - return Ok(quote! { py }); - } + holders: &mut Holders, + ctx: &Ctx, +) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + let args_array = syn::Ident::new("output", Span::call_site()); - if arg.is_cancel_handle { - return Ok(quote! { __cancel_handle }); + match arg { + FnArg::Regular(arg) => { + let from_py_with = format_ident!("from_py_with_{}", pos); + let arg_value = quote!(#args_array[#option_pos].as_deref()); + *option_pos += 1; + impl_regular_arg_param(arg, from_py_with, arg_value, holders, ctx) + } + FnArg::VarArgs(arg) => { + let holder = holders.push_holder(arg.name.span()); + let name_str = arg.name.to_string(); + quote_spanned! { arg.name.span() => + #pyo3_path::impl_::extract_argument::extract_argument( + &_args, + &mut #holder, + #name_str + )? + } + } + FnArg::KwArgs(arg) => { + let holder = holders.push_holder(arg.name.span()); + let name_str = arg.name.to_string(); + quote_spanned! { arg.name.span() => + #pyo3_path::impl_::extract_argument::extract_argument_with_default( + _kwargs.as_deref(), + &mut #holder, + #name_str, + || ::std::option::Option::None + )? + } + } + FnArg::Py(..) => quote! { py }, + FnArg::CancelHandle(..) => quote! { __cancel_handle }, } +} - let name = arg.name; - let name_str = name.to_string(); +/// Re option_pos: The option slice doesn't contain the py: Python argument, so the argument +/// index and the index in option diverge when using py: Python +pub(crate) fn impl_regular_arg_param( + arg: &RegularArg<'_>, + from_py_with: syn::Ident, + arg_value: TokenStream, // expected type: Option<&'a Bound<'py, PyAny>> + holders: &mut Holders, + ctx: &Ctx, +) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + let pyo3_path = pyo3_path.to_tokens_spanned(arg.ty.span()); - let mut push_holder = || { - let holder = syn::Ident::new(&format!("holder_{}", holders.len()), arg.ty.span()); - holders.push(quote_arg_span! { - #[allow(clippy::let_unit_value)] - let mut #holder = _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT; - }); - holder + // Use this macro inside this function, to ensure that all code generated here is associated + // with the function argument + let use_probe = quote! { + #[allow(unused_imports)] + use #pyo3_path::impl_::pyclass::Probe as _; }; - - if arg.is_varargs { - ensure_spanned!( - arg.optional.is_none(), - arg.name.span() => "args cannot be optional" - ); - let holder = push_holder(); - return Ok(quote_arg_span! { - _pyo3::impl_::extract_argument::extract_argument( - _args, - &mut #holder, - #name_str - )? - }); - } else if arg.is_kwargs { - ensure_spanned!( - arg.optional.is_some(), - arg.name.span() => "kwargs must be Option<_>" - ); - let holder = push_holder(); - return Ok(quote_arg_span! { - _pyo3::impl_::extract_argument::extract_optional_argument( - _kwargs.map(::std::convert::AsRef::as_ref), - &mut #holder, - #name_str, - || ::std::option::Option::None - )? - }); + macro_rules! quote_arg_span { + ($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => { #use_probe $($tokens)* }) } } - let arg_value = quote_arg_span!(#args_array[#option_pos]); - *option_pos += 1; - - let mut default = arg.default.as_ref().map(|expr| quote!(#expr)); + let name_str = arg.name.to_string(); + let mut default = arg.default_value.as_ref().map(|expr| quote!(#expr)); // Option arguments have special treatment: the default should be specified _without_ the // Some() wrapper. Maybe this should be changed in future?! - if arg.optional.is_some() { - default = Some(default.map_or_else(|| quote!(::std::option::Option::None), some_wrap)); + if arg.option_wrapped_type.is_some() { + default = default.map(|tokens| some_wrap(tokens, ctx)); } - let tokens = if let Some(expr_path) = arg.attrs.from_py_with.as_ref().map(|attr| &attr.value) { + if let Some(FromPyWithAttribute { kw, .. }) = arg.from_py_with { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #from_py_with; from_py_with } + }; if let Some(default) = default { quote_arg_span! { - #[allow(clippy::redundant_closure)] - _pyo3::impl_::extract_argument::from_py_with_with_default( - #arg_value.map(_pyo3::PyNativeType::as_borrowed).as_deref(), + #pyo3_path::impl_::extract_argument::from_py_with_with_default( + #arg_value, #name_str, - #expr_path as fn(_) -> _, - || #default + #extractor, + #[allow(clippy::redundant_closure)] + { + || #default + } )? } } else { + let unwrap = quote! {unsafe { #pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value) }}; quote_arg_span! { - _pyo3::impl_::extract_argument::from_py_with( - &_pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value).as_borrowed(), + #pyo3_path::impl_::extract_argument::from_py_with( + #unwrap, #name_str, - #expr_path as fn(_) -> _, + #extractor, )? } } - } else if arg.optional.is_some() { - let holder = push_holder(); - quote_arg_span! { - #[allow(clippy::redundant_closure)] - _pyo3::impl_::extract_argument::extract_optional_argument( - #arg_value, - &mut #holder, - #name_str, - || #default - )? - } } else if let Some(default) = default { - let holder = push_holder(); + let holder = holders.push_holder(arg.name.span()); quote_arg_span! { - #[allow(clippy::redundant_closure)] - _pyo3::impl_::extract_argument::extract_argument_with_default( + #pyo3_path::impl_::extract_argument::extract_argument_with_default( #arg_value, &mut #holder, #name_str, - || #default + #[allow(clippy::redundant_closure)] + { + || #default + } )? } } else { - let holder = push_holder(); + let holder = holders.push_holder(arg.name.span()); + let unwrap = quote! {unsafe { #pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value) }}; quote_arg_span! { - _pyo3::impl_::extract_argument::extract_argument( - _pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value), + #pyo3_path::impl_::extract_argument::extract_argument( + #unwrap, &mut #holder, #name_str )? } - }; - Ok(tokens) + } } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 02024011366..07fccf3d655 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1,26 +1,39 @@ use std::borrow::Cow; +use std::fmt::Debug; + +use proc_macro2::{Ident, Span, TokenStream}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; +use syn::ext::IdentExt; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::{parse_quote, parse_quote_spanned, spanned::Spanned, ImplItemFn, Result, Token}; use crate::attributes::kw::frozen; use crate::attributes::{ self, kw, take_pyo3_options, CrateAttribute, ExtendsAttribute, FreelistAttribute, - ModuleAttribute, NameAttribute, NameLitStr, RenameAllAttribute, + ModuleAttribute, NameAttribute, NameLitStr, RenameAllAttribute, StrFormatterAttribute, +}; +use crate::combine_errors::CombineErrors; +#[cfg(feature = "experimental-inspect")] +use crate::introspection::{ + class_introspection_code, function_introspection_code, introspection_id_const, }; -use crate::deprecations::Deprecations; use crate::konst::{ConstAttributes, ConstSpec}; -use crate::method::{FnArg, FnSpec}; -use crate::pyimpl::{gen_py_const, PyClassMethodsType}; +use crate::method::{FnArg, FnSpec, PyArg, RegularArg}; +use crate::pyfunction::ConstructorAttribute; +#[cfg(feature = "experimental-inspect")] +use crate::pyfunction::FunctionSignature; +use crate::pyimpl::{gen_py_const, get_cfg_attributes, PyClassMethodsType}; +#[cfg(feature = "experimental-inspect")] +use crate::pymethod::field_python_name; use crate::pymethod::{ - impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType, - SlotDef, __INT__, __REPR__, __RICHCMP__, + impl_py_class_attribute, impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, + MethodAndSlotDef, PropertyType, SlotDef, __GETITEM__, __HASH__, __INT__, __LEN__, __REPR__, + __RICHCMP__, __STR__, }; -use crate::utils::{self, apply_renaming_rule, get_pyo3_crate, PythonDoc}; +use crate::pyversions::{is_abi3_before, is_py_before}; +use crate::utils::{self, apply_renaming_rule, Ctx, PythonDoc}; use crate::PyFunctionOptions; -use proc_macro2::{Ident, Span, TokenStream}; -use quote::{format_ident, quote}; -use syn::ext::IdentExt; -use syn::parse::{Parse, ParseStream}; -use syn::punctuated::Punctuated; -use syn::{parse_quote, spanned::Spanned, Result, Token}; /// If the class is derived from a Rust `struct` or `enum`. #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -44,7 +57,7 @@ impl PyClassArgs { }) } - pub fn parse_stuct_args(input: ParseStream<'_>) -> syn::Result { + pub fn parse_struct_args(input: ParseStream<'_>) -> syn::Result { Self::parse(input, PyClassKind::Struct) } @@ -57,37 +70,55 @@ impl PyClassArgs { pub struct PyClassPyO3Options { pub krate: Option, pub dict: Option, + pub eq: Option, + pub eq_int: Option, pub extends: Option, pub get_all: Option, pub freelist: Option, pub frozen: Option, + pub hash: Option, + pub immutable_type: Option, pub mapping: Option, pub module: Option, pub name: Option, + pub ord: Option, pub rename_all: Option, pub sequence: Option, pub set_all: Option, + pub str: Option, pub subclass: Option, pub unsendable: Option, pub weakref: Option, + pub generic: Option, + pub from_py_object: Option, + pub skip_from_py_object: Option, } -enum PyClassPyO3Option { +pub enum PyClassPyO3Option { Crate(CrateAttribute), Dict(kw::dict), + Eq(kw::eq), + EqInt(kw::eq_int), Extends(ExtendsAttribute), Freelist(FreelistAttribute), Frozen(kw::frozen), GetAll(kw::get_all), + Hash(kw::hash), + ImmutableType(kw::immutable_type), Mapping(kw::mapping), Module(ModuleAttribute), Name(NameAttribute), + Ord(kw::ord), RenameAll(RenameAllAttribute), Sequence(kw::sequence), SetAll(kw::set_all), + Str(StrFormatterAttribute), Subclass(kw::subclass), Unsendable(kw::unsendable), Weakref(kw::weakref), + Generic(kw::generic), + FromPyObject(kw::from_py_object), + SkipFromPyObject(kw::skip_from_py_object), } impl Parse for PyClassPyO3Option { @@ -97,6 +128,10 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::Crate) } else if lookahead.peek(kw::dict) { input.parse().map(PyClassPyO3Option::Dict) + } else if lookahead.peek(kw::eq) { + input.parse().map(PyClassPyO3Option::Eq) + } else if lookahead.peek(kw::eq_int) { + input.parse().map(PyClassPyO3Option::EqInt) } else if lookahead.peek(kw::extends) { input.parse().map(PyClassPyO3Option::Extends) } else if lookahead.peek(attributes::kw::freelist) { @@ -105,24 +140,38 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::Frozen) } else if lookahead.peek(attributes::kw::get_all) { input.parse().map(PyClassPyO3Option::GetAll) + } else if lookahead.peek(attributes::kw::hash) { + input.parse().map(PyClassPyO3Option::Hash) + } else if lookahead.peek(attributes::kw::immutable_type) { + input.parse().map(PyClassPyO3Option::ImmutableType) } else if lookahead.peek(attributes::kw::mapping) { input.parse().map(PyClassPyO3Option::Mapping) } else if lookahead.peek(attributes::kw::module) { input.parse().map(PyClassPyO3Option::Module) } else if lookahead.peek(kw::name) { input.parse().map(PyClassPyO3Option::Name) + } else if lookahead.peek(attributes::kw::ord) { + input.parse().map(PyClassPyO3Option::Ord) } else if lookahead.peek(kw::rename_all) { input.parse().map(PyClassPyO3Option::RenameAll) } else if lookahead.peek(attributes::kw::sequence) { input.parse().map(PyClassPyO3Option::Sequence) } else if lookahead.peek(attributes::kw::set_all) { input.parse().map(PyClassPyO3Option::SetAll) + } else if lookahead.peek(attributes::kw::str) { + input.parse().map(PyClassPyO3Option::Str) } else if lookahead.peek(attributes::kw::subclass) { input.parse().map(PyClassPyO3Option::Subclass) } else if lookahead.peek(attributes::kw::unsendable) { input.parse().map(PyClassPyO3Option::Unsendable) } else if lookahead.peek(attributes::kw::weakref) { input.parse().map(PyClassPyO3Option::Weakref) + } else if lookahead.peek(attributes::kw::generic) { + input.parse().map(PyClassPyO3Option::Generic) + } else if lookahead.peek(attributes::kw::from_py_object) { + input.parse().map(PyClassPyO3Option::FromPyObject) + } else if lookahead.peek(attributes::kw::skip_from_py_object) { + input.parse().map(PyClassPyO3Option::SkipFromPyObject) } else { Err(lookahead.error()) } @@ -163,20 +212,59 @@ impl PyClassPyO3Options { match option { PyClassPyO3Option::Crate(krate) => set_option!(krate), - PyClassPyO3Option::Dict(dict) => set_option!(dict), + PyClassPyO3Option::Dict(dict) => { + ensure_spanned!( + !is_abi3_before(3, 9), + dict.span() => "`dict` requires Python >= 3.9 when using the `abi3` feature" + ); + set_option!(dict); + } + PyClassPyO3Option::Eq(eq) => set_option!(eq), + PyClassPyO3Option::EqInt(eq_int) => set_option!(eq_int), PyClassPyO3Option::Extends(extends) => set_option!(extends), PyClassPyO3Option::Freelist(freelist) => set_option!(freelist), PyClassPyO3Option::Frozen(frozen) => set_option!(frozen), PyClassPyO3Option::GetAll(get_all) => set_option!(get_all), + PyClassPyO3Option::ImmutableType(immutable_type) => { + ensure_spanned!( + !(is_py_before(3, 10) || is_abi3_before(3, 14)), + immutable_type.span() => "`immutable_type` requires Python >= 3.10 or >= 3.14 (ABI3)" + ); + set_option!(immutable_type) + } + PyClassPyO3Option::Hash(hash) => set_option!(hash), PyClassPyO3Option::Mapping(mapping) => set_option!(mapping), PyClassPyO3Option::Module(module) => set_option!(module), PyClassPyO3Option::Name(name) => set_option!(name), + PyClassPyO3Option::Ord(ord) => set_option!(ord), PyClassPyO3Option::RenameAll(rename_all) => set_option!(rename_all), PyClassPyO3Option::Sequence(sequence) => set_option!(sequence), PyClassPyO3Option::SetAll(set_all) => set_option!(set_all), + PyClassPyO3Option::Str(str) => set_option!(str), PyClassPyO3Option::Subclass(subclass) => set_option!(subclass), PyClassPyO3Option::Unsendable(unsendable) => set_option!(unsendable), - PyClassPyO3Option::Weakref(weakref) => set_option!(weakref), + PyClassPyO3Option::Weakref(weakref) => { + ensure_spanned!( + !is_abi3_before(3, 9), + weakref.span() => "`weakref` requires Python >= 3.9 when using the `abi3` feature" + ); + set_option!(weakref); + } + PyClassPyO3Option::Generic(generic) => set_option!(generic), + PyClassPyO3Option::SkipFromPyObject(skip_from_py_object) => { + ensure_spanned!( + self.from_py_object.is_none(), + skip_from_py_object.span() => "`skip_from_py_object` and `from_py_object` are mutually exclusive" + ); + set_option!(skip_from_py_object) + } + PyClassPyO3Option::FromPyObject(from_py_object) => { + ensure_spanned!( + self.skip_from_py_object.is_none(), + from_py_object.span() => "`skip_from_py_object` and `from_py_object` are mutually exclusive" + ); + set_option!(from_py_object) + } } Ok(()) } @@ -188,52 +276,63 @@ pub fn build_py_class( methods_type: PyClassMethodsType, ) -> syn::Result { args.options.take_pyo3_options(&mut class.attrs)?; - let doc = utils::get_doc(&class.attrs, None); - let krate = get_pyo3_crate(&args.options.krate); + + let ctx = &Ctx::new(&args.options.krate, None); + let doc = utils::get_doc(&class.attrs, None, ctx)?; if let Some(lt) = class.generics.lifetimes().next() { bail_spanned!( - lt.span() => - "#[pyclass] cannot have lifetime parameters. \ - For an explanation, see https://pyo3.rs/latest/class.html#no-lifetime-parameters" + lt.span() => concat!( + "#[pyclass] cannot have lifetime parameters. For an explanation, see \ + https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#no-lifetime-parameters" + ) ); } ensure_spanned!( class.generics.params.is_empty(), - class.generics.span() => - "#[pyclass] cannot have generic parameters. \ - For an explanation, see https://pyo3.rs/latest/class.html#no-generic-parameters" + class.generics.span() => concat!( + "#[pyclass] cannot have generic parameters. For an explanation, see \ + https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#no-generic-parameters" + ) ); let mut field_options: Vec<(&syn::Field, FieldPyO3Options)> = match &mut class.fields { syn::Fields::Named(fields) => fields .named .iter_mut() - .map(|field| { - FieldPyO3Options::take_pyo3_options(&mut field.attrs) - .map(move |options| (&*field, options)) - }) - .collect::>()?, + .map( + |field| match FieldPyO3Options::take_pyo3_options(&mut field.attrs) { + Ok(options) => Ok((&*field, options)), + Err(e) => Err(e), + }, + ) + .collect::>(), syn::Fields::Unnamed(fields) => fields .unnamed .iter_mut() - .map(|field| { - FieldPyO3Options::take_pyo3_options(&mut field.attrs) - .map(move |options| (&*field, options)) - }) - .collect::>()?, + .map( + |field| match FieldPyO3Options::take_pyo3_options(&mut field.attrs) { + Ok(options) => Ok((&*field, options)), + Err(e) => Err(e), + }, + ) + .collect::>(), syn::Fields::Unit => { + let mut results = Vec::new(); + if let Some(attr) = args.options.set_all { - return Err(syn::Error::new_spanned(attr, UNIT_SET)); + results.push(Err(syn::Error::new_spanned(attr, UNIT_SET))); }; if let Some(attr) = args.options.get_all { - return Err(syn::Error::new_spanned(attr, UNIT_GET)); + results.push(Err(syn::Error::new_spanned(attr, UNIT_GET))); }; - // No fields for unit struct - Vec::new() + + results } - }; + } + .into_iter() + .try_combine_syn_errors()?; if let Some(attr) = args.options.get_all { for (_, FieldPyO3Options { get, .. }) in &mut field_options { @@ -251,7 +350,7 @@ pub fn build_py_class( } } - impl_class(&class.ident, &args, doc, field_options, methods_type, krate) + impl_class(&class.ident, &args, doc, field_options, methods_type, ctx) } enum Annotated { @@ -336,39 +435,87 @@ fn get_class_python_name<'a>(cls: &'a syn::Ident, args: &'a PyClassArgs) -> Cow< .unwrap_or_else(|| Cow::Owned(cls.unraw())) } +#[cfg(feature = "experimental-inspect")] +fn get_class_type_hint(cls: &Ident, args: &PyClassArgs, ctx: &Ctx) -> TokenStream { + let pyo3_path = &ctx.pyo3_path; + let name = get_class_python_name(cls, args).to_string(); + if let Some(module) = &args.options.module { + let module = module.value.value(); + quote! { #pyo3_path::inspect::TypeHint::module_attr(#module, #name) } + } else { + quote! { #pyo3_path::inspect::TypeHint::builtin(#name) } + } +} + fn impl_class( cls: &syn::Ident, args: &PyClassArgs, doc: PythonDoc, field_options: Vec<(&syn::Field, FieldPyO3Options)>, methods_type: PyClassMethodsType, - krate: syn::Path, + ctx: &Ctx, ) -> syn::Result { - let pytypeinfo_impl = impl_pytypeinfo(cls, args, None); + let Ctx { pyo3_path, .. } = ctx; + let pytypeinfo_impl = impl_pytypeinfo(cls, args, ctx); + + if let Some(str) = &args.options.str { + if str.value.is_some() { + // check if any renaming is present + let no_naming_conflict = field_options.iter().all(|x| x.1.name.is_none()) + & args.options.name.is_none() + & args.options.rename_all.is_none(); + ensure_spanned!(no_naming_conflict, str.value.span() => "The format string syntax is incompatible with any renaming via `name` or `rename_all`"); + } + } - let py_class_impl = PyClassImplsBuilder::new( + let mut default_methods = descriptors_to_items( cls, - args, - methods_type, - descriptors_to_items( - cls, - args.options.rename_all.as_ref(), - args.options.frozen, - field_options, - )?, - vec![], - ) - .doc(doc) - .impl_all()?; + args.options.rename_all.as_ref(), + args.options.frozen, + field_options, + ctx, + )?; + + let (default_class_geitem, default_class_geitem_method) = + pyclass_class_geitem(&args.options, &syn::parse_quote!(#cls), ctx)?; + + if let Some(default_class_geitem_method) = default_class_geitem_method { + default_methods.push(default_class_geitem_method); + } + + let (default_str, default_str_slot) = + implement_pyclass_str(&args.options, &syn::parse_quote!(#cls), ctx); + + let (default_richcmp, default_richcmp_slot) = + pyclass_richcmp(&args.options, &syn::parse_quote!(#cls), ctx)?; + + let (default_hash, default_hash_slot) = + pyclass_hash(&args.options, &syn::parse_quote!(#cls), ctx)?; + + let mut slots = Vec::new(); + slots.extend(default_richcmp_slot); + slots.extend(default_hash_slot); + slots.extend(default_str_slot); + + let py_class_impl = PyClassImplsBuilder::new(cls, args, methods_type, default_methods, slots) + .doc(doc) + .impl_all(ctx)?; Ok(quote! { - const _: () = { - use #krate as _pyo3; + impl #pyo3_path::types::DerefToPyAny for #cls {} - #pytypeinfo_impl + #pytypeinfo_impl - #py_class_impl - }; + #py_class_impl + + #[doc(hidden)] + #[allow(non_snake_case)] + impl #cls { + #default_richcmp + #default_hash + #default_str + #default_class_geitem + } }) } @@ -401,6 +548,7 @@ pub fn build_py_enum( ) -> syn::Result { args.options.take_pyo3_options(&mut enum_.attrs)?; + let ctx = &Ctx::new(&args.options.krate, None); if let Some(extends) = &args.options.extends { bail_spanned!(extends.span() => "enums can't extend from other classes"); } else if let Some(subclass) = &args.options.subclass { @@ -409,9 +557,13 @@ pub fn build_py_enum( bail_spanned!(enum_.brace_token.span.join() => "#[pyclass] can't be used on enums without any variants"); } - let doc = utils::get_doc(&enum_.attrs, None); + if let Some(generic) = &args.options.generic { + bail_spanned!(generic.span() => "enums do not support #[pyclass(generic)]"); + } + + let doc = utils::get_doc(&enum_.attrs, None, ctx)?; let enum_ = PyClassEnum::new(enum_)?; - impl_enum(enum_, &args, doc, method_type) + impl_enum(enum_, &args, doc, method_type, ctx) } struct PyClassSimpleEnum<'a> { @@ -442,7 +594,12 @@ impl<'a> PyClassSimpleEnum<'a> { _ => bail_spanned!(variant.span() => "Must be a unit variant."), }; let options = EnumVariantPyO3Options::take_pyo3_options(&mut variant.attrs)?; - Ok(PyClassEnumUnitVariant { ident, options }) + let cfg_attrs = get_cfg_attributes(&variant.attrs); + Ok(PyClassEnumUnitVariant { + ident, + options, + cfg_attrs, + }) } let ident = &enum_.ident; @@ -500,10 +657,10 @@ impl<'a> PyClassComplexEnum<'a> { let variant = match &variant.fields { Fields::Unit => { bail_spanned!(variant.span() => format!( - "Unit variant `{ident}` is not yet supported in a complex enum\n\ - = help: change to a struct variant with no fields: `{ident} {{ }}`\n\ - = note: the enum is complex because of non-unit variant `{witness}`", - ident=ident, witness=witness)) + "Unit variant `{ident}` is not yet supported in a complex enum\n\ + = help: change to an empty tuple variant instead: `{ident}()`\n\ + = note: the enum is complex because of non-unit variant `{witness}`", + ident=ident, witness=witness)) } Fields::Named(fields) => { let fields = fields @@ -522,12 +679,21 @@ impl<'a> PyClassComplexEnum<'a> { options, }) } - Fields::Unnamed(_) => { - bail_spanned!(variant.span() => format!( - "Tuple variant `{ident}` is not yet supported in a complex enum\n\ - = help: change to a struct variant with named fields: `{ident} {{ /* fields */ }}`\n\ - = note: the enum is complex because of non-unit variant `{witness}`", - ident=ident, witness=witness)) + Fields::Unnamed(types) => { + let fields = types + .unnamed + .iter() + .map(|field| PyClassEnumVariantUnnamedField { + ty: &field.ty, + span: field.span(), + }) + .collect(); + + PyClassEnumVariant::Tuple(PyClassEnumTupleVariant { + ident, + fields, + options, + }) } }; @@ -549,7 +715,7 @@ impl<'a> PyClassComplexEnum<'a> { enum PyClassEnumVariant<'a> { // TODO(mkovaxx): Unit(PyClassEnumUnitVariant<'a>), Struct(PyClassEnumStructVariant<'a>), - // TODO(mkovaxx): Tuple(PyClassEnumTupleVariant<'a>), + Tuple(PyClassEnumTupleVariant<'a>), } trait EnumVariant { @@ -573,16 +739,18 @@ trait EnumVariant { } } -impl<'a> EnumVariant for PyClassEnumVariant<'a> { +impl EnumVariant for PyClassEnumVariant<'_> { fn get_ident(&self) -> &syn::Ident { match self { PyClassEnumVariant::Struct(struct_variant) => struct_variant.ident, + PyClassEnumVariant::Tuple(tuple_variant) => tuple_variant.ident, } } fn get_options(&self) -> &EnumVariantPyO3Options { match self { PyClassEnumVariant::Struct(struct_variant) => &struct_variant.options, + PyClassEnumVariant::Tuple(tuple_variant) => &tuple_variant.options, } } } @@ -591,9 +759,10 @@ impl<'a> EnumVariant for PyClassEnumVariant<'a> { struct PyClassEnumUnitVariant<'a> { ident: &'a syn::Ident, options: EnumVariantPyO3Options, + cfg_attrs: Vec<&'a syn::Attribute>, } -impl<'a> EnumVariant for PyClassEnumUnitVariant<'a> { +impl EnumVariant for PyClassEnumUnitVariant<'_> { fn get_ident(&self) -> &syn::Ident { self.ident } @@ -610,19 +779,33 @@ struct PyClassEnumStructVariant<'a> { options: EnumVariantPyO3Options, } +struct PyClassEnumTupleVariant<'a> { + ident: &'a syn::Ident, + fields: Vec>, + options: EnumVariantPyO3Options, +} + struct PyClassEnumVariantNamedField<'a> { ident: &'a syn::Ident, ty: &'a syn::Type, span: Span, } +struct PyClassEnumVariantUnnamedField<'a> { + ty: &'a syn::Type, + span: Span, +} + /// `#[pyo3()]` options for pyclass enum variants +#[derive(Clone, Default)] struct EnumVariantPyO3Options { name: Option, + constructor: Option, } enum EnumVariantPyO3Option { Name(NameAttribute), + Constructor(ConstructorAttribute), } impl Parse for EnumVariantPyO3Option { @@ -630,6 +813,8 @@ impl Parse for EnumVariantPyO3Option { let lookahead = input.lookahead1(); if lookahead.peek(attributes::kw::name) { input.parse().map(EnumVariantPyO3Option::Name) + } else if lookahead.peek(attributes::kw::constructor) { + input.parse().map(EnumVariantPyO3Option::Constructor) } else { Err(lookahead.error()) } @@ -638,21 +823,100 @@ impl Parse for EnumVariantPyO3Option { impl EnumVariantPyO3Options { fn take_pyo3_options(attrs: &mut Vec) -> Result { - let mut options = EnumVariantPyO3Options { name: None }; + let mut options = EnumVariantPyO3Options::default(); - for option in take_pyo3_options(attrs)? { - match option { - EnumVariantPyO3Option::Name(name) => { + take_pyo3_options(attrs)? + .into_iter() + .try_for_each(|option| options.set_option(option))?; + + Ok(options) + } + + fn set_option(&mut self, option: EnumVariantPyO3Option) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + { ensure_spanned!( - options.name.is_none(), - name.span() => "`name` may only be specified once" + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once") ); - options.name = Some(name); + self.$key = Some($key); } - } + }; } - Ok(options) + match option { + EnumVariantPyO3Option::Constructor(constructor) => set_option!(constructor), + EnumVariantPyO3Option::Name(name) => set_option!(name), + } + Ok(()) + } +} + +// todo(remove this dead code allowance once __repr__ is implemented +#[allow(dead_code)] +pub enum PyFmtName { + Str, + Repr, +} + +fn implement_py_formatting( + ty: &syn::Type, + ctx: &Ctx, + option: &StrFormatterAttribute, +) -> (ImplItemFn, MethodAndSlotDef) { + let mut fmt_impl = match &option.value { + Some(opt) => { + let fmt = &opt.fmt; + let args = &opt + .args + .iter() + .map(|member| quote! {self.#member}) + .collect::>(); + let fmt_impl: ImplItemFn = syn::parse_quote! { + fn __pyo3__generated____str__(&self) -> ::std::string::String { + ::std::format!(#fmt, #(#args, )*) + } + }; + fmt_impl + } + None => { + let fmt_impl: syn::ImplItemFn = syn::parse_quote! { + fn __pyo3__generated____str__(&self) -> ::std::string::String { + ::std::format!("{}", &self) + } + }; + fmt_impl + } + }; + let fmt_slot = generate_protocol_slot( + ty, + &mut fmt_impl, + &__STR__, + "__str__", + #[cfg(feature = "experimental-inspect")] + FunctionIntrospectionData { + names: &["__str__"], + arguments: Vec::new(), + returns: parse_quote! { ::std::string::String }, + }, + ctx, + ) + .unwrap(); + (fmt_impl, fmt_slot) +} + +fn implement_pyclass_str( + options: &PyClassPyO3Options, + ty: &syn::Type, + ctx: &Ctx, +) -> (Option, Option) { + match &options.str { + Some(option) => { + let (default_str, default_str_slot) = implement_py_formatting(ty, ctx, option); + (Some(default_str), Some(default_str_slot)) + } + _ => (None, None), } } @@ -661,11 +925,18 @@ fn impl_enum( args: &PyClassArgs, doc: PythonDoc, methods_type: PyClassMethodsType, + ctx: &Ctx, ) -> Result { + if let Some(str_fmt) = &args.options.str { + ensure_spanned!(str_fmt.value.is_none(), str_fmt.value.span() => "The format string syntax cannot be used with enums") + } + match enum_ { - PyClassEnum::Simple(simple_enum) => impl_simple_enum(simple_enum, args, doc, methods_type), + PyClassEnum::Simple(simple_enum) => { + impl_simple_enum(simple_enum, args, doc, methods_type, ctx) + } PyClassEnum::Complex(complex_enum) => { - impl_complex_enum(complex_enum, args, doc, methods_type) + impl_complex_enum(complex_enum, args, doc, methods_type, ctx) } } } @@ -675,97 +946,78 @@ fn impl_simple_enum( args: &PyClassArgs, doc: PythonDoc, methods_type: PyClassMethodsType, + ctx: &Ctx, ) -> Result { - let krate = get_pyo3_crate(&args.options.krate); let cls = simple_enum.ident; let ty: syn::Type = syn::parse_quote!(#cls); let variants = simple_enum.variants; - let pytypeinfo = impl_pytypeinfo(cls, args, None); + let pytypeinfo = impl_pytypeinfo(cls, args, ctx); + + for variant in &variants { + ensure_spanned!(variant.options.constructor.is_none(), variant.options.constructor.span() => "`constructor` can't be used on a simple enum variant"); + } + + let variant_cfg_check = generate_cfg_check(&variants, cls); let (default_repr, default_repr_slot) = { let variants_repr = variants.iter().map(|variant| { let variant_name = variant.ident; + let cfg_attrs = &variant.cfg_attrs; // Assuming all variants are unit variants because they are the only type we support. let repr = format!( "{}.{}", get_class_python_name(cls, args), variant.get_python_name(args), ); - quote! { #cls::#variant_name => #repr, } + quote! { #(#cfg_attrs)* #cls::#variant_name => #repr, } }); let mut repr_impl: syn::ImplItemFn = syn::parse_quote! { fn __pyo3__repr__(&self) -> &'static str { - match self { + match *self { #(#variants_repr)* } } }; - let repr_slot = generate_default_protocol_slot(&ty, &mut repr_impl, &__REPR__).unwrap(); + let repr_slot = generate_default_protocol_slot(&ty, &mut repr_impl, &__REPR__, ctx)?; (repr_impl, repr_slot) }; + let (default_str, default_str_slot) = implement_pyclass_str(&args.options, &ty, ctx); + let repr_type = &simple_enum.repr_type; let (default_int, default_int_slot) = { // This implementation allows us to convert &T to #repr_type without implementing `Copy` let variants_to_int = variants.iter().map(|variant| { let variant_name = variant.ident; - quote! { #cls::#variant_name => #cls::#variant_name as #repr_type, } + let cfg_attrs = &variant.cfg_attrs; + quote! { #(#cfg_attrs)* #cls::#variant_name => #cls::#variant_name as #repr_type, } }); let mut int_impl: syn::ImplItemFn = syn::parse_quote! { fn __pyo3__int__(&self) -> #repr_type { - match self { + match *self { #(#variants_to_int)* } } }; - let int_slot = generate_default_protocol_slot(&ty, &mut int_impl, &__INT__).unwrap(); + let int_slot = generate_default_protocol_slot(&ty, &mut int_impl, &__INT__, ctx)?; (int_impl, int_slot) }; - let (default_richcmp, default_richcmp_slot) = { - let mut richcmp_impl: syn::ImplItemFn = syn::parse_quote! { - fn __pyo3__richcmp__( - &self, - py: _pyo3::Python, - other: &_pyo3::PyAny, - op: _pyo3::basic::CompareOp - ) -> _pyo3::PyResult<_pyo3::PyObject> { - use _pyo3::conversion::ToPyObject; - use ::core::result::Result::*; - match op { - _pyo3::basic::CompareOp::Eq => { - let self_val = self.__pyo3__int__(); - if let Ok(i) = other.extract::<#repr_type>() { - return Ok((self_val == i).to_object(py)); - } - if let Ok(other) = other.extract::<_pyo3::PyRef>() { - return Ok((self_val == other.__pyo3__int__()).to_object(py)); - } - - return Ok(py.NotImplemented()); - } - _pyo3::basic::CompareOp::Ne => { - let self_val = self.__pyo3__int__(); - if let Ok(i) = other.extract::<#repr_type>() { - return Ok((self_val != i).to_object(py)); - } - if let Ok(other) = other.extract::<_pyo3::PyRef>() { - return Ok((self_val != other.__pyo3__int__()).to_object(py)); - } - - return Ok(py.NotImplemented()); - } - _ => Ok(py.NotImplemented()), - } - } - }; - let richcmp_slot = - generate_default_protocol_slot(&ty, &mut richcmp_impl, &__RICHCMP__).unwrap(); - (richcmp_impl, richcmp_slot) - }; - - let default_slots = vec![default_repr_slot, default_int_slot, default_richcmp_slot]; + let (default_richcmp, default_richcmp_slot) = pyclass_richcmp_simple_enum( + &args.options, + &ty, + repr_type, + #[cfg(feature = "experimental-inspect")] + &get_class_python_name(cls, args).to_string(), + ctx, + )?; + let (default_hash, default_hash_slot) = pyclass_hash(&args.options, &ty, ctx)?; + + let mut default_slots = vec![default_repr_slot, default_int_slot]; + default_slots.extend(default_richcmp_slot); + default_slots.extend(default_hash_slot); + default_slots.extend(default_str_slot); let pyclass_impls = PyClassImplsBuilder::new( cls, @@ -773,29 +1025,32 @@ fn impl_simple_enum( methods_type, simple_enum_default_methods( cls, - variants.iter().map(|v| (v.ident, v.get_python_name(args))), + variants + .iter() + .map(|v| (v.ident, v.get_python_name(args), &v.cfg_attrs)), + ctx, ), default_slots, ) .doc(doc) - .impl_all()?; + .impl_all(ctx)?; Ok(quote! { - const _: () = { - use #krate as _pyo3; + #variant_cfg_check - #pytypeinfo + #pytypeinfo - #pyclass_impls + #pyclass_impls - #[doc(hidden)] - #[allow(non_snake_case)] - impl #cls { - #default_repr - #default_int - #default_richcmp - } - }; + #[doc(hidden)] + #[allow(non_snake_case)] + impl #cls { + #default_repr + #default_int + #default_richcmp + #default_hash + #default_str + } }) } @@ -804,7 +1059,12 @@ fn impl_complex_enum( args: &PyClassArgs, doc: PythonDoc, methods_type: PyClassMethodsType, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path, .. } = ctx; + let cls = complex_enum.ident; + let ty: syn::Type = syn::parse_quote!(#cls); + // Need to rig the enum PyClass options let args = { let mut rigged_args = args.clone(); @@ -815,12 +1075,20 @@ fn impl_complex_enum( rigged_args }; - let krate = get_pyo3_crate(&args.options.krate); + let ctx = &Ctx::new(&args.options.krate, None); let cls = complex_enum.ident; let variants = complex_enum.variants; - let pytypeinfo = impl_pytypeinfo(cls, &args, None); + let pytypeinfo = impl_pytypeinfo(cls, &args, ctx); + + let (default_richcmp, default_richcmp_slot) = pyclass_richcmp(&args.options, &ty, ctx)?; + let (default_hash, default_hash_slot) = pyclass_hash(&args.options, &ty, ctx)?; - let default_slots = vec![]; + let (default_str, default_str_slot) = implement_pyclass_str(&args.options, &ty, ctx); + + let mut default_slots = vec![]; + default_slots.extend(default_richcmp_slot); + default_slots.extend(default_hash_slot); + default_slots.extend(default_str_slot); let impl_builder = PyClassImplsBuilder::new( cls, @@ -831,31 +1099,41 @@ fn impl_complex_enum( variants .iter() .map(|v| (v.get_ident(), v.get_python_name(&args))), + ctx, ), default_slots, ) .doc(doc); - // Need to customize the into_py impl so that it returns the variant PyClass - let enum_into_py_impl = { - let match_arms: Vec = variants + let enum_into_pyobject_impl = { + let match_arms = variants .iter() .map(|variant| { let variant_ident = variant.get_ident(); let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); quote! { #cls::#variant_ident { .. } => { - let pyclass_init = _pyo3::PyClassInitializer::from(self).add_subclass(#variant_cls); - let variant_value = _pyo3::Py::new(py, pyclass_init).unwrap(); - _pyo3::IntoPy::into_py(variant_value, py) + let pyclass_init = <#pyo3_path::PyClassInitializer as ::std::convert::From>::from(self).add_subclass(#variant_cls); + unsafe { #pyo3_path::Bound::new(py, pyclass_init).map(|b| b.cast_into_unchecked()) } } } - }) - .collect(); - + }); + let output_type = if cfg!(feature = "experimental-inspect") { + quote!(const OUTPUT_TYPE: #pyo3_path::inspect::TypeHint = <#cls as #pyo3_path::PyTypeInfo>::TYPE_HINT;) + } else { + TokenStream::new() + }; quote! { - impl _pyo3::IntoPy<_pyo3::PyObject> for #cls { - fn into_py(self, py: _pyo3::Python) -> _pyo3::PyObject { + impl<'py> #pyo3_path::conversion::IntoPyObject<'py> for #cls { + type Target = Self; + type Output = #pyo3_path::Bound<'py, >::Target>; + type Error = #pyo3_path::PyErr; + #output_type + + fn into_pyobject(self, py: #pyo3_path::Python<'py>) -> ::std::result::Result< + ::Output, + ::Error, + > { match self { #(#match_arms)* } @@ -864,12 +1142,13 @@ fn impl_complex_enum( } }; - let pyclass_impls: TokenStream = vec![ - impl_builder.impl_pyclass(), - impl_builder.impl_extractext(), - enum_into_py_impl, - impl_builder.impl_pyclassimpl()?, - impl_builder.impl_freelist(), + let pyclass_impls: TokenStream = [ + impl_builder.impl_pyclass(ctx), + enum_into_pyobject_impl, + impl_builder.impl_pyclassimpl(ctx)?, + impl_builder.impl_add_to_module(ctx), + impl_builder.impl_freelist(ctx), + impl_builder.impl_introspection(ctx), ] .into_iter() .collect(); @@ -878,7 +1157,7 @@ fn impl_complex_enum( let mut variant_cls_pytypeinfos = vec![]; let mut variant_cls_pyclass_impls = vec![]; let mut variant_cls_impls = vec![]; - for variant in &variants { + for variant in variants { let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); let variant_cls_zst = quote! { @@ -891,67 +1170,108 @@ fn impl_complex_enum( let variant_args = PyClassArgs { class_kind: PyClassKind::Struct, // TODO(mkovaxx): propagate variant.options - options: parse_quote!(extends = #cls, frozen), + options: { + let mut rigged_options: PyClassPyO3Options = parse_quote!(extends = #cls, frozen); + // If a specific module was given to the base class, use it for all variants. + rigged_options.module.clone_from(&args.options.module); + rigged_options + }, }; - let variant_cls_pytypeinfo = impl_pytypeinfo(&variant_cls, &variant_args, None); + let variant_cls_pytypeinfo = impl_pytypeinfo(&variant_cls, &variant_args, ctx); variant_cls_pytypeinfos.push(variant_cls_pytypeinfo); - let variant_new = complex_enum_variant_new(cls, variant)?; - - let (variant_cls_impl, field_getters) = impl_complex_enum_variant_cls(cls, variant)?; + let (variant_cls_impl, field_getters, mut slots) = + impl_complex_enum_variant_cls(cls, &variant, ctx)?; variant_cls_impls.push(variant_cls_impl); + let variant_new = complex_enum_variant_new(cls, variant, ctx)?; + slots.push(variant_new); + let pyclass_impl = PyClassImplsBuilder::new( &variant_cls, &variant_args, methods_type, field_getters, - vec![variant_new], + slots, ) - .impl_all()?; + .impl_all(ctx)?; variant_cls_pyclass_impls.push(pyclass_impl); } Ok(quote! { - const _: () = { - use #krate as _pyo3; + #pytypeinfo - #pytypeinfo + #pyclass_impls - #pyclass_impls - - #[doc(hidden)] - #[allow(non_snake_case)] - impl #cls {} + #[doc(hidden)] + #[allow(non_snake_case)] + impl #cls { + #default_richcmp + #default_hash + #default_str + } - #(#variant_cls_zsts)* + #(#variant_cls_zsts)* - #(#variant_cls_pytypeinfos)* + #(#variant_cls_pytypeinfos)* - #(#variant_cls_pyclass_impls)* + #(#variant_cls_pyclass_impls)* - #(#variant_cls_impls)* - }; + #(#variant_cls_impls)* }) } fn impl_complex_enum_variant_cls( enum_name: &syn::Ident, variant: &PyClassEnumVariant<'_>, -) -> Result<(TokenStream, Vec)> { + ctx: &Ctx, +) -> Result<(TokenStream, Vec, Vec)> { match variant { PyClassEnumVariant::Struct(struct_variant) => { - impl_complex_enum_struct_variant_cls(enum_name, struct_variant) + impl_complex_enum_struct_variant_cls(enum_name, struct_variant, ctx) + } + PyClassEnumVariant::Tuple(tuple_variant) => { + impl_complex_enum_tuple_variant_cls(enum_name, tuple_variant, ctx) } } } +fn impl_complex_enum_variant_match_args( + ctx @ Ctx { pyo3_path, .. }: &Ctx, + variant_cls_type: &syn::Type, + field_names: &[Ident], +) -> syn::Result<(MethodAndMethodDef, syn::ImplItemFn)> { + let ident = format_ident!("__match_args__"); + let field_names_unraw = field_names.iter().map(|name| name.unraw()); + let mut match_args_impl: syn::ImplItemFn = { + parse_quote! { + #[classattr] + fn #ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Bound<'_, #pyo3_path::types::PyTuple>> { + #pyo3_path::types::PyTuple::new::<&str, _>(py, [ + #(stringify!(#field_names_unraw),)* + ]) + } + } + }; + + let spec = FnSpec::parse( + &mut match_args_impl.sig, + &mut match_args_impl.attrs, + Default::default(), + )?; + let variant_match_args = impl_py_class_attribute(variant_cls_type, &spec, ctx)?; + + Ok((variant_match_args, match_args_impl)) +} + fn impl_complex_enum_struct_variant_cls( enum_name: &syn::Ident, variant: &PyClassEnumStructVariant<'_>, -) -> Result<(TokenStream, Vec)> { + ctx: &Ctx, +) -> Result<(TokenStream, Vec, Vec)> { + let Ctx { pyo3_path, .. } = ctx; let variant_ident = &variant.ident; let variant_cls = gen_complex_enum_variant_class_ident(enum_name, variant.ident); let variant_cls_type = parse_quote!(#variant_cls); @@ -965,18 +1285,20 @@ fn impl_complex_enum_struct_variant_cls( let field_type = field.ty; let field_with_type = quote! { #field_name: #field_type }; - let field_getter = complex_enum_variant_field_getter( - &variant_cls_type, - field_name, - field_type, - field.span, - )?; + let field_getter = + complex_enum_variant_field_getter(&variant_cls_type, field_name, field.span, ctx)?; let field_getter_impl = quote! { - fn #field_name(slf: _pyo3::PyRef) -> _pyo3::PyResult<#field_type> { + fn #field_name(slf: #pyo3_path::PyClassGuard<'_, Self>, py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Py<#pyo3_path::PyAny>> { + #[allow(unused_imports)] + use #pyo3_path::impl_::pyclass::Probe as _; match &*slf.into_super() { - #enum_name::#variant_ident { #field_name, .. } => Ok(#field_name.clone()), - _ => unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), + #enum_name::#variant_ident { #field_name, .. } => + #pyo3_path::impl_::pyclass::ConvertField::< + { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#field_type>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#field_type>::VALUE }, + >::convert_field::<#field_type>(#field_name, py), + _ => ::core::unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), } } }; @@ -987,48 +1309,290 @@ fn impl_complex_enum_struct_variant_cls( field_getter_impls.push(field_getter_impl); } + let (variant_match_args, match_args_const_impl) = + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &field_names)?; + + field_getters.push(variant_match_args); + let cls_impl = quote! { #[doc(hidden)] #[allow(non_snake_case)] impl #variant_cls { - fn __pymethod_constructor__(py: _pyo3::Python<'_>, #(#fields_with_types,)*) -> _pyo3::PyClassInitializer<#variant_cls> { + #[allow(clippy::too_many_arguments)] + fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#fields_with_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { let base_value = #enum_name::#variant_ident { #(#field_names,)* }; - _pyo3::PyClassInitializer::from(base_value).add_subclass(#variant_cls) + <#pyo3_path::PyClassInitializer<#enum_name> as ::std::convert::From<#enum_name>>::from(base_value).add_subclass(#variant_cls) + } + + #match_args_const_impl + + #(#field_getter_impls)* + } + }; + + Ok((cls_impl, field_getters, Vec::new())) +} + +fn impl_complex_enum_tuple_variant_field_getters( + ctx: &Ctx, + variant: &PyClassEnumTupleVariant<'_>, + enum_name: &syn::Ident, + variant_cls_type: &syn::Type, + variant_ident: &&Ident, + field_names: &mut Vec, + fields_types: &mut Vec, +) -> Result<(Vec, Vec)> { + let Ctx { pyo3_path, .. } = ctx; + + let mut field_getters = vec![]; + let mut field_getter_impls = vec![]; + + for (index, field) in variant.fields.iter().enumerate() { + let field_name = format_ident!("_{}", index); + let field_type = field.ty; + + let field_getter = + complex_enum_variant_field_getter(variant_cls_type, &field_name, field.span, ctx)?; + + // Generate the match arms needed to destructure the tuple and access the specific field + let field_access_tokens: Vec<_> = (0..variant.fields.len()) + .map(|i| { + if i == index { + quote! { val } + } else { + quote! { _ } + } + }) + .collect(); + let field_getter_impl: syn::ImplItemFn = parse_quote! { + fn #field_name(slf: #pyo3_path::PyClassGuard<'_, Self>, py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Py<#pyo3_path::PyAny>> { + #[allow(unused_imports)] + use #pyo3_path::impl_::pyclass::Probe as _; + match &*slf.into_super() { + #enum_name::#variant_ident ( #(#field_access_tokens), *) => + #pyo3_path::impl_::pyclass::ConvertField::< + { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#field_type>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#field_type>::VALUE }, + >::convert_field::<#field_type>(val, py), + _ => ::core::unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), + } + } + }; + + field_names.push(field_name); + fields_types.push(field_type.clone()); + field_getters.push(field_getter); + field_getter_impls.push(field_getter_impl); + } + + Ok((field_getters, field_getter_impls)) +} + +fn impl_complex_enum_tuple_variant_len( + ctx: &Ctx, + + variant_cls_type: &syn::Type, + num_fields: usize, +) -> Result<(MethodAndSlotDef, syn::ImplItemFn)> { + let Ctx { pyo3_path, .. } = ctx; + + let mut len_method_impl: syn::ImplItemFn = parse_quote! { + fn __len__(slf: #pyo3_path::PyClassGuard<'_, Self>) -> #pyo3_path::PyResult { + ::std::result::Result::Ok(#num_fields) + } + }; + + let variant_len = + generate_default_protocol_slot(variant_cls_type, &mut len_method_impl, &__LEN__, ctx)?; + + Ok((variant_len, len_method_impl)) +} + +fn impl_complex_enum_tuple_variant_getitem( + ctx: &Ctx, + variant_cls: &syn::Ident, + variant_cls_type: &syn::Type, + num_fields: usize, +) -> Result<(MethodAndSlotDef, syn::ImplItemFn)> { + let Ctx { pyo3_path, .. } = ctx; + + let match_arms: Vec<_> = (0..num_fields) + .map(|i| { + let field_access = format_ident!("_{}", i); + quote! { #i => + #pyo3_path::IntoPyObjectExt::into_py_any(#variant_cls::#field_access(slf, py)?, py) + } + }) + .collect(); + + let mut get_item_method_impl: syn::ImplItemFn = parse_quote! { + fn __getitem__(slf: #pyo3_path::PyClassGuard<'_, Self>, py: #pyo3_path::Python<'_>, idx: usize) -> #pyo3_path::PyResult< #pyo3_path::Py<#pyo3_path::PyAny>> { + match idx { + #( #match_arms, )* + _ => ::std::result::Result::Err(#pyo3_path::exceptions::PyIndexError::new_err("tuple index out of range")), + } + } + }; + + let variant_getitem = generate_default_protocol_slot( + variant_cls_type, + &mut get_item_method_impl, + &__GETITEM__, + ctx, + )?; + + Ok((variant_getitem, get_item_method_impl)) +} + +fn impl_complex_enum_tuple_variant_cls( + enum_name: &syn::Ident, + variant: &PyClassEnumTupleVariant<'_>, + ctx: &Ctx, +) -> Result<(TokenStream, Vec, Vec)> { + let Ctx { pyo3_path, .. } = ctx; + let variant_ident = &variant.ident; + let variant_cls = gen_complex_enum_variant_class_ident(enum_name, variant.ident); + let variant_cls_type = parse_quote!(#variant_cls); + + let mut slots = vec![]; + + // represents the index of the field + let mut field_names: Vec = vec![]; + let mut field_types: Vec = vec![]; + + let (mut field_getters, field_getter_impls) = impl_complex_enum_tuple_variant_field_getters( + ctx, + variant, + enum_name, + &variant_cls_type, + variant_ident, + &mut field_names, + &mut field_types, + )?; + + let num_fields = variant.fields.len(); + + let (variant_len, len_method_impl) = + impl_complex_enum_tuple_variant_len(ctx, &variant_cls_type, num_fields)?; + + slots.push(variant_len); + + let (variant_getitem, getitem_method_impl) = + impl_complex_enum_tuple_variant_getitem(ctx, &variant_cls, &variant_cls_type, num_fields)?; + + slots.push(variant_getitem); + + let (variant_match_args, match_args_method_impl) = + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &field_names)?; + + field_getters.push(variant_match_args); + + let cls_impl = quote! { + #[doc(hidden)] + #[allow(non_snake_case)] + impl #variant_cls { + #[allow(clippy::too_many_arguments)] + fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#field_names : #field_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { + let base_value = #enum_name::#variant_ident ( #(#field_names,)* ); + <#pyo3_path::PyClassInitializer<#enum_name> as ::std::convert::From<#enum_name>>::from(base_value).add_subclass(#variant_cls) } + #len_method_impl + + #getitem_method_impl + + #match_args_method_impl + #(#field_getter_impls)* } }; - Ok((cls_impl, field_getters)) + Ok((cls_impl, field_getters, slots)) } fn gen_complex_enum_variant_class_ident(enum_: &syn::Ident, variant: &syn::Ident) -> syn::Ident { format_ident!("{}_{}", enum_, variant) } +#[cfg(feature = "experimental-inspect")] +struct FunctionIntrospectionData<'a> { + names: &'a [&'a str], + arguments: Vec>, + returns: syn::Type, +} + +fn generate_protocol_slot( + cls: &syn::Type, + method: &mut syn::ImplItemFn, + slot: &SlotDef, + name: &str, + #[cfg(feature = "experimental-inspect")] introspection_data: FunctionIntrospectionData<'_>, + ctx: &Ctx, +) -> syn::Result { + let spec = FnSpec::parse( + &mut method.sig, + &mut Vec::new(), + PyFunctionOptions::default(), + )?; + #[cfg_attr(not(feature = "experimental-inspect"), allow(unused_mut))] + let mut def = slot.generate_type_slot(&syn::parse_quote!(#cls), &spec, name, ctx)?; + #[cfg(feature = "experimental-inspect")] + { + // We generate introspection data + let signature = FunctionSignature::from_arguments(introspection_data.arguments); + let returns = introspection_data.returns; + def.add_introspection( + introspection_data + .names + .iter() + .flat_map(|name| { + function_introspection_code( + &ctx.pyo3_path, + None, + name, + &signature, + Some("self"), + parse_quote!(-> #returns), + [], + Some(cls), + ) + }) + .collect(), + ); + } + Ok(def) +} + fn generate_default_protocol_slot( cls: &syn::Type, method: &mut syn::ImplItemFn, slot: &SlotDef, + ctx: &Ctx, ) -> syn::Result { let spec = FnSpec::parse( &mut method.sig, &mut Vec::new(), PyFunctionOptions::default(), - ) - .unwrap(); + )?; let name = spec.name.to_string(); slot.generate_type_slot( &syn::parse_quote!(#cls), &spec, - &format!("__default_{}__", name), + &format!("__default_{name}__"), + ctx, ) } fn simple_enum_default_methods<'a>( cls: &'a syn::Ident, - unit_variant_names: impl IntoIterator)>, + unit_variant_names: impl IntoIterator< + Item = ( + &'a syn::Ident, + Cow<'a, syn::Ident>, + &'a Vec<&'a syn::Attribute>, + ), + >, + ctx: &Ctx, ) -> Vec { let cls_type = syn::parse_quote!(#cls); let variant_to_attribute = |var_ident: &syn::Ident, py_ident: &syn::Ident| ConstSpec { @@ -1039,18 +1603,36 @@ fn simple_enum_default_methods<'a>( kw: syn::parse_quote! { name }, value: NameLitStr(py_ident.clone()), }), - deprecations: Default::default(), }, }; unit_variant_names .into_iter() - .map(|(var, py_name)| gen_py_const(&cls_type, &variant_to_attribute(var, &py_name))) + .map(|(var, py_name, attrs)| { + let method = gen_py_const(&cls_type, &variant_to_attribute(var, &py_name), ctx); + let associated_method_tokens = method.associated_method; + let method_def_tokens = method.method_def; + + let associated_method = quote! { + #(#attrs)* + #associated_method_tokens + }; + let method_def = quote! { + #(#attrs)* + #method_def_tokens + }; + + MethodAndMethodDef { + associated_method, + method_def, + } + }) .collect() } fn complex_enum_default_methods<'a>( cls: &'a syn::Ident, variant_names: impl IntoIterator)>, + ctx: &Ctx, ) -> Vec { let cls_type = syn::parse_quote!(#cls); let variant_to_attribute = |var_ident: &syn::Ident, py_ident: &syn::Ident| ConstSpec { @@ -1061,13 +1643,12 @@ fn complex_enum_default_methods<'a>( kw: syn::parse_quote! { name }, value: NameLitStr(py_ident.clone()), }), - deprecations: Default::default(), }, }; variant_names .into_iter() .map(|(var, py_name)| { - gen_complex_enum_variant_attr(cls, &cls_type, &variant_to_attribute(var, &py_name)) + gen_complex_enum_variant_attr(cls, &cls_type, &variant_to_attribute(var, &py_name), ctx) }) .collect() } @@ -1076,25 +1657,25 @@ pub fn gen_complex_enum_variant_attr( cls: &syn::Ident, cls_type: &syn::Type, spec: &ConstSpec, + ctx: &Ctx, ) -> MethodAndMethodDef { + let Ctx { pyo3_path, .. } = ctx; let member = &spec.rust_ident; let wrapper_ident = format_ident!("__pymethod_variant_cls_{}__", member); - let deprecations = &spec.attributes.deprecations; - let python_name = &spec.null_terminated_python_name(); + let python_name = spec.null_terminated_python_name(); let variant_cls = format_ident!("{}_{}", cls, member); let associated_method = quote! { - fn #wrapper_ident(py: _pyo3::Python<'_>) -> _pyo3::PyResult<_pyo3::PyObject> { - #deprecations - ::std::result::Result::Ok(py.get_type::<#variant_cls>().into()) + fn #wrapper_ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Py<#pyo3_path::PyAny>> { + ::std::result::Result::Ok(py.get_type::<#variant_cls>().into_any().unbind()) } }; let method_def = quote! { - _pyo3::class::PyMethodDefType::ClassAttribute({ - _pyo3::class::PyClassAttributeDef::new( + #pyo3_path::impl_::pymethods::PyMethodDefType::ClassAttribute({ + #pyo3_path::impl_::pymethods::PyClassAttributeDef::new( #python_name, - _pyo3::impl_::pymethods::PyClassAttributeFactory(#cls_type::#wrapper_ident) + #cls_type::#wrapper_ident ) }) }; @@ -1107,107 +1688,172 @@ pub fn gen_complex_enum_variant_attr( fn complex_enum_variant_new<'a>( cls: &'a syn::Ident, - variant: &'a PyClassEnumVariant<'a>, + variant: PyClassEnumVariant<'a>, + ctx: &Ctx, ) -> Result { match variant { PyClassEnumVariant::Struct(struct_variant) => { - complex_enum_struct_variant_new(cls, struct_variant) + complex_enum_struct_variant_new(cls, struct_variant, ctx) + } + PyClassEnumVariant::Tuple(tuple_variant) => { + complex_enum_tuple_variant_new(cls, tuple_variant, ctx) } } } fn complex_enum_struct_variant_new<'a>( cls: &'a syn::Ident, - variant: &'a PyClassEnumStructVariant<'a>, + variant: PyClassEnumStructVariant<'a>, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path, .. } = ctx; let variant_cls = format_ident!("{}_{}", cls, variant.ident); let variant_cls_type: syn::Type = parse_quote!(#variant_cls); let arg_py_ident: syn::Ident = parse_quote!(py); - let arg_py_type: syn::Type = parse_quote!(_pyo3::Python<'_>); + let arg_py_type: syn::Type = parse_quote!(#pyo3_path::Python<'_>); let args = { - let mut no_pyo3_attrs = vec![]; - let attrs = crate::pyfunction::PyFunctionArgPyO3Attributes::from_attrs(&mut no_pyo3_attrs)?; - let mut args = vec![ // py: Python<'_> - FnArg { + FnArg::Py(PyArg { name: &arg_py_ident, ty: &arg_py_type, - optional: None, - default: None, - py: true, - attrs: attrs.clone(), - is_varargs: false, - is_kwargs: false, - is_cancel_handle: false, - }, + }), ]; for field in &variant.fields { - args.push(FnArg { - name: field.ident, + args.push(FnArg::Regular(RegularArg { + name: Cow::Borrowed(field.ident), ty: field.ty, - optional: None, - default: None, - py: false, - attrs: attrs.clone(), - is_varargs: false, - is_kwargs: false, - is_cancel_handle: false, - }); + from_py_with: None, + default_value: None, + option_wrapped_type: None, + #[cfg(feature = "experimental-inspect")] + annotation: None, + })); + } + args + }; + + let signature = if let Some(constructor) = variant.options.constructor { + crate::pyfunction::FunctionSignature::from_arguments_and_attribute( + args, + constructor.into_signature(), + )? + } else { + crate::pyfunction::FunctionSignature::from_arguments(args) + }; + + let spec = FnSpec { + tp: crate::method::FnType::FnNew, + name: &format_ident!("__pymethod_constructor__"), + python_name: format_ident!("__new__"), + signature, + convention: crate::method::CallingConvention::TpNew, + text_signature: None, + asyncness: None, + unsafety: None, + warnings: vec![], + #[cfg(feature = "experimental-inspect")] + output: syn::ReturnType::Default, + }; + + crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec, ctx) +} + +fn complex_enum_tuple_variant_new<'a>( + cls: &'a syn::Ident, + variant: PyClassEnumTupleVariant<'a>, + ctx: &Ctx, +) -> Result { + let Ctx { pyo3_path, .. } = ctx; + + let variant_cls: Ident = format_ident!("{}_{}", cls, variant.ident); + let variant_cls_type: syn::Type = parse_quote!(#variant_cls); + + let arg_py_ident: syn::Ident = parse_quote!(py); + let arg_py_type: syn::Type = parse_quote!(#pyo3_path::Python<'_>); + + let args = { + let mut args = vec![FnArg::Py(PyArg { + name: &arg_py_ident, + ty: &arg_py_type, + })]; + + for (i, field) in variant.fields.iter().enumerate() { + args.push(FnArg::Regular(RegularArg { + name: std::borrow::Cow::Owned(format_ident!("_{}", i)), + ty: field.ty, + from_py_with: None, + default_value: None, + option_wrapped_type: None, + #[cfg(feature = "experimental-inspect")] + annotation: None, + })); } args }; - let signature = crate::pyfunction::FunctionSignature::from_arguments(args)?; + + let signature = if let Some(constructor) = variant.options.constructor { + crate::pyfunction::FunctionSignature::from_arguments_and_attribute( + args, + constructor.into_signature(), + )? + } else { + crate::pyfunction::FunctionSignature::from_arguments(args) + }; let spec = FnSpec { tp: crate::method::FnType::FnNew, name: &format_ident!("__pymethod_constructor__"), python_name: format_ident!("__new__"), signature, - output: variant_cls_type.clone(), convention: crate::method::CallingConvention::TpNew, text_signature: None, asyncness: None, unsafety: None, - deprecations: Deprecations::default(), + warnings: vec![], + #[cfg(feature = "experimental-inspect")] + output: syn::ReturnType::Default, }; - crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec) + crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec, ctx) } fn complex_enum_variant_field_getter<'a>( variant_cls_type: &'a syn::Type, field_name: &'a syn::Ident, - field_type: &'a syn::Type, field_span: Span, + ctx: &Ctx, ) -> Result { - let signature = crate::pyfunction::FunctionSignature::from_arguments(vec![])?; + let mut arg = parse_quote!(py: Python<'_>); + let py = FnArg::parse(&mut arg)?; + let signature = crate::pyfunction::FunctionSignature::from_arguments(vec![py]); - let self_type = crate::method::SelfType::TryFromPyCell(field_span); + let self_type = crate::method::SelfType::TryFromBoundRef(field_span); let spec = FnSpec { tp: crate::method::FnType::Getter(self_type.clone()), name: field_name, - python_name: field_name.clone(), + python_name: field_name.unraw(), signature, - output: field_type.clone(), convention: crate::method::CallingConvention::Noargs, text_signature: None, asyncness: None, unsafety: None, - deprecations: Deprecations::default(), + warnings: vec![], + #[cfg(feature = "experimental-inspect")] + output: parse_quote!(-> #variant_cls_type), }; let property_type = crate::pymethod::PropertyType::Function { self_type: &self_type, spec: &spec, - doc: crate::get_doc(&[], None), + doc: crate::get_doc(&[], None, ctx)?, }; - let getter = crate::pymethod::impl_py_getter_def(variant_cls_type, property_type)?; + let getter = crate::pymethod::impl_py_getter_def(variant_cls_type, property_type, ctx)?; Ok(getter) } @@ -1216,6 +1862,7 @@ fn descriptors_to_items( rename_all: Option<&RenameAllAttribute>, frozen: Option, field_options: Vec<(&syn::Field, FieldPyO3Options)>, + ctx: &Ctx, ) -> syn::Result> { let ty = syn::parse_quote!(#cls); let mut items = Vec::new(); @@ -1230,40 +1877,80 @@ fn descriptors_to_items( } if options.get.is_some() { - let getter = impl_py_getter_def( + let renaming_rule = rename_all.map(|rename_all| rename_all.value.rule); + #[cfg_attr(not(feature = "experimental-inspect"), allow(unused_mut))] + let mut getter = impl_py_getter_def( &ty, PropertyType::Descriptor { field_index, field, python_name: options.name.as_ref(), - renaming_rule: rename_all.map(|rename_all| rename_all.value.rule), + renaming_rule, }, + ctx, )?; + #[cfg(feature = "experimental-inspect")] + { + // We generate introspection data + let return_type = &field.ty; + getter.add_introspection(function_introspection_code( + &ctx.pyo3_path, + None, + &field_python_name(field, options.name.as_ref(), renaming_rule)?, + &FunctionSignature::from_arguments(vec![]), + Some("self"), + parse_quote!(-> #return_type), + vec!["property".into()], + Some(&parse_quote!(#cls)), + )); + } items.push(getter); } if let Some(set) = options.set { ensure_spanned!(frozen.is_none(), set.span() => "cannot use `#[pyo3(set)]` on a `frozen` class"); - let setter = impl_py_setter_def( + let renaming_rule = rename_all.map(|rename_all| rename_all.value.rule); + #[cfg_attr(not(feature = "experimental-inspect"), allow(unused_mut))] + let mut setter = impl_py_setter_def( &ty, PropertyType::Descriptor { field_index, field, python_name: options.name.as_ref(), - renaming_rule: rename_all.map(|rename_all| rename_all.value.rule), + renaming_rule, }, + ctx, )?; + #[cfg(feature = "experimental-inspect")] + { + // We generate introspection data + let name = field_python_name(field, options.name.as_ref(), renaming_rule)?; + setter.add_introspection(function_introspection_code( + &ctx.pyo3_path, + None, + &name, + &FunctionSignature::from_arguments(vec![FnArg::Regular(RegularArg { + name: Cow::Owned(format_ident!("value")), + ty: &field.ty, + from_py_with: None, + default_value: None, + option_wrapped_type: None, + annotation: None, + })]), + Some("self"), + syn::ReturnType::Default, + vec![format!("{name}.setter")], + Some(&parse_quote!(#cls)), + )); + } items.push(setter); }; } Ok(items) } -fn impl_pytypeinfo( - cls: &syn::Ident, - attr: &PyClassArgs, - deprecations: Option<&Deprecations>, -) -> TokenStream { +fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let cls_name = get_class_python_name(cls, attr).to_string(); let module = if let Some(ModuleAttribute { value, .. }) = &attr.options.module { @@ -1272,27 +1959,320 @@ fn impl_pytypeinfo( quote! { ::core::option::Option::None } }; - quote! { - unsafe impl _pyo3::type_object::HasPyGilRef for #cls { - type AsRefTarget = _pyo3::PyCell; - } + #[cfg(feature = "experimental-inspect")] + let type_hint = { + let type_hint = get_class_type_hint(cls, attr, ctx); + quote! { const TYPE_HINT: #pyo3_path::inspect::TypeHint = #type_hint; } + }; + #[cfg(not(feature = "experimental-inspect"))] + let type_hint = quote! {}; - unsafe impl _pyo3::type_object::PyTypeInfo for #cls { + quote! { + unsafe impl #pyo3_path::type_object::PyTypeInfo for #cls { const NAME: &'static str = #cls_name; const MODULE: ::std::option::Option<&'static str> = #module; - #[inline] - fn type_object_raw(py: _pyo3::Python<'_>) -> *mut _pyo3::ffi::PyTypeObject { - #deprecations + #type_hint - <#cls as _pyo3::impl_::pyclass::PyClassImpl>::lazy_type_object() - .get_or_init(py) + #[inline] + fn type_object_raw(py: #pyo3_path::Python<'_>) -> *mut #pyo3_path::ffi::PyTypeObject { + use #pyo3_path::prelude::PyTypeMethods; + <#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::lazy_type_object() + .get_or_try_init(py) + .unwrap_or_else(|e| #pyo3_path::impl_::pyclass::type_object_init_failed( + py, + e, + ::NAME + )) .as_type_ptr() } } } } +fn pyclass_richcmp_arms( + options: &PyClassPyO3Options, + ctx: &Ctx, +) -> std::result::Result { + let Ctx { pyo3_path, .. } = ctx; + + let eq_arms = options + .eq + .map(|eq| eq.span) + .or(options.eq_int.map(|eq_int| eq_int.span)) + .map(|span| { + quote_spanned! { span => + #pyo3_path::pyclass::CompareOp::Eq => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val == other, py) + }, + #pyo3_path::pyclass::CompareOp::Ne => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val != other, py) + }, + } + }) + .unwrap_or_default(); + + if let Some(ord) = options.ord { + ensure_spanned!(options.eq.is_some(), ord.span() => "The `ord` option requires the `eq` option."); + } + + let ord_arms = options + .ord + .map(|ord| { + quote_spanned! { ord.span() => + #pyo3_path::pyclass::CompareOp::Gt => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val > other, py) + }, + #pyo3_path::pyclass::CompareOp::Lt => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val < other, py) + }, + #pyo3_path::pyclass::CompareOp::Le => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val <= other, py) + }, + #pyo3_path::pyclass::CompareOp::Ge => { + #pyo3_path::IntoPyObjectExt::into_py_any(self_val >= other, py) + }, + } + }) + .unwrap_or_else(|| quote! { _ => ::std::result::Result::Ok(py.NotImplemented()) }); + + Ok(quote! { + #eq_arms + #ord_arms + }) +} + +fn pyclass_richcmp_simple_enum( + options: &PyClassPyO3Options, + cls: &syn::Type, + repr_type: &syn::Ident, + #[cfg(feature = "experimental-inspect")] class_name: &str, + ctx: &Ctx, +) -> Result<(Option, Option)> { + let Ctx { pyo3_path, .. } = ctx; + if let Some(eq_int) = options.eq_int { + ensure_spanned!(options.eq.is_some(), eq_int.span() => "The `eq_int` option requires the `eq` option."); + } + + if options.eq.is_none() && options.eq_int.is_none() { + return Ok((None, None)); + } + + let arms = pyclass_richcmp_arms(options, ctx)?; + + let eq = options.eq.map(|eq| { + quote_spanned! { eq.span() => + let self_val = self; + if let ::std::result::Result::Ok(other) = other.cast::() { + let other = &*other.borrow(); + return match op { + #arms + } + } + } + }); + + let eq_int = options.eq_int.map(|eq_int| { + quote_spanned! { eq_int.span() => + let self_val = self.__pyo3__int__(); + if let ::std::result::Result::Ok(other) = #pyo3_path::types::PyAnyMethods::extract::<#repr_type>(other).or_else(|_| { + other.cast::().map(|o| o.borrow().__pyo3__int__()) + }) { + return match op { + #arms + } + } + } + }); + + let mut richcmp_impl = parse_quote! { + fn __pyo3__generated____richcmp__( + &self, + py: #pyo3_path::Python, + other: &#pyo3_path::Bound<'_, #pyo3_path::PyAny>, + op: #pyo3_path::pyclass::CompareOp + ) -> #pyo3_path::PyResult<#pyo3_path::Py<#pyo3_path::PyAny>> { + #eq + + #eq_int + + ::std::result::Result::Ok(py.NotImplemented()) + } + }; + let richcmp_slot = if options.eq.is_some() { + generate_protocol_slot( + cls, + &mut richcmp_impl, + &__RICHCMP__, + "__richcmp__", + #[cfg(feature = "experimental-inspect")] + FunctionIntrospectionData { + names: &["__eq__", "__ne__"], + arguments: vec![FnArg::Regular(RegularArg { + name: Cow::Owned(format_ident!("other")), + // we need to set a type, let's pick something small, it is overridden by annotation anyway + ty: &parse_quote!(!), + from_py_with: None, + default_value: None, + option_wrapped_type: None, + annotation: Some(match (options.eq.is_some(), options.eq_int.is_some()) { + (true, true) => { + format!("{class_name} | int") + } + (true, false) => class_name.into(), + (false, true) => "int".into(), + (false, false) => unreachable!(), + }), + })], + returns: parse_quote! { ::std::primitive::bool }, + }, + ctx, + )? + } else { + generate_default_protocol_slot(cls, &mut richcmp_impl, &__RICHCMP__, ctx)? + }; + Ok((Some(richcmp_impl), Some(richcmp_slot))) +} + +fn pyclass_richcmp( + options: &PyClassPyO3Options, + cls: &syn::Type, + ctx: &Ctx, +) -> Result<(Option, Option)> { + let Ctx { pyo3_path, .. } = ctx; + if let Some(eq_int) = options.eq_int { + bail_spanned!(eq_int.span() => "`eq_int` can only be used on simple enums.") + } + + let arms = pyclass_richcmp_arms(options, ctx)?; + if options.eq.is_some() { + let mut richcmp_impl = parse_quote! { + fn __pyo3__generated____richcmp__( + &self, + py: #pyo3_path::Python, + other: &#pyo3_path::Bound<'_, #pyo3_path::PyAny>, + op: #pyo3_path::pyclass::CompareOp + ) -> #pyo3_path::PyResult<#pyo3_path::Py<#pyo3_path::PyAny>> { + let self_val = self; + if let ::std::result::Result::Ok(other) = other.cast::() { + let other = &*other.borrow(); + match op { + #arms + } + } else { + ::std::result::Result::Ok(py.NotImplemented()) + } + } + }; + let richcmp_slot = generate_protocol_slot( + cls, + &mut richcmp_impl, + &__RICHCMP__, + "__richcmp__", + #[cfg(feature = "experimental-inspect")] + FunctionIntrospectionData { + names: if options.ord.is_some() { + &["__eq__", "__ne__", "__lt__", "__le__", "__gt__", "__ge__"] + } else { + &["__eq__", "__ne__"] + }, + arguments: vec![FnArg::Regular(RegularArg { + name: Cow::Owned(format_ident!("other")), + ty: &parse_quote!(&#cls), + from_py_with: None, + default_value: None, + option_wrapped_type: None, + annotation: None, + })], + returns: parse_quote! { ::std::primitive::bool }, + }, + ctx, + )?; + Ok((Some(richcmp_impl), Some(richcmp_slot))) + } else { + Ok((None, None)) + } +} + +fn pyclass_hash( + options: &PyClassPyO3Options, + cls: &syn::Type, + ctx: &Ctx, +) -> Result<(Option, Option)> { + if options.hash.is_some() { + ensure_spanned!( + options.frozen.is_some(), options.hash.span() => "The `hash` option requires the `frozen` option."; + options.eq.is_some(), options.hash.span() => "The `hash` option requires the `eq` option."; + ); + } + match options.hash { + Some(opt) => { + let mut hash_impl = parse_quote_spanned! { opt.span() => + fn __pyo3__generated____hash__(&self) -> u64 { + let mut s = ::std::collections::hash_map::DefaultHasher::new(); + ::std::hash::Hash::hash(self, &mut s); + ::std::hash::Hasher::finish(&s) + } + }; + let hash_slot = generate_protocol_slot( + cls, + &mut hash_impl, + &__HASH__, + "__hash__", + #[cfg(feature = "experimental-inspect")] + FunctionIntrospectionData { + names: &["__hash__"], + arguments: Vec::new(), + returns: parse_quote! { ::std::primitive::u64 }, + }, + ctx, + )?; + Ok((Some(hash_impl), Some(hash_slot))) + } + None => Ok((None, None)), + } +} + +fn pyclass_class_geitem( + options: &PyClassPyO3Options, + cls: &syn::Type, + ctx: &Ctx, +) -> Result<(Option, Option)> { + let Ctx { pyo3_path, .. } = ctx; + match options.generic { + Some(_) => { + let ident = format_ident!("__class_getitem__"); + let mut class_geitem_impl: syn::ImplItemFn = { + parse_quote! { + #[classmethod] + fn #ident<'py>( + cls: &#pyo3_path::Bound<'py, #pyo3_path::types::PyType>, + key: &#pyo3_path::Bound<'py, #pyo3_path::types::PyAny> + ) -> #pyo3_path::PyResult<#pyo3_path::Bound<'py, #pyo3_path::types::PyGenericAlias>> { + #pyo3_path::types::PyGenericAlias::new(cls.py(), cls.as_any(), key) + } + } + }; + + let spec = FnSpec::parse( + &mut class_geitem_impl.sig, + &mut class_geitem_impl.attrs, + Default::default(), + )?; + + let class_geitem_method = crate::pymethod::impl_py_method_def( + cls, + &spec, + &spec.get_doc(&class_geitem_impl.attrs, ctx)?, + Some(quote!(#pyo3_path::ffi::METH_CLASS)), + ctx, + )?; + Ok((Some(class_geitem_impl), Some(class_geitem_method))) + } + None => Ok((None, None)), + } +} + /// Implements most traits used by `#[pyclass]`. /// /// Specifically, it implements traits that only depend on class name, @@ -1332,82 +2312,59 @@ impl<'a> PyClassImplsBuilder<'a> { } } - fn impl_all(&self) -> Result { - let tokens = vec![ - self.impl_pyclass(), - self.impl_extractext(), - self.impl_into_py(), - self.impl_pyclassimpl()?, - self.impl_freelist(), + fn impl_all(&self, ctx: &Ctx) -> Result { + Ok([ + self.impl_pyclass(ctx), + self.impl_into_py(ctx), + self.impl_pyclassimpl(ctx)?, + self.impl_add_to_module(ctx), + self.impl_freelist(ctx), + self.impl_introspection(ctx), ] .into_iter() - .collect(); - Ok(tokens) + .collect()) } - fn impl_pyclass(&self) -> TokenStream { + fn impl_pyclass(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; let frozen = if self.attr.options.frozen.is_some() { - quote! { _pyo3::pyclass::boolean_struct::True } + quote! { #pyo3_path::pyclass::boolean_struct::True } } else { - quote! { _pyo3::pyclass::boolean_struct::False } + quote! { #pyo3_path::pyclass::boolean_struct::False } }; quote! { - impl _pyo3::PyClass for #cls { + impl #pyo3_path::PyClass for #cls { type Frozen = #frozen; } } } - fn impl_extractext(&self) -> TokenStream { - let cls = self.cls; - if self.attr.options.frozen.is_some() { - quote! { - impl<'a, 'py> _pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls - { - type Holder = ::std::option::Option<_pyo3::PyRef<'py, #cls>>; - - #[inline] - fn extract(obj: &'py _pyo3::PyAny, holder: &'a mut Self::Holder) -> _pyo3::PyResult { - _pyo3::impl_::extract_argument::extract_pyclass_ref(obj, holder) - } - } - } - } else { - quote! { - impl<'a, 'py> _pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls - { - type Holder = ::std::option::Option<_pyo3::PyRef<'py, #cls>>; - - #[inline] - fn extract(obj: &'py _pyo3::PyAny, holder: &'a mut Self::Holder) -> _pyo3::PyResult { - _pyo3::impl_::extract_argument::extract_pyclass_ref(obj, holder) - } - } - - impl<'a, 'py> _pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a mut #cls - { - type Holder = ::std::option::Option<_pyo3::PyRefMut<'py, #cls>>; - - #[inline] - fn extract(obj: &'py _pyo3::PyAny, holder: &'a mut Self::Holder) -> _pyo3::PyResult { - _pyo3::impl_::extract_argument::extract_pyclass_ref_mut(obj, holder) - } - } - } - } - } - fn impl_into_py(&self) -> TokenStream { + fn impl_into_py(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; let attr = self.attr; // If #cls is not extended type, we allow Self->PyObject conversion if attr.options.extends.is_none() { + let output_type = if cfg!(feature = "experimental-inspect") { + quote!(const OUTPUT_TYPE: #pyo3_path::inspect::TypeHint = <#cls as #pyo3_path::PyTypeInfo>::TYPE_HINT;) + } else { + TokenStream::new() + }; quote! { - impl _pyo3::IntoPy<_pyo3::PyObject> for #cls { - fn into_py(self, py: _pyo3::Python) -> _pyo3::PyObject { - _pyo3::IntoPy::into_py(_pyo3::Py::new(py, self).unwrap(), py) + impl<'py> #pyo3_path::conversion::IntoPyObject<'py> for #cls { + type Target = Self; + type Output = #pyo3_path::Bound<'py, >::Target>; + type Error = #pyo3_path::PyErr; + #output_type + + fn into_pyobject(self, py: #pyo3_path::Python<'py>) -> ::std::result::Result< + ::Output, + ::Error, + > { + #pyo3_path::Bound::new(py, self) } } } @@ -1415,17 +2372,22 @@ impl<'a> PyClassImplsBuilder<'a> { quote! {} } } - fn impl_pyclassimpl(&self) -> Result { + fn impl_pyclassimpl(&self, ctx: &Ctx) -> Result { + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; - let doc = self.doc.as_ref().map_or(quote! {"\0"}, |doc| quote! {#doc}); + let doc = self + .doc + .as_ref() + .map_or(c"".to_token_stream(), PythonDoc::to_token_stream); let is_basetype = self.attr.options.subclass.is_some(); let base = match &self.attr.options.extends { Some(extends_attr) => extends_attr.value.clone(), - None => parse_quote! { _pyo3::PyAny }, + None => parse_quote! { #pyo3_path::PyAny }, }; let is_subclass = self.attr.options.extends.is_some(); let is_mapping: bool = self.attr.options.mapping.is_some(); let is_sequence: bool = self.attr.options.sequence.is_some(); + let is_immutable_type = self.attr.options.immutable_type.is_some(); ensure_spanned!( !(is_mapping && is_sequence), @@ -1434,8 +2396,8 @@ impl<'a> PyClassImplsBuilder<'a> { let dict_offset = if self.attr.options.dict.is_some() { quote! { - fn dict_offset() -> ::std::option::Option<_pyo3::ffi::Py_ssize_t> { - ::std::option::Option::Some(_pyo3::impl_::pyclass::dict_offset::()) + fn dict_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> { + ::std::option::Option::Some(#pyo3_path::impl_::pyclass::dict_offset::()) } } } else { @@ -1445,8 +2407,8 @@ impl<'a> PyClassImplsBuilder<'a> { // insert space for weak ref let weaklist_offset = if self.attr.options.weakref.is_some() { quote! { - fn weaklist_offset() -> ::std::option::Option<_pyo3::ffi::Py_ssize_t> { - ::std::option::Option::Some(_pyo3::impl_::pyclass::weaklist_offset::()) + fn weaklist_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> { + ::std::option::Option::Some(#pyo3_path::impl_::pyclass::weaklist_offset::()) } } } else { @@ -1454,9 +2416,9 @@ impl<'a> PyClassImplsBuilder<'a> { }; let thread_checker = if self.attr.options.unsendable.is_some() { - quote! { _pyo3::impl_::pyclass::ThreadCheckerImpl } + quote! { #pyo3_path::impl_::pyclass::ThreadCheckerImpl } } else { - quote! { _pyo3::impl_::pyclass::SendablePyClass<#cls> } + quote! { #pyo3_path::impl_::pyclass::SendablePyClass<#cls> } }; let (pymethods_items, inventory, inventory_class) = match self.methods_type { @@ -1471,13 +2433,13 @@ impl<'a> PyClassImplsBuilder<'a> { quote! { ::std::boxed::Box::new( ::std::iter::Iterator::map( - _pyo3::inventory::iter::<::Inventory>(), - _pyo3::impl_::pyclass::PyClassInventory::items + #pyo3_path::inventory::iter::<::Inventory>(), + #pyo3_path::impl_::pyclass::PyClassInventory::items ) ) }, Some(quote! { type Inventory = #inventory_class_name; }), - Some(define_inventory_class(&inventory_class_name)), + Some(define_inventory_class(&inventory_class_name, ctx)), ) } }; @@ -1494,7 +2456,7 @@ impl<'a> PyClassImplsBuilder<'a> { let default_method_defs = self.default_methods.iter().map(|meth| &meth.method_def); let default_slot_defs = self.default_slots.iter().map(|slot| &slot.slot_def); - let freelist_slots = self.freelist_slots(); + let freelist_slots = self.freelist_slots(ctx); let class_mutability = if self.attr.options.frozen.is_some() { quote! { @@ -1509,41 +2471,98 @@ impl<'a> PyClassImplsBuilder<'a> { let cls = self.cls; let attr = self.attr; let dict = if attr.options.dict.is_some() { - quote! { _pyo3::impl_::pyclass::PyClassDictSlot } + quote! { #pyo3_path::impl_::pyclass::PyClassDictSlot } } else { - quote! { _pyo3::impl_::pyclass::PyClassDummySlot } + quote! { #pyo3_path::impl_::pyclass::PyClassDummySlot } }; // insert space for weak ref let weakref = if attr.options.weakref.is_some() { - quote! { _pyo3::impl_::pyclass::PyClassWeakRefSlot } + quote! { #pyo3_path::impl_::pyclass::PyClassWeakRefSlot } } else { - quote! { _pyo3::impl_::pyclass::PyClassDummySlot } + quote! { #pyo3_path::impl_::pyclass::PyClassDummySlot } }; let base_nativetype = if attr.options.extends.is_some() { - quote! { ::BaseNativeType } + quote! { ::BaseNativeType } } else { - quote! { _pyo3::PyAny } + quote! { #pyo3_path::PyAny } + }; + + let pyclass_base_type_impl = attr.options.subclass.map(|subclass| { + quote_spanned! { subclass.span() => + impl #pyo3_path::impl_::pyclass::PyClassBaseType for #cls { + type LayoutAsBase = #pyo3_path::impl_::pycell::PyClassObject; + type BaseNativeType = ::BaseNativeType; + type Initializer = #pyo3_path::pyclass_init::PyClassInitializer; + type PyClassMutability = ::PyClassMutability; + } + } + }); + + let assertions = if attr.options.unsendable.is_some() { + TokenStream::new() + } else { + let assert = quote_spanned! { cls.span() => #pyo3_path::impl_::pyclass::assert_pyclass_sync::<#cls>(); }; + quote! { + const _: () = { + #assert + }; + } + }; + + let extract_pyclass_with_clone = if let Some(from_py_object) = + self.attr.options.from_py_object + { + let input_type = if cfg!(feature = "experimental-inspect") { + quote!(const INPUT_TYPE: #pyo3_path::inspect::TypeHint = <#cls as #pyo3_path::PyTypeInfo>::TYPE_HINT;) + } else { + TokenStream::new() + }; + quote_spanned! { from_py_object.span() => + impl<'a, 'py> #pyo3_path::FromPyObject<'a, 'py> for #cls + where + Self: ::std::clone::Clone, + { + type Error = #pyo3_path::pyclass::PyClassGuardError<'a, 'py>; + + #input_type + + fn extract(obj: #pyo3_path::Borrowed<'a, 'py, #pyo3_path::PyAny>) -> ::std::result::Result { + ::std::result::Result::Ok(::std::clone::Clone::clone(&*obj.extract::<#pyo3_path::PyClassGuard<'_, #cls>>()?)) + } + } + } + } else if self.attr.options.skip_from_py_object.is_none() { + quote!( impl #pyo3_path::impl_::pyclass::ExtractPyClassWithClone for #cls {} ) + } else { + TokenStream::new() }; Ok(quote! { - impl _pyo3::impl_::pyclass::PyClassImpl for #cls { + #extract_pyclass_with_clone + + #assertions + + #pyclass_base_type_impl + + impl #pyo3_path::impl_::pyclass::PyClassImpl for #cls { const IS_BASETYPE: bool = #is_basetype; const IS_SUBCLASS: bool = #is_subclass; const IS_MAPPING: bool = #is_mapping; const IS_SEQUENCE: bool = #is_sequence; + const IS_IMMUTABLE_TYPE: bool = #is_immutable_type; type BaseType = #base; type ThreadChecker = #thread_checker; #inventory - type PyClassMutability = <<#base as _pyo3::impl_::pyclass::PyClassBaseType>::PyClassMutability as _pyo3::impl_::pycell::PyClassMutability>::#class_mutability; + type PyClassMutability = <<#base as #pyo3_path::impl_::pyclass::PyClassBaseType>::PyClassMutability as #pyo3_path::impl_::pycell::PyClassMutability>::#class_mutability; type Dict = #dict; type WeakRef = #weakref; type BaseNativeType = #base_nativetype; - fn items_iter() -> _pyo3::impl_::pyclass::PyClassItemsIter { - use _pyo3::impl_::pyclass::*; + fn items_iter() -> #pyo3_path::impl_::pyclass::PyClassItemsIter { + use #pyo3_path::impl_::pyclass::*; let collector = PyClassImplCollector::::new(); static INTRINSIC_ITEMS: PyClassItems = PyClassItems { methods: &[#(#default_method_defs),*], @@ -1552,21 +2571,26 @@ impl<'a> PyClassImplsBuilder<'a> { PyClassItemsIter::new(&INTRINSIC_ITEMS, #pymethods_items) } - fn doc(py: _pyo3::Python<'_>) -> _pyo3::PyResult<&'static ::std::ffi::CStr> { - use _pyo3::impl_::pyclass::*; - static DOC: _pyo3::sync::GILOnceCell<::std::borrow::Cow<'static, ::std::ffi::CStr>> = _pyo3::sync::GILOnceCell::new(); - DOC.get_or_try_init(py, || { - let collector = PyClassImplCollector::::new(); - build_pyclass_doc(<#cls as _pyo3::PyTypeInfo>::NAME, #doc, collector.new_text_signature()) - }).map(::std::ops::Deref::deref) - } + const RAW_DOC: &'static ::std::ffi::CStr = #doc; + + const DOC: &'static ::std::ffi::CStr = { + use #pyo3_path::impl_ as impl_; + use impl_::pyclass::Probe as _; + const DOC_PIECES: &'static [&'static [u8]] = impl_::pyclass::doc::PyClassDocGenerator::< + #cls, + { impl_::pyclass::HasNewTextSignature::<#cls>::VALUE } + >::DOC_PIECES; + const LEN: usize = impl_::concat::combined_len(DOC_PIECES); + const DOC: &'static [u8] = &impl_::concat::combine_to_array::(DOC_PIECES); + impl_::pyclass::doc::doc_bytes_as_cstr(DOC) + }; #dict_offset #weaklist_offset - fn lazy_type_object() -> &'static _pyo3::impl_::pyclass::LazyTypeObject { - use _pyo3::impl_::pyclass::LazyTypeObject; + fn lazy_type_object() -> &'static #pyo3_path::impl_::pyclass::LazyTypeObject { + use #pyo3_path::impl_::pyclass::LazyTypeObject; static TYPE_OBJECT: LazyTypeObject<#cls> = LazyTypeObject::new(); &TYPE_OBJECT } @@ -1582,44 +2606,51 @@ impl<'a> PyClassImplsBuilder<'a> { }) } - fn impl_freelist(&self) -> TokenStream { + fn impl_add_to_module(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + let cls = self.cls; + quote! { + impl #cls { + #[doc(hidden)] + pub const _PYO3_DEF: #pyo3_path::impl_::pymodule::AddClassToModule = #pyo3_path::impl_::pymodule::AddClassToModule::new(); + } + } + } + + fn impl_freelist(&self, ctx: &Ctx) -> TokenStream { let cls = self.cls; + let Ctx { pyo3_path, .. } = ctx; - self.attr.options.freelist.as_ref().map_or(quote!{}, |freelist| { + self.attr.options.freelist.as_ref().map_or(quote! {}, |freelist| { let freelist = &freelist.value; quote! { - impl _pyo3::impl_::pyclass::PyClassWithFreeList for #cls { + impl #pyo3_path::impl_::pyclass::PyClassWithFreeList for #cls { #[inline] - fn get_free_list(py: _pyo3::Python<'_>) -> &mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> { - static mut FREELIST: *mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> = 0 as *mut _; - unsafe { - if FREELIST.is_null() { - FREELIST = ::std::boxed::Box::into_raw(::std::boxed::Box::new( - _pyo3::impl_::freelist::FreeList::with_capacity(#freelist))); - } - &mut *FREELIST - } + fn get_free_list(py: #pyo3_path::Python<'_>) -> &'static ::std::sync::Mutex<#pyo3_path::impl_::freelist::PyObjectFreeList> { + static FREELIST: #pyo3_path::sync::PyOnceLock<::std::sync::Mutex<#pyo3_path::impl_::freelist::PyObjectFreeList>> = #pyo3_path::sync::PyOnceLock::new(); + &FREELIST.get_or_init(py, || ::std::sync::Mutex::new(#pyo3_path::impl_::freelist::PyObjectFreeList::with_capacity(#freelist))) } } } }) } - fn freelist_slots(&self) -> Vec { + fn freelist_slots(&self, ctx: &Ctx) -> Vec { + let Ctx { pyo3_path, .. } = ctx; let cls = self.cls; if self.attr.options.freelist.is_some() { vec![ quote! { - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::Py_tp_alloc, - pfunc: _pyo3::impl_::pyclass::alloc_with_freelist::<#cls> as *mut _, + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_alloc, + pfunc: #pyo3_path::impl_::pyclass::alloc_with_freelist::<#cls> as *mut _, } }, quote! { - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::Py_tp_free, - pfunc: _pyo3::impl_::pyclass::free_with_freelist::<#cls> as *mut _, + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_free, + pfunc: #pyo3_path::impl_::pyclass::free_with_freelist::<#cls> as *mut _, } }, ] @@ -1627,27 +2658,79 @@ impl<'a> PyClassImplsBuilder<'a> { Vec::new() } } + + #[cfg(feature = "experimental-inspect")] + fn impl_introspection(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + let name = get_class_python_name(self.cls, self.attr).to_string(); + let ident = self.cls; + let static_introspection = class_introspection_code(pyo3_path, ident, &name); + let introspection_id = introspection_id_const(); + quote! { + #static_introspection + impl #ident { + #introspection_id + } + } + } + + #[cfg(not(feature = "experimental-inspect"))] + fn impl_introspection(&self, _ctx: &Ctx) -> TokenStream { + quote! {} + } } -fn define_inventory_class(inventory_class_name: &syn::Ident) -> TokenStream { +fn define_inventory_class(inventory_class_name: &syn::Ident, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; quote! { #[doc(hidden)] pub struct #inventory_class_name { - items: _pyo3::impl_::pyclass::PyClassItems, + items: #pyo3_path::impl_::pyclass::PyClassItems, } impl #inventory_class_name { - pub const fn new(items: _pyo3::impl_::pyclass::PyClassItems) -> Self { + pub const fn new(items: #pyo3_path::impl_::pyclass::PyClassItems) -> Self { Self { items } } } - impl _pyo3::impl_::pyclass::PyClassInventory for #inventory_class_name { - fn items(&self) -> &_pyo3::impl_::pyclass::PyClassItems { + impl #pyo3_path::impl_::pyclass::PyClassInventory for #inventory_class_name { + fn items(&self) -> &#pyo3_path::impl_::pyclass::PyClassItems { &self.items } } - _pyo3::inventory::collect!(#inventory_class_name); + #pyo3_path::inventory::collect!(#inventory_class_name); + } +} + +fn generate_cfg_check(variants: &[PyClassEnumUnitVariant<'_>], cls: &syn::Ident) -> TokenStream { + if variants.is_empty() { + return quote! {}; + } + + let mut conditions = Vec::new(); + + for variant in variants { + let cfg_attrs = &variant.cfg_attrs; + + if cfg_attrs.is_empty() { + // There's at least one variant of the enum without cfg attributes, + // so the check is not necessary + return quote! {}; + } + + for attr in cfg_attrs { + if let syn::Meta::List(meta) = &attr.meta { + let cfg_tokens = &meta.tokens; + conditions.push(quote! { not(#cfg_tokens) }); + } + } + } + + quote_spanned! { + cls.span() => + #[cfg(all(#(#conditions),*))] + ::core::compile_error!(concat!("#[pyclass] can't be used on enums without any variants - all variants of enum `", stringify!(#cls), "` have been configured out by cfg attributes")); } } diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index b265a34d39f..1333e6d2bcd 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -1,24 +1,28 @@ +use crate::attributes::KeywordAttribute; +use crate::combine_errors::CombineErrors; +#[cfg(feature = "experimental-inspect")] +use crate::introspection::{function_introspection_code, introspection_id_const}; +use crate::utils::Ctx; use crate::{ attributes::{ self, get_pyo3_options, take_attributes, take_pyo3_options, CrateAttribute, FromPyWithAttribute, NameAttribute, TextSignatureAttribute, }, - deprecations::Deprecations, method::{self, CallingConvention, FnArg}, pymethod::check_generic, - utils::get_pyo3_crate, -}; -use proc_macro2::TokenStream; -use quote::{format_ident, quote}; -use syn::{ext::IdentExt, spanned::Spanned, Result}; -use syn::{ - parse::{Parse, ParseStream}, - token::Comma, }; +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote, ToTokens}; +use std::cmp::PartialEq; +use std::ffi::CString; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::LitCStr; +use syn::{ext::IdentExt, spanned::Spanned, LitStr, Path, Result, Token}; mod signature; -pub use self::signature::{FunctionSignature, SignatureAttribute}; +pub use self::signature::{ConstructorAttribute, FunctionSignature, SignatureAttribute}; #[derive(Clone, Debug)] pub struct PyFunctionArgPyO3Attributes { @@ -84,6 +88,149 @@ impl PyFunctionArgPyO3Attributes { } } +type PyFunctionWarningMessageAttribute = KeywordAttribute; +type PyFunctionWarningCategoryAttribute = KeywordAttribute; + +pub struct PyFunctionWarningAttribute { + pub message: PyFunctionWarningMessageAttribute, + pub category: Option, + pub span: Span, +} + +#[derive(PartialEq, Clone)] +pub enum PyFunctionWarningCategory { + Path(Path), + UserWarning, + DeprecationWarning, // TODO: unused for now, intended for pyo3(deprecated) special-case +} + +#[derive(Clone)] +pub struct PyFunctionWarning { + pub message: LitStr, + pub category: PyFunctionWarningCategory, + pub span: Span, +} + +impl From for PyFunctionWarning { + fn from(value: PyFunctionWarningAttribute) -> Self { + Self { + message: value.message.value, + category: value + .category + .map_or(PyFunctionWarningCategory::UserWarning, |cat| { + PyFunctionWarningCategory::Path(cat.value) + }), + span: value.span, + } + } +} + +pub trait WarningFactory { + fn build_py_warning(&self, ctx: &Ctx) -> TokenStream; + fn span(&self) -> Span; +} + +impl WarningFactory for PyFunctionWarning { + fn build_py_warning(&self, ctx: &Ctx) -> TokenStream { + let message = &self.message.value(); + let c_message = LitCStr::new( + &CString::new(message.clone()).unwrap(), + Spanned::span(&message), + ); + let pyo3_path = &ctx.pyo3_path; + let category = match &self.category { + PyFunctionWarningCategory::Path(path) => quote! {#path}, + PyFunctionWarningCategory::UserWarning => { + quote! {#pyo3_path::exceptions::PyUserWarning} + } + PyFunctionWarningCategory::DeprecationWarning => { + quote! {#pyo3_path::exceptions::PyDeprecationWarning} + } + }; + quote! { + #pyo3_path::PyErr::warn(py, &<#category as #pyo3_path::PyTypeInfo>::type_object(py), #c_message, 1)?; + } + } + + fn span(&self) -> Span { + self.span + } +} + +impl WarningFactory for Vec { + fn build_py_warning(&self, ctx: &Ctx) -> TokenStream { + let warnings = self.iter().map(|warning| warning.build_py_warning(ctx)); + + quote! { + #(#warnings)* + } + } + + fn span(&self) -> Span { + self.iter() + .map(|val| val.span()) + .reduce(|acc, span| acc.join(span).unwrap_or(acc)) + .unwrap() + } +} + +impl Parse for PyFunctionWarningAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let mut message: Option = None; + let mut category: Option = None; + + let span = input.parse::()?.span(); + + let content; + syn::parenthesized!(content in input); + + while !content.is_empty() { + let lookahead = content.lookahead1(); + + if lookahead.peek(attributes::kw::message) { + message = content + .parse::() + .map(Some)?; + } else if lookahead.peek(attributes::kw::category) { + category = content + .parse::() + .map(Some)?; + } else { + return Err(lookahead.error()); + } + + if content.peek(Token![,]) { + content.parse::()?; + } + } + + Ok(PyFunctionWarningAttribute { + message: message.ok_or(syn::Error::new( + content.span(), + "missing `message` in `warn` attribute", + ))?, + category, + span, + }) + } +} + +impl ToTokens for PyFunctionWarningAttribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + let message_tokens = self.message.to_token_stream(); + let category_tokens = self + .category + .as_ref() + .map_or(quote! {}, |cat| cat.to_token_stream()); + + let token_stream = quote! { + warn(#message_tokens, #category_tokens) + }; + + tokens.extend(token_stream); + } +} + #[derive(Default)] pub struct PyFunctionOptions { pub pass_module: Option, @@ -91,30 +238,15 @@ pub struct PyFunctionOptions { pub signature: Option, pub text_signature: Option, pub krate: Option, + pub warnings: Vec, } impl Parse for PyFunctionOptions { fn parse(input: ParseStream<'_>) -> Result { let mut options = PyFunctionOptions::default(); - while !input.is_empty() { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::name) - || lookahead.peek(attributes::kw::pass_module) - || lookahead.peek(attributes::kw::signature) - || lookahead.peek(attributes::kw::text_signature) - { - options.add_attributes(std::iter::once(input.parse()?))?; - if !input.is_empty() { - let _: Comma = input.parse()?; - } - } else if lookahead.peek(syn::Token![crate]) { - // TODO needs duplicate check? - options.krate = Some(input.parse()?); - } else { - return Err(lookahead.error()); - } - } + let attrs = Punctuated::::parse_terminated(input)?; + options.add_attributes(attrs)?; Ok(options) } @@ -126,6 +258,7 @@ pub enum PyFunctionOption { Signature(SignatureAttribute), TextSignature(TextSignatureAttribute), Crate(CrateAttribute), + Warning(PyFunctionWarningAttribute), } impl Parse for PyFunctionOption { @@ -141,6 +274,8 @@ impl Parse for PyFunctionOption { input.parse().map(PyFunctionOption::TextSignature) } else if lookahead.peek(syn::Token![crate]) { input.parse().map(PyFunctionOption::Crate) + } else if lookahead.peek(attributes::kw::warn) { + input.parse().map(PyFunctionOption::Warning) } else { Err(lookahead.error()) } @@ -176,6 +311,9 @@ impl PyFunctionOptions { PyFunctionOption::Signature(signature) => set_option!(signature), PyFunctionOption::TextSignature(text_signature) => set_option!(text_signature), PyFunctionOption::Crate(krate) => set_option!(krate), + PyFunctionOption::Warning(warning) => { + self.warnings.push(warning.into()); + } } } Ok(()) @@ -203,9 +341,16 @@ pub fn impl_wrap_pyfunction( signature, text_signature, krate, + warnings, } = options; - let python_name = name.map_or_else(|| func.sig.ident.unraw(), |name| name.value.0); + let ctx = &Ctx::new(&krate, Some(&func.sig)); + let Ctx { pyo3_path, .. } = &ctx; + + let python_name = name + .as_ref() + .map_or_else(|| &func.sig.ident, |name| &name.value.0) + .unraw(); let tp = if pass_module.is_some() { let span = match func.sig.inputs.first() { @@ -229,15 +374,34 @@ pub fn impl_wrap_pyfunction( 0 }) .map(FnArg::parse) - .collect::>>()?; + .try_combine_syn_errors()?; let signature = if let Some(signature) = signature { FunctionSignature::from_arguments_and_attribute(arguments, signature)? } else { - FunctionSignature::from_arguments(arguments)? + FunctionSignature::from_arguments(arguments) }; - let ty = method::get_return_info(&func.sig.output); + let vis = &func.vis; + let name = &func.sig.ident; + + #[cfg(feature = "experimental-inspect")] + let introspection = function_introspection_code( + pyo3_path, + Some(name), + &name.to_string(), + &signature, + None, + func.sig.output.clone(), + [] as [String; 0], + None, + ); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; + #[cfg(feature = "experimental-inspect")] + let introspection_id = introspection_id_const(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_id = quote! {}; let spec = method::FnSpec { tp, @@ -245,45 +409,50 @@ pub fn impl_wrap_pyfunction( convention: CallingConvention::from_signature(&signature), python_name, signature, - output: ty, text_signature, asyncness: func.sig.asyncness, unsafety: func.sig.unsafety, - deprecations: Deprecations::new(), + warnings, + #[cfg(feature = "experimental-inspect")] + output: func.sig.output.clone(), }; - let krate = get_pyo3_crate(&krate); - - let vis = &func.vis; - let name = &func.sig.ident; - let wrapper_ident = format_ident!("__pyfunction_{}", spec.name); - let wrapper = spec.get_wrapper_function(&wrapper_ident, None)?; - let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs)); + if spec.asyncness.is_some() { + ensure_spanned!( + cfg!(feature = "experimental-async"), + spec.asyncness.span() => "async functions are only supported with the `experimental-async` feature" + ); + } + let wrapper = spec.get_wrapper_function(&wrapper_ident, None, ctx)?; + let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs, ctx)?, ctx); let wrapped_pyfunction = quote! { - // Create a module with the same name as the `#[pyfunction]` - this way `use ` // will actually bring both the module and the function into scope. #[doc(hidden)] #vis mod #name { pub(crate) struct MakeDef; - pub const DEF: #krate::impl_::pyfunction::PyMethodDef = MakeDef::DEF; + pub static _PYO3_DEF: #pyo3_path::impl_::pyfunction::PyFunctionDef = MakeDef::_PYO3_DEF; + #introspection_id } - // Generate the definition inside an anonymous function in the same scope as the original function - + // Generate the definition in the same scope as the original function - // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pyfunction] is // inside a function body) - const _: () = { - use #krate as _pyo3; - impl #name::MakeDef { - const DEF: #krate::impl_::pyfunction::PyMethodDef = #methoddef; - } + #[allow(unknown_lints, non_local_definitions)] + impl #name::MakeDef { + // We're using this to initialize a static, so it's fine. + #[allow(clippy::declare_interior_mutable_const)] + const _PYO3_DEF: #pyo3_path::impl_::pyfunction::PyFunctionDef = + #pyo3_path::impl_::pyfunction::PyFunctionDef::from_method_def(#methoddef); + } - #[allow(non_snake_case)] - #wrapper - }; + #[allow(non_snake_case)] + #wrapper + + #introspection }; Ok(wrapped_pyfunction) } diff --git a/pyo3-macros-backend/src/pyfunction/signature.rs b/pyo3-macros-backend/src/pyfunction/signature.rs index baf01285658..306c42f791a 100644 --- a/pyo3-macros-backend/src/pyfunction/signature.rs +++ b/pyo3-macros-backend/src/pyfunction/signature.rs @@ -1,3 +1,7 @@ +use crate::{ + attributes::{kw, KeywordAttribute}, + method::{FnArg, RegularArg}, +}; use proc_macro2::{Span, TokenStream}; use quote::ToTokens; use syn::{ @@ -8,63 +12,74 @@ use syn::{ Token, }; -use crate::{ - attributes::{kw, KeywordAttribute}, - method::FnArg, -}; - +#[derive(Clone)] pub struct Signature { paren_token: syn::token::Paren, pub items: Punctuated, + pub returns: Option<(Token![->], PyTypeAnnotation)>, } impl Parse for Signature { fn parse(input: ParseStream<'_>) -> syn::Result { let content; let paren_token = syn::parenthesized!(content in input); - let items = content.parse_terminated(SignatureItem::parse, Token![,])?; - - Ok(Signature { paren_token, items }) + let returns = if input.peek(Token![->]) { + Some((input.parse()?, input.parse()?)) + } else { + None + }; + Ok(Signature { + paren_token, + items, + returns, + }) } } impl ToTokens for Signature { fn to_tokens(&self, tokens: &mut TokenStream) { self.paren_token - .surround(tokens, |tokens| self.items.to_tokens(tokens)) + .surround(tokens, |tokens| self.items.to_tokens(tokens)); + if let Some((arrow, returns)) = &self.returns { + arrow.to_tokens(tokens); + returns.to_tokens(tokens); + } } } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemArgument { pub ident: syn::Ident, + pub colon_and_annotation: Option<(Token![:], PyTypeAnnotation)>, pub eq_and_default: Option<(Token![=], syn::Expr)>, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemPosargsSep { pub slash: Token![/], } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemVarargsSep { pub asterisk: Token![*], } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemVarargs { pub sep: SignatureItemVarargsSep, pub ident: syn::Ident, + pub colon_and_annotation: Option<(Token![:], PyTypeAnnotation)>, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct SignatureItemKwargs { pub asterisks: (Token![*], Token![*]), pub ident: syn::Ident, + pub colon_and_annotation: Option<(Token![:], PyTypeAnnotation)>, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum SignatureItem { Argument(Box), PosargsSep(SignatureItemPosargsSep), @@ -87,6 +102,11 @@ impl Parse for SignatureItem { Ok(SignatureItem::Varargs(SignatureItemVarargs { sep, ident: input.parse()?, + colon_and_annotation: if input.peek(Token![:]) { + Some((input.parse()?, input.parse()?)) + } else { + None + }, })) } } @@ -114,6 +134,11 @@ impl Parse for SignatureItemArgument { fn parse(input: ParseStream<'_>) -> syn::Result { Ok(Self { ident: input.parse()?, + colon_and_annotation: if input.peek(Token![:]) { + Some((input.parse()?, input.parse()?)) + } else { + None + }, eq_and_default: if input.peek(Token![=]) { Some((input.parse()?, input.parse()?)) } else { @@ -126,6 +151,10 @@ impl Parse for SignatureItemArgument { impl ToTokens for SignatureItemArgument { fn to_tokens(&self, tokens: &mut TokenStream) { self.ident.to_tokens(tokens); + if let Some((colon, annotation)) = &self.colon_and_annotation { + colon.to_tokens(tokens); + annotation.to_tokens(tokens); + } if let Some((eq, default)) = &self.eq_and_default { eq.to_tokens(tokens); default.to_tokens(tokens); @@ -152,6 +181,11 @@ impl Parse for SignatureItemVarargs { Ok(Self { sep: input.parse()?, ident: input.parse()?, + colon_and_annotation: if input.peek(Token![:]) { + Some((input.parse()?, input.parse()?)) + } else { + None + }, }) } } @@ -168,6 +202,11 @@ impl Parse for SignatureItemKwargs { Ok(Self { asterisks: (input.parse()?, input.parse()?), ident: input.parse()?, + colon_and_annotation: if input.peek(Token![:]) { + Some((input.parse()?, input.parse()?)) + } else { + None + }, }) } } @@ -194,9 +233,40 @@ impl ToTokens for SignatureItemPosargsSep { } } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PyTypeAnnotation(syn::LitStr); + +impl Parse for PyTypeAnnotation { + fn parse(input: ParseStream<'_>) -> syn::Result { + Ok(Self(input.parse()?)) + } +} + +impl ToTokens for PyTypeAnnotation { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0.to_tokens(tokens); + } +} + +impl PyTypeAnnotation { + pub fn to_python(&self) -> String { + self.0.value() + } +} + pub type SignatureAttribute = KeywordAttribute; +pub type ConstructorAttribute = KeywordAttribute; -#[derive(Default)] +impl ConstructorAttribute { + pub fn into_signature(self) -> SignatureAttribute { + SignatureAttribute { + kw: kw::signature(self.kw.span), + value: self.value, + } + } +} + +#[derive(Default, Clone)] pub struct PythonSignature { pub positional_parameters: Vec, pub positional_only_parameters: usize, @@ -216,6 +286,7 @@ impl PythonSignature { } } +#[derive(Clone)] pub struct FunctionSignature<'a> { pub arguments: Vec>, pub python_signature: PythonSignature, @@ -351,42 +422,52 @@ impl<'a> FunctionSignature<'a> { let mut next_non_py_argument_checked = |name: &syn::Ident| { for fn_arg in args_iter.by_ref() { - if fn_arg.py { - // If the user incorrectly tried to include py: Python in the - // signature, give a useful error as a hint. - ensure_spanned!( - name != fn_arg.name, - name.span() => "arguments of type `Python` must not be part of the signature" - ); - // Otherwise try next argument. - continue; - } - if fn_arg.is_cancel_handle { - // If the user incorrectly tried to include cancel: CoroutineCancel in the - // signature, give a useful error as a hint. - ensure_spanned!( - name != fn_arg.name, - name.span() => "`cancel_handle` argument must not be part of the signature" - ); - // Otherwise try next argument. - continue; + match fn_arg { + FnArg::Py(..) => { + // If the user incorrectly tried to include py: Python in the + // signature, give a useful error as a hint. + ensure_spanned!( + name != fn_arg.name(), + name.span() => "arguments of type `Python` must not be part of the signature" + ); + // Otherwise try next argument. + continue; + } + FnArg::CancelHandle(..) => { + // If the user incorrectly tried to include cancel: CoroutineCancel in the + // signature, give a useful error as a hint. + ensure_spanned!( + name != fn_arg.name(), + name.span() => "`cancel_handle` argument must not be part of the signature" + ); + // Otherwise try next argument. + continue; + } + _ => { + ensure_spanned!( + name == fn_arg.name(), + name.span() => format!( + "expected argument from function definition `{}` but got argument `{}`", + fn_arg.name().unraw(), + name.unraw(), + ) + ); + return Ok(fn_arg); + } } - - ensure_spanned!( - name == fn_arg.name, - name.span() => format!( - "expected argument from function definition `{}` but got argument `{}`", - fn_arg.name.unraw(), - name.unraw(), - ) - ); - return Ok(fn_arg); } bail_spanned!( name.span() => "signature entry does not have a corresponding function argument" ) }; + if let Some(returns) = &attribute.value.returns { + ensure_spanned!( + cfg!(feature = "experimental-inspect"), + returns.1.span() => "Return type annotation in the signature is only supported with the `experimental-inspect` feature" + ); + } + for item in &attribute.value.items { match item { SignatureItem::Argument(arg) => { @@ -397,8 +478,25 @@ impl<'a> FunctionSignature<'a> { arg.eq_and_default.is_none(), arg.span(), )?; + let FnArg::Regular(fn_arg) = fn_arg else { + unreachable!( + "`Python` and `CancelHandle` are already handled above and `*args`/`**kwargs` are \ + parsed and transformed below. Because the have to come last and are only allowed \ + once, this has to be a regular argument." + ); + }; if let Some((_, default)) = &arg.eq_and_default { - fn_arg.default = Some(default.clone()); + fn_arg.default_value = Some(default.clone()); + } + if let Some((_, annotation)) = &arg.colon_and_annotation { + ensure_spanned!( + cfg!(feature = "experimental-inspect"), + annotation.span() => "Type annotations in the signature is only supported with the `experimental-inspect` feature" + ); + #[cfg(feature = "experimental-inspect")] + { + fn_arg.annotation = Some(annotation.to_python()); + } } } SignatureItem::VarargsSep(sep) => { @@ -406,13 +504,47 @@ impl<'a> FunctionSignature<'a> { } SignatureItem::Varargs(varargs) => { let fn_arg = next_non_py_argument_checked(&varargs.ident)?; - fn_arg.is_varargs = true; + fn_arg.to_varargs_mut()?; parse_state.add_varargs(&mut python_signature, varargs)?; + if let Some((_, annotation)) = &varargs.colon_and_annotation { + ensure_spanned!( + cfg!(feature = "experimental-inspect"), + annotation.span() => "Type annotations in the signature is only supported with the `experimental-inspect` feature" + ); + #[cfg(feature = "experimental-inspect")] + { + let FnArg::VarArgs(fn_arg) = fn_arg else { + unreachable!( + "`Python` and `CancelHandle` are already handled above and `*args`/`**kwargs` are \ + parsed and transformed below. Because the have to come last and are only allowed \ + once, this has to be a regular argument." + ); + }; + fn_arg.annotation = Some(annotation.to_python()); + } + } } SignatureItem::Kwargs(kwargs) => { let fn_arg = next_non_py_argument_checked(&kwargs.ident)?; - fn_arg.is_kwargs = true; + fn_arg.to_kwargs_mut()?; parse_state.add_kwargs(&mut python_signature, kwargs)?; + if let Some((_, annotation)) = &kwargs.colon_and_annotation { + ensure_spanned!( + cfg!(feature = "experimental-inspect"), + annotation.span() => "Type annotations in the signature is only supported with the `experimental-inspect` feature" + ); + #[cfg(feature = "experimental-inspect")] + { + let FnArg::KwArgs(fn_arg) = fn_arg else { + unreachable!( + "`Python` and `CancelHandle` are already handled above and `*args`/`**kwargs` are \ + parsed and transformed below. Because the have to come last and are only allowed \ + once, this has to be a regular argument." + ); + }; + fn_arg.annotation = Some(annotation.to_python()); + } + } } SignatureItem::PosargsSep(sep) => { parse_state.finish_pos_only_args(&mut python_signature, sep.span())? @@ -421,9 +553,11 @@ impl<'a> FunctionSignature<'a> { } // Ensure no non-py arguments remain - if let Some(arg) = args_iter.find(|arg| !arg.py && !arg.is_cancel_handle) { + if let Some(arg) = + args_iter.find(|arg| !matches!(arg, FnArg::Py(..) | FnArg::CancelHandle(..))) + { bail_spanned!( - attribute.kw.span() => format!("missing signature entry for argument `{}`", arg.name) + attribute.kw.span() => format!("missing signature entry for argument `{}`", arg.name()) ); } @@ -435,20 +569,19 @@ impl<'a> FunctionSignature<'a> { } /// Without `#[pyo3(signature)]` or `#[args]` - just take the Rust function arguments as positional. - pub fn from_arguments(arguments: Vec>) -> syn::Result { + pub fn from_arguments(arguments: Vec>) -> Self { let mut python_signature = PythonSignature::default(); for arg in &arguments { // Python<'_> arguments don't show in Python signature - if arg.py || arg.is_cancel_handle { + if matches!(arg, FnArg::Py(..) | FnArg::CancelHandle(..)) { continue; } - if arg.optional.is_none() { + if let FnArg::Regular(RegularArg { .. }) = arg { // This argument is required, all previous arguments must also have been required - ensure_spanned!( - python_signature.required_positional_parameters == python_signature.positional_parameters.len(), - arg.ty.span() => "required arguments after an `Option<_>` argument are ambiguous\n\ - = help: add a `#[pyo3(signature)]` annotation on this function to unambiguously specify the default values for all optional parameters" + assert_eq!( + python_signature.required_positional_parameters, + python_signature.positional_parameters.len(), ); python_signature.required_positional_parameters = @@ -457,52 +590,22 @@ impl<'a> FunctionSignature<'a> { python_signature .positional_parameters - .push(arg.name.unraw().to_string()); + .push(arg.name().unraw().to_string()); } - Ok(Self { + Self { arguments, python_signature, attribute: None, - }) + } } fn default_value_for_parameter(&self, parameter: &str) -> String { - let mut default = "...".to_string(); - if let Some(fn_arg) = self.arguments.iter().find(|arg| arg.name == parameter) { - if let Some(arg_default) = fn_arg.default.as_ref() { - match arg_default { - // literal values - syn::Expr::Lit(syn::ExprLit { lit, .. }) => match lit { - syn::Lit::Str(s) => default = s.token().to_string(), - syn::Lit::Char(c) => default = c.token().to_string(), - syn::Lit::Int(i) => default = i.base10_digits().to_string(), - syn::Lit::Float(f) => default = f.base10_digits().to_string(), - syn::Lit::Bool(b) => { - default = if b.value() { - "True".to_string() - } else { - "False".to_string() - } - } - _ => {} - }, - // None - syn::Expr::Path(syn::ExprPath { - qself: None, path, .. - }) if path.is_ident("None") => { - default = "None".to_string(); - } - // others, unsupported yet so defaults to `...` - _ => {} - } - } else if fn_arg.optional.is_some() { - // functions without a `#[pyo3(signature = (...))]` option - // will treat trailing `Option` arguments as having a default of `None` - default = "None".to_string(); - } + if let Some(fn_arg) = self.arguments.iter().find(|arg| arg.name() == parameter) { + fn_arg.default_value() + } else { + "...".to_string() } - default } pub fn text_signature(&self, self_argument: Option<&str>) -> String { diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index f5ae111bf48..36d413d21b9 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -1,19 +1,29 @@ use std::collections::HashSet; +use crate::combine_errors::CombineErrors; +#[cfg(feature = "experimental-inspect")] +use crate::introspection::{attribute_introspection_code, function_introspection_code}; +#[cfg(feature = "experimental-inspect")] +use crate::method::{FnSpec, FnType}; +#[cfg(feature = "experimental-inspect")] +use crate::utils::expr_to_python; +use crate::utils::{has_attribute, has_attribute_with_namespace, Ctx, PyO3CratePath}; use crate::{ attributes::{take_pyo3_options, CrateAttribute}, konst::{ConstAttributes, ConstSpec}, pyfunction::PyFunctionOptions, - pymethod::{self, is_proto_method, MethodAndMethodDef, MethodAndSlotDef}, - utils::get_pyo3_crate, + pymethod::{ + self, is_proto_method, GeneratedPyMethod, MethodAndMethodDef, MethodAndSlotDef, PyMethod, + }, }; use proc_macro2::TokenStream; -use pymethod::GeneratedPyMethod; use quote::{format_ident, quote}; +#[cfg(feature = "experimental-inspect")] +use syn::Ident; use syn::{ parse::{Parse, ParseStream}, spanned::Spanned, - Result, + ImplItemFn, Result, }; /// The mechanism used to collect `#[pymethods]` into the type object @@ -84,124 +94,158 @@ pub fn build_py_methods( } } +fn check_pyfunction(pyo3_path: &PyO3CratePath, meth: &mut ImplItemFn) -> syn::Result<()> { + let mut error = None; + + meth.attrs.retain(|attr| { + let attrs = [attr.clone()]; + + if has_attribute(&attrs, "pyfunction") + || has_attribute_with_namespace(&attrs, Some(pyo3_path), &["pyfunction"]) + || has_attribute_with_namespace(&attrs, Some(pyo3_path), &["prelude", "pyfunction"]) { + error = Some(err_spanned!(meth.sig.span() => "functions inside #[pymethods] do not need to be annotated with #[pyfunction]")); + false + } else { + true + } + }); + + error.map_or(Ok(()), Err) +} + pub fn impl_methods( ty: &syn::Type, impls: &mut [syn::ImplItem], methods_type: PyClassMethodsType, options: PyImplOptions, ) -> syn::Result { - let mut trait_impls = Vec::new(); + let mut extra_fragments = Vec::new(); let mut proto_impls = Vec::new(); let mut methods = Vec::new(); let mut associated_methods = Vec::new(); let mut implemented_proto_fragments = HashSet::new(); - for iimpl in impls { - match iimpl { - syn::ImplItem::Fn(meth) => { - let mut fun_options = PyFunctionOptions::from_attrs(&mut meth.attrs)?; - fun_options.krate = fun_options.krate.or_else(|| options.krate.clone()); - match pymethod::gen_py_method(ty, &mut meth.sig, &mut meth.attrs, fun_options)? { - GeneratedPyMethod::Method(MethodAndMethodDef { - associated_method, - method_def, - }) => { - let attrs = get_cfg_attributes(&meth.attrs); - associated_methods.push(quote!(#(#attrs)* #associated_method)); - methods.push(quote!(#(#attrs)* #method_def)); - } - GeneratedPyMethod::SlotTraitImpl(method_name, token_stream) => { - implemented_proto_fragments.insert(method_name); - let attrs = get_cfg_attributes(&meth.attrs); - trait_impls.push(quote!(#(#attrs)* #token_stream)); - } - GeneratedPyMethod::Proto(MethodAndSlotDef { - associated_method, - slot_def, - }) => { - let attrs = get_cfg_attributes(&meth.attrs); - proto_impls.push(quote!(#(#attrs)* #slot_def)); - associated_methods.push(quote!(#(#attrs)* #associated_method)); + let _: Vec<()> = impls + .iter_mut() + .map(|iimpl| { + match iimpl { + syn::ImplItem::Fn(meth) => { + let ctx = &Ctx::new(&options.krate, Some(&meth.sig)); + let mut fun_options = PyFunctionOptions::from_attrs(&mut meth.attrs)?; + fun_options.krate = fun_options.krate.or_else(|| options.krate.clone()); + + check_pyfunction(&ctx.pyo3_path, meth)?; + let method = PyMethod::parse(&mut meth.sig, &mut meth.attrs, fun_options)?; + #[cfg(feature = "experimental-inspect")] + extra_fragments.push(method_introspection_code(&method.spec, ty, ctx)); + match pymethod::gen_py_method(ty, method, &meth.attrs, ctx)? { + GeneratedPyMethod::Method(MethodAndMethodDef { + associated_method, + method_def, + }) => { + let attrs = get_cfg_attributes(&meth.attrs); + associated_methods.push(quote!(#(#attrs)* #associated_method)); + methods.push(quote!(#(#attrs)* #method_def)); + } + GeneratedPyMethod::SlotTraitImpl(method_name, token_stream) => { + implemented_proto_fragments.insert(method_name); + let attrs = get_cfg_attributes(&meth.attrs); + extra_fragments.push(quote!(#(#attrs)* #token_stream)); + } + GeneratedPyMethod::Proto(MethodAndSlotDef { + associated_method, + slot_def, + }) => { + let attrs = get_cfg_attributes(&meth.attrs); + proto_impls.push(quote!(#(#attrs)* #slot_def)); + associated_methods.push(quote!(#(#attrs)* #associated_method)); + } } } - } - syn::ImplItem::Const(konst) => { - let attributes = ConstAttributes::from_attrs(&mut konst.attrs)?; - if attributes.is_class_attr { - let spec = ConstSpec { - rust_ident: konst.ident.clone(), - attributes, - }; - let attrs = get_cfg_attributes(&konst.attrs); - let MethodAndMethodDef { - associated_method, - method_def, - } = gen_py_const(ty, &spec); - methods.push(quote!(#(#attrs)* #method_def)); - associated_methods.push(quote!(#(#attrs)* #associated_method)); - if is_proto_method(&spec.python_name().to_string()) { - // If this is a known protocol method e.g. __contains__, then allow this - // symbol even though it's not an uppercase constant. - konst - .attrs - .push(syn::parse_quote!(#[allow(non_upper_case_globals)])); + syn::ImplItem::Const(konst) => { + let ctx = &Ctx::new(&options.krate, None); + let attributes = ConstAttributes::from_attrs(&mut konst.attrs)?; + if attributes.is_class_attr { + let spec = ConstSpec { + rust_ident: konst.ident.clone(), + attributes, + }; + let attrs = get_cfg_attributes(&konst.attrs); + let MethodAndMethodDef { + associated_method, + method_def, + } = gen_py_const(ty, &spec, ctx); + methods.push(quote!(#(#attrs)* #method_def)); + associated_methods.push(quote!(#(#attrs)* #associated_method)); + if is_proto_method(&spec.python_name().to_string()) { + // If this is a known protocol method e.g. __contains__, then allow this + // symbol even though it's not an uppercase constant. + konst + .attrs + .push(syn::parse_quote!(#[allow(non_upper_case_globals)])); + } + #[cfg(feature = "experimental-inspect")] + extra_fragments.push(attribute_introspection_code( + &ctx.pyo3_path, + Some(ty), + spec.python_name().to_string(), + expr_to_python(&konst.expr), + konst.ty.clone(), + true, + )); } } + syn::ImplItem::Macro(m) => bail_spanned!( + m.span() => + "macros cannot be used as items in `#[pymethods]` impl blocks\n\ + = note: this was previously accepted and ignored" + ), + _ => {} } - syn::ImplItem::Macro(m) => bail_spanned!( - m.span() => - "macros cannot be used as items in `#[pymethods]` impl blocks\n\ - = note: this was previously accepted and ignored" - ), - _ => {} - } - } + Ok(()) + }) + .try_combine_syn_errors()?; - add_shared_proto_slots(ty, &mut proto_impls, implemented_proto_fragments); + let ctx = &Ctx::new(&options.krate, None); - let krate = get_pyo3_crate(&options.krate); + add_shared_proto_slots(ty, &mut proto_impls, implemented_proto_fragments, ctx); let items = match methods_type { - PyClassMethodsType::Specialization => impl_py_methods(ty, methods, proto_impls), - PyClassMethodsType::Inventory => submit_methods_inventory(ty, methods, proto_impls), + PyClassMethodsType::Specialization => impl_py_methods(ty, methods, proto_impls, ctx), + PyClassMethodsType::Inventory => submit_methods_inventory(ty, methods, proto_impls, ctx), }; Ok(quote! { - const _: () = { - use #krate as _pyo3; + #(#extra_fragments)* - #(#trait_impls)* + #items - #items - - #[doc(hidden)] - #[allow(non_snake_case)] - impl #ty { - #(#associated_methods)* - } - }; + #[doc(hidden)] + #[allow(non_snake_case)] + impl #ty { + #(#associated_methods)* + } }) } -pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec) -> MethodAndMethodDef { +pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec, ctx: &Ctx) -> MethodAndMethodDef { let member = &spec.rust_ident; let wrapper_ident = format_ident!("__pymethod_{}__", member); - let deprecations = &spec.attributes.deprecations; - let python_name = &spec.null_terminated_python_name(); + let python_name = spec.null_terminated_python_name(); + let Ctx { pyo3_path, .. } = ctx; let associated_method = quote! { - fn #wrapper_ident(py: _pyo3::Python<'_>) -> _pyo3::PyResult<_pyo3::PyObject> { - #deprecations - ::std::result::Result::Ok(_pyo3::IntoPy::into_py(#cls::#member, py)) + fn #wrapper_ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Py<#pyo3_path::PyAny>> { + #pyo3_path::IntoPyObjectExt::into_py_any(#cls::#member, py) } }; let method_def = quote! { - _pyo3::class::PyMethodDefType::ClassAttribute({ - _pyo3::class::PyClassAttributeDef::new( + #pyo3_path::impl_::pymethods::PyMethodDefType::ClassAttribute({ + #pyo3_path::impl_::pymethods::PyClassAttributeDef::new( #python_name, - _pyo3::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) + #cls::#wrapper_ident ) }) }; @@ -216,13 +260,16 @@ fn impl_py_methods( ty: &syn::Type, methods: Vec, proto_impls: Vec, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; quote! { - impl _pyo3::impl_::pyclass::PyMethods<#ty> - for _pyo3::impl_::pyclass::PyClassImplCollector<#ty> + #[allow(unknown_lints, non_local_definitions)] + impl #pyo3_path::impl_::pyclass::PyMethods<#ty> + for #pyo3_path::impl_::pyclass::PyClassImplCollector<#ty> { - fn py_methods(self) -> &'static _pyo3::impl_::pyclass::PyClassItems { - static ITEMS: _pyo3::impl_::pyclass::PyClassItems = _pyo3::impl_::pyclass::PyClassItems { + fn py_methods(self) -> &'static #pyo3_path::impl_::pyclass::PyClassItems { + static ITEMS: #pyo3_path::impl_::pyclass::PyClassItems = #pyo3_path::impl_::pyclass::PyClassItems { methods: &[#(#methods),*], slots: &[#(#proto_impls),*] }; @@ -236,13 +283,15 @@ fn add_shared_proto_slots( ty: &syn::Type, proto_impls: &mut Vec, mut implemented_proto_fragments: HashSet, + ctx: &Ctx, ) { + let Ctx { pyo3_path, .. } = ctx; macro_rules! try_add_shared_slot { ($slot:ident, $($fragments:literal),*) => {{ let mut implemented = false; $(implemented |= implemented_proto_fragments.remove($fragments));*; if implemented { - proto_impls.push(quote! { _pyo3::impl_::pyclass::$slot!(#ty) }) + proto_impls.push(quote! { #pyo3_path::impl_::pyclass::$slot!(#ty) }) } }}; } @@ -292,18 +341,101 @@ fn submit_methods_inventory( ty: &syn::Type, methods: Vec, proto_impls: Vec, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; quote! { - _pyo3::inventory::submit! { - type Inventory = <#ty as _pyo3::impl_::pyclass::PyClassImpl>::Inventory; - Inventory::new(_pyo3::impl_::pyclass::PyClassItems { methods: &[#(#methods),*], slots: &[#(#proto_impls),*] }) + #pyo3_path::inventory::submit! { + type Inventory = <#ty as #pyo3_path::impl_::pyclass::PyClassImpl>::Inventory; + Inventory::new(#pyo3_path::impl_::pyclass::PyClassItems { methods: &[#(#methods),*], slots: &[#(#proto_impls),*] }) } } } -fn get_cfg_attributes(attrs: &[syn::Attribute]) -> Vec<&syn::Attribute> { +pub(crate) fn get_cfg_attributes(attrs: &[syn::Attribute]) -> Vec<&syn::Attribute> { attrs .iter() .filter(|attr| attr.path().is_ident("cfg")) .collect() } + +#[cfg(feature = "experimental-inspect")] +fn method_introspection_code(spec: &FnSpec<'_>, parent: &syn::Type, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + + let name = spec.python_name.to_string(); + + // __richcmp__ special case + if name == "__richcmp__" { + // We expend into each individual method + return ["__eq__", "__ne__", "__lt__", "__le__", "__gt__", "__ge__"] + .into_iter() + .map(|method_name| { + let mut spec = (*spec).clone(); + spec.python_name = Ident::new(method_name, spec.python_name.span()); + // We remove the CompareOp arg, this is safe because the signature is always the same + // First the other value to compare with then the CompareOp + // We cant to keep the first argument type, hence this hack + spec.signature.arguments.pop(); + spec.signature.python_signature.positional_parameters.pop(); + method_introspection_code(&spec, parent, ctx) + }) + .collect(); + } + // We map or ignore some magic methods + // TODO: this might create a naming conflict + let name = match name.as_str() { + "__concat__" => "__add__".into(), + "__repeat__" => "__mul__".into(), + "__inplace_concat__" => "__iadd__".into(), + "__inplace_repeat__" => "__imul__".into(), + "__getbuffer__" | "__releasebuffer__" | "__traverse__" | "__clear__" => return quote! {}, + _ => name, + }; + + // We introduce self/cls argument and setup decorators + let mut first_argument = None; + let mut output = spec.output.clone(); + let mut decorators = Vec::new(); + match &spec.tp { + FnType::Getter(_) => { + first_argument = Some("self"); + decorators.push("property".into()); + } + FnType::Setter(_) => { + first_argument = Some("self"); + decorators.push(format!("{name}.setter")); + } + FnType::Fn(_) => { + first_argument = Some("self"); + } + FnType::FnNew | FnType::FnNewClass(_) => { + first_argument = Some("cls"); + output = syn::ReturnType::Default; // The __new__ Python function return type is None + } + FnType::FnClass(_) => { + first_argument = Some("cls"); + decorators.push("classmethod".into()); + } + FnType::FnStatic => { + decorators.push("staticmethod".into()); + } + FnType::FnModule(_) => (), // TODO: not sure this can happen + FnType::ClassAttribute => { + first_argument = Some("cls"); + // TODO: this combination only works with Python 3.9-3.11 https://docs.python.org/3.11/library/functions.html#classmethod + decorators.push("classmethod".into()); + decorators.push("property".into()); + } + } + function_introspection_code( + pyo3_path, + None, + &name, + &spec.signature, + first_argument, + output, + decorators, + Some(parent), + ) +} diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index d45d2e12f26..755bd367f27 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1,7 +1,13 @@ use std::borrow::Cow; - -use crate::attributes::{NameAttribute, RenamingRule}; -use crate::method::{CallingConvention, ExtractErrorMode}; +use std::ffi::CString; + +use crate::attributes::{FromPyWithAttribute, NameAttribute, RenamingRule}; +#[cfg(feature = "experimental-inspect")] +use crate::introspection::unique_element_id; +use crate::method::{CallingConvention, ExtractErrorMode, PyArg}; +use crate::params::{impl_regular_arg_param, Holders}; +use crate::pyfunction::WarningFactory; +use crate::utils::Ctx; use crate::utils::PythonDoc; use crate::{ method::{FnArg, FnSpec, FnType, SelfType}, @@ -9,8 +15,9 @@ use crate::{ }; use crate::{quotes, utils}; use proc_macro2::{Span, TokenStream}; -use quote::{format_ident, quote, ToTokens}; -use syn::{ext::IdentExt, spanned::Spanned, Result}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; +use syn::LitCStr; +use syn::{ext::IdentExt, spanned::Spanned, Field, Ident, Result}; /// Generated code for a single pymethod item. pub struct MethodAndMethodDef { @@ -20,6 +27,18 @@ pub struct MethodAndMethodDef { pub method_def: TokenStream, } +#[cfg(feature = "experimental-inspect")] +impl MethodAndMethodDef { + pub fn add_introspection(&mut self, data: TokenStream) { + let const_name = format_ident!("_{}", unique_element_id()); // We need an explicit name here + self.associated_method.extend(quote! { + const #const_name: () = { + #data + }; + }); + } +} + /// Generated code for a single pymethod item which is registered by a slot. pub struct MethodAndSlotDef { /// The implementation of the Python wrapper for the pymethod @@ -28,6 +47,18 @@ pub struct MethodAndSlotDef { pub slot_def: TokenStream, } +#[cfg(feature = "experimental-inspect")] +impl MethodAndSlotDef { + pub fn add_introspection(&mut self, data: TokenStream) { + let const_name = format_ident!("_{}", unique_element_id()); // We need an explicit name here + self.associated_method.extend(quote! { + const #const_name: () = { + #data + }; + }); + } +} + pub enum GeneratedPyMethod { Method(MethodAndMethodDef), Proto(MethodAndSlotDef), @@ -37,7 +68,7 @@ pub enum GeneratedPyMethod { pub struct PyMethod<'a> { kind: PyMethodKind, method_name: String, - spec: FnSpec<'a>, + pub spec: FnSpec<'a>, } enum PyMethodKind { @@ -93,7 +124,6 @@ impl PyMethodKind { "__ior__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IOR__)), "__getbuffer__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__GETBUFFER__)), "__releasebuffer__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__RELEASEBUFFER__)), - "__clear__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__CLEAR__)), // Protocols implemented through traits "__getattribute__" => { PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__GETATTRIBUTE__)) @@ -142,6 +172,7 @@ impl PyMethodKind { // Some tricky protocols which don't fit the pattern of the rest "__call__" => PyMethodKind::Proto(PyMethodProtoKind::Call), "__traverse__" => PyMethodKind::Proto(PyMethodProtoKind::Traverse), + "__clear__" => PyMethodKind::Proto(PyMethodProtoKind::Clear), // Not a proto _ => PyMethodKind::Fn, } @@ -152,15 +183,18 @@ enum PyMethodProtoKind { Slot(&'static SlotDef), Call, Traverse, + Clear, SlotFragment(&'static SlotFragmentDef), } impl<'a> PyMethod<'a> { - fn parse( + pub fn parse( sig: &'a mut syn::Signature, meth_attrs: &mut Vec, options: PyFunctionOptions, ) -> Result { + check_generic(sig)?; + ensure_function_options_valid(&options)?; let spec = FnSpec::parse(sig, meth_attrs, options)?; let method_name = spec.python_name.to_string(); @@ -183,36 +217,44 @@ pub fn is_proto_method(name: &str) -> bool { pub fn gen_py_method( cls: &syn::Type, - sig: &mut syn::Signature, - meth_attrs: &mut Vec, - options: PyFunctionOptions, + method: PyMethod<'_>, + meth_attrs: &[syn::Attribute], + ctx: &Ctx, ) -> Result { - check_generic(sig)?; - ensure_function_options_valid(&options)?; - let method = PyMethod::parse(sig, meth_attrs, options)?; let spec = &method.spec; + let Ctx { pyo3_path, .. } = ctx; + + if spec.asyncness.is_some() { + ensure_spanned!( + cfg!(feature = "experimental-async"), + spec.asyncness.span() => "async functions are only supported with the `experimental-async` feature" + ); + } Ok(match (method.kind, &spec.tp) { // Class attributes go before protos so that class attributes can be used to set proto // method to None. (_, FnType::ClassAttribute) => { - GeneratedPyMethod::Method(impl_py_class_attribute(cls, spec)?) + GeneratedPyMethod::Method(impl_py_class_attribute(cls, spec, ctx)?) } (PyMethodKind::Proto(proto_kind), _) => { ensure_no_forbidden_protocol_attributes(&proto_kind, spec, &method.method_name)?; match proto_kind { PyMethodProtoKind::Slot(slot_def) => { - let slot = slot_def.generate_type_slot(cls, spec, &method.method_name)?; + let slot = slot_def.generate_type_slot(cls, spec, &method.method_name, ctx)?; GeneratedPyMethod::Proto(slot) } PyMethodProtoKind::Call => { - GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec)?) + GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec, ctx)?) } PyMethodProtoKind::Traverse => { - GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec)?) + GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec, ctx)?) + } + PyMethodProtoKind::Clear => { + GeneratedPyMethod::Proto(impl_clear_slot(cls, spec, ctx)?) } PyMethodProtoKind::SlotFragment(slot_fragment_def) => { - let proto = slot_fragment_def.generate_pyproto_fragment(cls, spec)?; + let proto = slot_fragment_def.generate_pyproto_fragment(cls, spec, ctx)?; GeneratedPyMethod::SlotTraitImpl(method.method_name, proto) } } @@ -221,24 +263,27 @@ pub fn gen_py_method( (_, FnType::Fn(_)) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, - &spec.get_doc(meth_attrs), + &spec.get_doc(meth_attrs, ctx)?, None, + ctx, )?), (_, FnType::FnClass(_)) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, - &spec.get_doc(meth_attrs), - Some(quote!(_pyo3::ffi::METH_CLASS)), + &spec.get_doc(meth_attrs, ctx)?, + Some(quote!(#pyo3_path::ffi::METH_CLASS)), + ctx, )?), (_, FnType::FnStatic) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, - &spec.get_doc(meth_attrs), - Some(quote!(_pyo3::ffi::METH_STATIC)), + &spec.get_doc(meth_attrs, ctx)?, + Some(quote!(#pyo3_path::ffi::METH_STATIC)), + ctx, )?), // special prototypes (_, FnType::FnNew) | (_, FnType::FnNewClass(_)) => { - GeneratedPyMethod::Proto(impl_py_method_def_new(cls, spec)?) + GeneratedPyMethod::Proto(impl_py_method_def_new(cls, spec, ctx)?) } (_, FnType::Getter(self_type)) => GeneratedPyMethod::Method(impl_py_getter_def( @@ -246,16 +291,18 @@ pub fn gen_py_method( PropertyType::Function { self_type, spec, - doc: spec.get_doc(meth_attrs), + doc: spec.get_doc(meth_attrs, ctx)?, }, + ctx, )?), (_, FnType::Setter(self_type)) => GeneratedPyMethod::Method(impl_py_setter_def( cls, PropertyType::Function { self_type, spec, - doc: spec.get_doc(meth_attrs), + doc: spec.get_doc(meth_attrs, ctx)?, }, + ctx, )?), (_, FnType::FnModule(_)) => { unreachable!("methods cannot be FnModule") @@ -264,7 +311,7 @@ pub fn gen_py_method( } pub fn check_generic(sig: &syn::Signature) -> syn::Result<()> { - let err_msg = |typ| format!("Python functions cannot have generic {} parameters", typ); + let err_msg = |typ| format!("Python functions cannot have generic {typ} parameters"); for param in &sig.generics.params { match param { syn::GenericParam::Lifetime(_) => {} @@ -305,18 +352,20 @@ pub fn impl_py_method_def( spec: &FnSpec<'_>, doc: &PythonDoc, flags: Option, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path, .. } = ctx; let wrapper_ident = format_ident!("__pymethod_{}__", spec.python_name); - let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; + let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; let add_flags = flags.map(|flags| quote!(.flags(#flags))); let methoddef_type = match spec.tp { FnType::FnStatic => quote!(Static), FnType::FnClass(_) => quote!(Class), _ => quote!(Method), }; - let methoddef = spec.get_methoddef(quote! { #cls::#wrapper_ident }, doc); + let methoddef = spec.get_methoddef(quote! { #cls::#wrapper_ident }, doc, ctx); let method_def = quote! { - _pyo3::class::PyMethodDefType::#methoddef_type(#methoddef #add_flags) + #pyo3_path::impl_::pymethods::PyMethodDefType::#methoddef_type(#methoddef #add_flags) }; Ok(MethodAndMethodDef { associated_method, @@ -325,38 +374,38 @@ pub fn impl_py_method_def( } /// Also used by pyclass. -pub fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec<'_>) -> Result { +pub fn impl_py_method_def_new( + cls: &syn::Type, + spec: &FnSpec<'_>, + ctx: &Ctx, +) -> Result { + let Ctx { pyo3_path, .. } = ctx; let wrapper_ident = syn::Ident::new("__pymethod___new____", Span::call_site()); - let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; + let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; // Use just the text_signature_call_signature() because the class' Python name // isn't known to `#[pymethods]` - that has to be attached at runtime from the PyClassImpl // trait implementation created by `#[pyclass]`. - let text_signature_body = spec.text_signature_call_signature().map_or_else( - || quote!(::std::option::Option::None), - |text_signature| quote!(::std::option::Option::Some(#text_signature)), - ); - let deprecations = &spec.deprecations; + let text_signature_impl = spec.text_signature_call_signature().map(|text_signature| { + quote! { + #[allow(unknown_lints, non_local_definitions)] + impl #pyo3_path::impl_::pyclass::doc::PyClassNewTextSignature for #cls { + const TEXT_SIGNATURE: &'static str = #text_signature; + } + } + }); let slot_def = quote! { - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::Py_tp_new, + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_new, pfunc: { unsafe extern "C" fn trampoline( - subtype: *mut _pyo3::ffi::PyTypeObject, - args: *mut _pyo3::ffi::PyObject, - kwargs: *mut _pyo3::ffi::PyObject, - ) -> *mut _pyo3::ffi::PyObject - { - #deprecations - - use _pyo3::impl_::pyclass::*; - impl PyClassNewTextSignature<#cls> for PyClassImplCollector<#cls> { - #[inline] - fn new_text_signature(self) -> ::std::option::Option<&'static str> { - #text_signature_body - } - } + subtype: *mut #pyo3_path::ffi::PyTypeObject, + args: *mut #pyo3_path::ffi::PyObject, + kwargs: *mut #pyo3_path::ffi::PyObject, + ) -> *mut #pyo3_path::ffi::PyObject { + + #text_signature_impl - _pyo3::impl_::trampoline::newfunc( + #pyo3_path::impl_::trampoline::newfunc( subtype, args, kwargs, @@ -364,7 +413,7 @@ pub fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec<'_>) -> Result) -> Result) -> Result { +fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result { + let Ctx { pyo3_path, .. } = ctx; + // HACK: __call__ proto slot must always use varargs calling convention, so change the spec. // Probably indicates there's a refactoring opportunity somewhere. spec.convention = CallingConvention::Varargs; let wrapper_ident = syn::Ident::new("__pymethod___call____", Span::call_site()); - let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; + let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; let slot_def = quote! { - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::Py_tp_call, + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_call, pfunc: { unsafe extern "C" fn trampoline( - slf: *mut _pyo3::ffi::PyObject, - args: *mut _pyo3::ffi::PyObject, - kwargs: *mut _pyo3::ffi::PyObject, - ) -> *mut _pyo3::ffi::PyObject + slf: *mut #pyo3_path::ffi::PyObject, + args: *mut #pyo3_path::ffi::PyObject, + kwargs: *mut #pyo3_path::ffi::PyObject, + ) -> *mut #pyo3_path::ffi::PyObject { - _pyo3::impl_::trampoline::ternaryfunc( + #pyo3_path::impl_::trampoline::ternaryfunc( slf, args, kwargs, @@ -398,7 +449,7 @@ fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>) -> Result) -> Result) -> syn::Result { +fn impl_traverse_slot( + cls: &syn::Type, + spec: &FnSpec<'_>, + ctx: &Ctx, +) -> syn::Result { + let Ctx { pyo3_path, .. } = ctx; if let (Some(py_arg), _) = split_off_python_arg(&spec.signature.arguments) { return Err(syn::Error::new_spanned(py_arg.ty, "__traverse__ may not take `Python`. \ - Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. \ - Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, \ - i.e. `Python::with_gil` will panic.")); + Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` \ + should do nothing but calls to `visit.call`. Most importantly, safe access to the Python interpreter is \ + prohibited inside implementations of `__traverse__`, i.e. `Python::attach` will panic.")); } + // check that the receiver does not try to smuggle an (implicit) `Python` token into here + if let FnType::Fn(SelfType::TryFromBoundRef(span)) + | FnType::Fn(SelfType::Receiver { + mutable: true, + span, + }) = spec.tp + { + bail_spanned! { span => + "__traverse__ may not take a receiver other than `&self`. Usually, an implementation of \ + `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` \ + should do nothing but calls to `visit.call`. Most importantly, safe access to the Python interpreter is \ + prohibited inside implementations of `__traverse__`, i.e. `Python::attach` will panic." + } + } + + ensure_spanned!( + spec.warnings.is_empty(), + spec.warnings.span() => "__traverse__ cannot be used with #[pyo3(warn)]" + ); + let rust_fn_ident = spec.name; let associated_method = quote! { pub unsafe extern "C" fn __pymethod_traverse__( - slf: *mut _pyo3::ffi::PyObject, - visit: _pyo3::ffi::visitproc, - arg: *mut ::std::os::raw::c_void, - ) -> ::std::os::raw::c_int { - _pyo3::impl_::pymethods::_call_traverse::<#cls>(slf, #cls::#rust_fn_ident, visit, arg) + slf: *mut #pyo3_path::ffi::PyObject, + visit: #pyo3_path::ffi::visitproc, + arg: *mut ::std::ffi::c_void, + ) -> ::std::ffi::c_int { + #pyo3_path::impl_::pymethods::_call_traverse::<#cls>(slf, #cls::#rust_fn_ident, visit, arg, #cls::__pymethod_traverse__) } }; let slot_def = quote! { - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::Py_tp_traverse, - pfunc: #cls::__pymethod_traverse__ as _pyo3::ffi::traverseproc as _ + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_traverse, + pfunc: #cls::__pymethod_traverse__ as #pyo3_path::ffi::traverseproc as _ } }; Ok(MethodAndSlotDef { @@ -438,11 +514,68 @@ fn impl_traverse_slot(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result) -> syn::Result { +fn impl_clear_slot(cls: &syn::Type, spec: &FnSpec<'_>, ctx: &Ctx) -> syn::Result { + let Ctx { pyo3_path, .. } = ctx; + let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); + let self_type = match &spec.tp { + FnType::Fn(self_type) => self_type, + _ => bail_spanned!(spec.name.span() => "expected instance method for `__clear__` function"), + }; + let mut holders = Holders::new(); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise, &mut holders, ctx); + + if let [arg, ..] = args { + bail_spanned!(arg.ty().span() => "`__clear__` function expected to have no arguments"); + } + + let name = &spec.name; + let holders = holders.init_holders(ctx); + let fncall = if py_arg.is_some() { + quote!(#cls::#name(#slf, py)) + } else { + quote!(#cls::#name(#slf)) + }; + + let associated_method = quote! { + pub unsafe extern "C" fn __pymethod___clear____( + _slf: *mut #pyo3_path::ffi::PyObject, + ) -> ::std::ffi::c_int { + #pyo3_path::impl_::pymethods::_call_clear(_slf, |py, _slf| { + #holders + let result = #fncall; + let result = #pyo3_path::impl_::wrap::converter(&result).wrap(result)?; + ::std::result::Result::Ok(result) + }, #cls::__pymethod___clear____) + } + }; + let slot_def = quote! { + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_clear, + pfunc: #cls::__pymethod___clear____ as #pyo3_path::ffi::inquiry as _ + } + }; + Ok(MethodAndSlotDef { + associated_method, + slot_def, + }) +} + +pub(crate) fn impl_py_class_attribute( + cls: &syn::Type, + spec: &FnSpec<'_>, + ctx: &Ctx, +) -> syn::Result { + let Ctx { pyo3_path, .. } = ctx; let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); ensure_spanned!( args.is_empty(), - args[0].ty.span() => "#[classattr] can only have one argument (of type pyo3::Python)" + args[0].ty().span() => "#[classattr] can only have one argument (of type pyo3::Python)" + ); + + ensure_spanned!( + spec.warnings.is_empty(), + spec.warnings.span() + => "#[classattr] cannot be used with #[pyo3(warn)]" ); let name = &spec.name; @@ -454,20 +587,21 @@ fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result) -> _pyo3::PyResult<_pyo3::PyObject> { + fn #wrapper_ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Py<#pyo3_path::PyAny>> { let function = #cls::#name; // Shadow the method name to avoid #3017 - _pyo3::impl_::wrap::map_result_into_py(py, #body) + let result = #body; + #pyo3_path::impl_::wrap::converter(&result).map_into_pyobject(py, result) } }; let method_def = quote! { - _pyo3::class::PyMethodDefType::ClassAttribute({ - _pyo3::class::PyClassAttributeDef::new( + #pyo3_path::impl_::pymethods::PyMethodDefType::ClassAttribute({ + #pyo3_path::impl_::pymethods::PyClassAttributeDef::new( #python_name, - _pyo3::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) + #cls::#wrapper_ident ) }) }; @@ -482,16 +616,17 @@ fn impl_call_setter( cls: &syn::Type, spec: &FnSpec<'_>, self_type: &SelfType, - holders: &mut Vec, + holders: &mut Holders, + ctx: &Ctx, ) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); - let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders, ctx); if args.is_empty() { bail_spanned!(spec.name.span() => "setter function expected to have one argument"); } else if args.len() > 1 { bail_spanned!( - args[1].ty.span() => + args[1].ty().span() => "setter function can have at most two arguments ([pyo3::Python,] and value)" ); } @@ -510,10 +645,12 @@ fn impl_call_setter( pub fn impl_py_setter_def( cls: &syn::Type, property_type: PropertyType<'_>, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path, .. } = ctx; let python_name = property_type.null_terminated_python_name()?; - let doc = property_type.doc(); - let mut holders = Vec::new(); + let doc = property_type.doc(ctx)?; + let mut holders = Holders::new(); let setter_impl = match property_type { PropertyType::Descriptor { field_index, field, .. @@ -522,7 +659,7 @@ pub fn impl_py_setter_def( mutable: true, span: Span::call_site(), } - .receiver(cls, ExtractErrorMode::Raise, &mut holders); + .receiver(cls, ExtractErrorMode::Raise, &mut holders, ctx); if let Some(ident) = &field.ident { // named struct field quote!({ #slf.#ident = _val; }) @@ -534,7 +671,7 @@ pub fn impl_py_setter_def( } PropertyType::Function { spec, self_type, .. - } => impl_call_setter(cls, spec, self_type, &mut holders)?, + } => impl_call_setter(cls, spec, self_type, &mut holders, ctx)?, }; let wrapper_ident = match property_type { @@ -554,6 +691,59 @@ pub fn impl_py_setter_def( } }; + let extract = match &property_type { + PropertyType::Function { spec, .. } => { + let (_, args) = split_off_python_arg(&spec.signature.arguments); + let value_arg = &args[0]; + let (from_py_with, ident) = + if let Some(from_py_with) = &value_arg.from_py_with().as_ref().map(|f| &f.value) { + let ident = syn::Ident::new("from_py_with", from_py_with.span()); + ( + quote_spanned! { from_py_with.span() => + let #ident = #from_py_with; + }, + ident, + ) + } else { + (quote!(), syn::Ident::new("dummy", Span::call_site())) + }; + + let arg = if let FnArg::Regular(arg) = &value_arg { + arg + } else { + bail_spanned!(value_arg.name().span() => "The #[setter] value argument can't be *args, **kwargs or `cancel_handle`."); + }; + + let extract = impl_regular_arg_param( + arg, + ident, + quote!(::std::option::Option::Some(_value.into())), + &mut holders, + ctx, + ); + + quote! { + #from_py_with + let _val = #extract; + } + } + PropertyType::Descriptor { field, .. } => { + let span = field.ty.span(); + let name = field + .ident + .as_ref() + .map(|i| i.to_string()) + .unwrap_or_default(); + + let holder = holders.push_holder(span); + quote! { + #[allow(unused_imports)] + use #pyo3_path::impl_::pyclass::Probe as _; + let _val = #pyo3_path::impl_::extract_argument::extract_argument(_value.into(), &mut #holder, #name)?; + } + } + }; + let mut cfg_attrs = TokenStream::new(); if let PropertyType::Descriptor { field, .. } = &property_type { for attr in field @@ -565,30 +755,39 @@ pub fn impl_py_setter_def( } } + let warnings = if let PropertyType::Function { spec, .. } = &property_type { + spec.warnings.build_py_warning(ctx) + } else { + quote!() + }; + + let init_holders = holders.init_holders(ctx); let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( - py: _pyo3::Python<'_>, - _slf: *mut _pyo3::ffi::PyObject, - _value: *mut _pyo3::ffi::PyObject, - ) -> _pyo3::PyResult<::std::os::raw::c_int> { - let _value = py - .from_borrowed_ptr_or_opt(_value) + py: #pyo3_path::Python<'_>, + _slf: *mut #pyo3_path::ffi::PyObject, + _value: *mut #pyo3_path::ffi::PyObject, + ) -> #pyo3_path::PyResult<::std::ffi::c_int> { + use ::std::convert::Into; + let _value = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_value) .ok_or_else(|| { - _pyo3::exceptions::PyAttributeError::new_err("can't delete attribute") + #pyo3_path::exceptions::PyAttributeError::new_err("can't delete attribute") })?; - let _val = _pyo3::FromPyObject::extract(_value)?; - #( #holders )* - _pyo3::callback::convert(py, #setter_impl) + #init_holders + #extract + #warnings + let result = #setter_impl; + #pyo3_path::impl_::callback::convert(py, result) } }; let method_def = quote! { #cfg_attrs - _pyo3::class::PyMethodDefType::Setter( - _pyo3::class::PySetterDef::new( + #pyo3_path::impl_::pymethods::PyMethodDefType::Setter( + #pyo3_path::impl_::pymethods::PySetterDef::new( #python_name, - _pyo3::impl_::pymethods::PySetter(#cls::#wrapper_ident), + #cls::#wrapper_ident, #doc ) ) @@ -604,13 +803,14 @@ fn impl_call_getter( cls: &syn::Type, spec: &FnSpec<'_>, self_type: &SelfType, - holders: &mut Vec, + holders: &mut Holders, + ctx: &Ctx, ) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); - let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders, ctx); ensure_spanned!( args.is_empty(), - args[0].ty.span() => "getter function can only have one argument (of type pyo3::Python)" + args[0].ty().span() => "getter function can only have one argument (of type pyo3::Python)" ); let name = &spec.name; @@ -627,102 +827,112 @@ fn impl_call_getter( pub fn impl_py_getter_def( cls: &syn::Type, property_type: PropertyType<'_>, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path, .. } = ctx; let python_name = property_type.null_terminated_python_name()?; - let doc = property_type.doc(); + let doc = property_type.doc(ctx)?; - let mut holders = Vec::new(); - let body = match property_type { + let mut cfg_attrs = TokenStream::new(); + if let PropertyType::Descriptor { field, .. } = &property_type { + for attr in field + .attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + { + attr.to_tokens(&mut cfg_attrs); + } + } + + let mut holders = Holders::new(); + match property_type { PropertyType::Descriptor { field_index, field, .. } => { - let slf = SelfType::Receiver { - mutable: false, - span: Span::call_site(), - } - .receiver(cls, ExtractErrorMode::Raise, &mut holders); - let field_token = if let Some(ident) = &field.ident { - // named struct field + let ty = &field.ty; + let field = if let Some(ident) = &field.ident { ident.to_token_stream() } else { - // tuple struct field syn::Index::from(field_index).to_token_stream() }; - quotes::map_result_into_ptr(quotes::ok_wrap(quote! { - ::std::clone::Clone::clone(&(#slf.#field_token)) - })) + + let generator = quote_spanned! { ty.span() => + GENERATOR.generate(#python_name, #doc) + }; + // This is separate from `generator` so that the unsafe below does not inherit the span and thus does not + // trigger the `unsafe_code` lint + let method_def = quote! { + #cfg_attrs + { + #[allow(unused_imports)] // might not be used if all probes are positive + use #pyo3_path::impl_::pyclass::Probe as _; + + const GENERATOR: #pyo3_path::impl_::pyclass::PyClassGetterGenerator::< + #cls, + #ty, + { ::std::mem::offset_of!(#cls, #field) }, + { #pyo3_path::impl_::pyclass::IsPyT::<#ty>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#ty>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#ty>::VALUE }, + > = unsafe { #pyo3_path::impl_::pyclass::PyClassGetterGenerator::new() }; + #generator + } + }; + + Ok(MethodAndMethodDef { + associated_method: quote! {}, + method_def, + }) } // Forward to `IntoPyCallbackOutput`, to handle `#[getter]`s returning results. PropertyType::Function { spec, self_type, .. } => { - let call = impl_call_getter(cls, spec, self_type, &mut holders)?; - quote! { - _pyo3::callback::convert(py, #call) - } - } - }; + let wrapper_ident = format_ident!("__pymethod_get_{}__", spec.name); + let call = impl_call_getter(cls, spec, self_type, &mut holders, ctx)?; + let body = quote! { + #pyo3_path::impl_::callback::convert(py, #call) + }; - let wrapper_ident = match property_type { - PropertyType::Descriptor { - field: syn::Field { - ident: Some(ident), .. - }, - .. - } => { - format_ident!("__pymethod_get_{}__", ident) - } - PropertyType::Descriptor { field_index, .. } => { - format_ident!("__pymethod_get_field_{}__", field_index) - } - PropertyType::Function { spec, .. } => { - format_ident!("__pymethod_get_{}__", spec.name) - } - }; + let init_holders = holders.init_holders(ctx); + let warnings = spec.warnings.build_py_warning(ctx); + + let associated_method = quote! { + #cfg_attrs + unsafe fn #wrapper_ident( + py: #pyo3_path::Python<'_>, + _slf: *mut #pyo3_path::ffi::PyObject + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + #init_holders + #warnings + let result = #body; + result + } + }; - let mut cfg_attrs = TokenStream::new(); - if let PropertyType::Descriptor { field, .. } = &property_type { - for attr in field - .attrs - .iter() - .filter(|attr| attr.path().is_ident("cfg")) - { - attr.to_tokens(&mut cfg_attrs); - } - } + let method_def = quote! { + #cfg_attrs + #pyo3_path::impl_::pymethods::PyMethodDefType::Getter( + #pyo3_path::impl_::pymethods::PyGetterDef::new( + #python_name, + #cls::#wrapper_ident, + #doc + ) + ) + }; - let associated_method = quote! { - #cfg_attrs - unsafe fn #wrapper_ident( - py: _pyo3::Python<'_>, - _slf: *mut _pyo3::ffi::PyObject - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { - #( #holders )* - #body + Ok(MethodAndMethodDef { + associated_method, + method_def, + }) } - }; - - let method_def = quote! { - #cfg_attrs - _pyo3::class::PyMethodDefType::Getter( - _pyo3::class::PyGetterDef::new( - #python_name, - _pyo3::impl_::pymethods::PyGetter(#cls::#wrapper_ident), - #doc - ) - ) - }; - - Ok(MethodAndMethodDef { - associated_method, - method_def, - }) + } } /// Split an argument of pyo3::Python from the front of the arg list, if present -fn split_off_python_arg<'a>(args: &'a [FnArg<'a>]) -> (Option<&FnArg<'_>>, &[FnArg<'_>]) { +fn split_off_python_arg<'a, 'b>(args: &'a [FnArg<'b>]) -> (Option<&'a PyArg<'b>>, &'a [FnArg<'b>]) { match args { - [py, args @ ..] if utils::is_python(py.ty) => (Some(py), args), + [FnArg::Py(py), args @ ..] => (Some(py), args), args => (None, args), } } @@ -742,7 +952,7 @@ pub enum PropertyType<'a> { } impl PropertyType<'_> { - fn null_terminated_python_name(&self) -> Result { + fn null_terminated_python_name(&self) -> Result { match self { PropertyType::Descriptor { field, @@ -750,42 +960,30 @@ impl PropertyType<'_> { renaming_rule, .. } => { - let name = match (python_name, &field.ident) { - (Some(name), _) => name.value.0.to_string(), - (None, Some(field_name)) => { - let mut name = field_name.unraw().to_string(); - if let Some(rule) = renaming_rule { - name = utils::apply_renaming_rule(*rule, &name); - } - name.push('\0'); - name - } - (None, None) => { - bail_spanned!(field.span() => "`get` and `set` with tuple struct fields require `name`"); - } - }; - Ok(syn::LitStr::new(&name, field.span())) + let name = field_python_name(field, *python_name, *renaming_rule)?; + let name = CString::new(name).unwrap(); + Ok(LitCStr::new(&name, field.span())) } PropertyType::Function { spec, .. } => Ok(spec.null_terminated_python_name()), } } - fn doc(&self) -> Cow<'_, PythonDoc> { + fn doc(&self, ctx: &Ctx) -> Result> { match self { PropertyType::Descriptor { field, .. } => { - Cow::Owned(utils::get_doc(&field.attrs, None)) + utils::get_doc(&field.attrs, None, ctx).map(Cow::Owned) } - PropertyType::Function { doc, .. } => Cow::Borrowed(doc), + PropertyType::Function { doc, .. } => Ok(Cow::Borrowed(doc)), } } } -const __STR__: SlotDef = SlotDef::new("Py_tp_str", "reprfunc"); +pub const __STR__: SlotDef = SlotDef::new("Py_tp_str", "reprfunc"); pub const __REPR__: SlotDef = SlotDef::new("Py_tp_repr", "reprfunc"); -const __HASH__: SlotDef = SlotDef::new("Py_tp_hash", "hashfunc") +pub const __HASH__: SlotDef = SlotDef::new("Py_tp_hash", "hashfunc") .ret_ty(Ty::PyHashT) .return_conversion(TokenGenerator( - || quote! { _pyo3::callback::HashCallbackOutput }, + |Ctx { pyo3_path, .. }: &Ctx| quote! { #pyo3_path::impl_::callback::HashCallbackOutput }, )); pub const __RICHCMP__: SlotDef = SlotDef::new("Py_tp_richcompare", "richcmpfunc") .extract_error_mode(ExtractErrorMode::NotImplemented) @@ -795,16 +993,18 @@ const __GET__: SlotDef = SlotDef::new("Py_tp_descr_get", "descrgetfunc") const __ITER__: SlotDef = SlotDef::new("Py_tp_iter", "getiterfunc"); const __NEXT__: SlotDef = SlotDef::new("Py_tp_iternext", "iternextfunc") .return_specialized_conversion( - TokenGenerator(|| quote! { IterBaseKind, IterOptionKind, IterResultOptionKind }), - TokenGenerator(|| quote! { iter_tag }), + TokenGenerator(|_| quote! { IterBaseKind, IterOptionKind, IterResultOptionKind }), + TokenGenerator(|_| quote! { iter_tag }), ); const __AWAIT__: SlotDef = SlotDef::new("Py_am_await", "unaryfunc"); const __AITER__: SlotDef = SlotDef::new("Py_am_aiter", "unaryfunc"); const __ANEXT__: SlotDef = SlotDef::new("Py_am_anext", "unaryfunc").return_specialized_conversion( - TokenGenerator(|| quote! { AsyncIterBaseKind, AsyncIterOptionKind, AsyncIterResultOptionKind }), - TokenGenerator(|| quote! { async_iter_tag }), + TokenGenerator( + |_| quote! { AsyncIterBaseKind, AsyncIterOptionKind, AsyncIterResultOptionKind }, + ), + TokenGenerator(|_| quote! { async_iter_tag }), ); -const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc").ret_ty(Ty::PySsizeT); +pub const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc").ret_ty(Ty::PySsizeT); const __CONTAINS__: SlotDef = SlotDef::new("Py_sq_contains", "objobjproc") .arguments(&[Ty::Object]) .ret_ty(Ty::Int); @@ -814,7 +1014,8 @@ const __INPLACE_CONCAT__: SlotDef = SlotDef::new("Py_sq_concat", "binaryfunc").arguments(&[Ty::Object]); const __INPLACE_REPEAT__: SlotDef = SlotDef::new("Py_sq_repeat", "ssizeargfunc").arguments(&[Ty::PySsizeT]); -const __GETITEM__: SlotDef = SlotDef::new("Py_mp_subscript", "binaryfunc").arguments(&[Ty::Object]); +pub const __GETITEM__: SlotDef = + SlotDef::new("Py_mp_subscript", "binaryfunc").arguments(&[Ty::Object]); const __POS__: SlotDef = SlotDef::new("Py_nb_positive", "unaryfunc"); const __NEG__: SlotDef = SlotDef::new("Py_nb_negative", "unaryfunc"); @@ -904,16 +1105,21 @@ enum Ty { } impl Ty { - fn ffi_type(self) -> TokenStream { + fn ffi_type(self, ctx: &Ctx) -> TokenStream { + let Ctx { + pyo3_path, + output_span, + } = ctx; + let pyo3_path = pyo3_path.to_tokens_spanned(*output_span); match self { - Ty::Object | Ty::MaybeNullObject => quote! { *mut _pyo3::ffi::PyObject }, - Ty::NonNullObject => quote! { ::std::ptr::NonNull<_pyo3::ffi::PyObject> }, - Ty::IPowModulo => quote! { _pyo3::impl_::pymethods::IPowModulo }, - Ty::Int | Ty::CompareOp => quote! { ::std::os::raw::c_int }, - Ty::PyHashT => quote! { _pyo3::ffi::Py_hash_t }, - Ty::PySsizeT => quote! { _pyo3::ffi::Py_ssize_t }, + Ty::Object | Ty::MaybeNullObject => quote! { *mut #pyo3_path::ffi::PyObject }, + Ty::NonNullObject => quote! { ::std::ptr::NonNull<#pyo3_path::ffi::PyObject> }, + Ty::IPowModulo => quote! { #pyo3_path::impl_::pymethods::IPowModulo }, + Ty::Int | Ty::CompareOp => quote! { ::std::ffi::c_int }, + Ty::PyHashT => quote! { #pyo3_path::ffi::Py_hash_t }, + Ty::PySsizeT => quote! { #pyo3_path::ffi::Py_ssize_t }, Ty::Void => quote! { () }, - Ty::PyBuffer => quote! { *mut _pyo3::ffi::Py_buffer }, + Ty::PyBuffer => quote! { *mut #pyo3_path::ffi::Py_buffer }, } } @@ -922,60 +1128,63 @@ impl Ty { ident: &syn::Ident, arg: &FnArg<'_>, extract_error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, + ctx: &Ctx, ) -> TokenStream { - let name_str = arg.name.unraw().to_string(); + let Ctx { pyo3_path, .. } = ctx; match self { Ty::Object => extract_object( extract_error_mode, holders, - &name_str, - quote! { - py.from_borrowed_ptr::<_pyo3::PyAny>(#ident) - }, + arg, + format_ident!("ref_from_ptr"), + quote! { #ident }, + ctx ), Ty::MaybeNullObject => extract_object( extract_error_mode, holders, - &name_str, + arg, + format_ident!("ref_from_ptr"), quote! { - py.from_borrowed_ptr::<_pyo3::PyAny>( - if #ident.is_null() { - _pyo3::ffi::Py_None() - } else { - #ident - } - ) + if #ident.is_null() { + #pyo3_path::ffi::Py_None() + } else { + #ident + } }, + ctx ), Ty::NonNullObject => extract_object( extract_error_mode, holders, - &name_str, - quote! { - py.from_borrowed_ptr::<_pyo3::PyAny>(#ident.as_ptr()) - }, + arg, + format_ident!("ref_from_non_null"), + quote! { #ident }, + ctx ), Ty::IPowModulo => extract_object( extract_error_mode, holders, - &name_str, - quote! { - #ident.to_borrowed_any(py) - }, + arg, + format_ident!("ref_from_ptr"), + quote! { #ident.as_ptr() }, + ctx ), Ty::CompareOp => extract_error_mode.handle_error( quote! { - _pyo3::class::basic::CompareOp::from_raw(#ident) - .ok_or_else(|| _pyo3::exceptions::PyValueError::new_err("invalid comparison operator")) + #pyo3_path::class::basic::CompareOp::from_raw(#ident) + .ok_or_else(|| #pyo3_path::exceptions::PyValueError::new_err("invalid comparison operator")) }, + ctx ), Ty::PySsizeT => { - let ty = arg.ty; + let ty = arg.ty(); extract_error_mode.handle_error( quote! { - ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| _pyo3::exceptions::PyValueError::new_err(e.to_string())) + ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| #pyo3_path::exceptions::PyValueError::new_err(e.to_string())) }, + ctx ) } // Just pass other types through unmodified @@ -986,22 +1195,46 @@ impl Ty { fn extract_object( extract_error_mode: ExtractErrorMode, - holders: &mut Vec, - name: &str, - source: TokenStream, + holders: &mut Holders, + arg: &FnArg<'_>, + ref_from_method: Ident, + source_ptr: TokenStream, + ctx: &Ctx, ) -> TokenStream { - let holder = syn::Ident::new(&format!("holder_{}", holders.len()), Span::call_site()); - holders.push(quote! { - #[allow(clippy::let_unit_value)] - let mut #holder = _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT; - }); - extract_error_mode.handle_error(quote! { - _pyo3::impl_::extract_argument::extract_argument( - #source, - &mut #holder, - #name - ) - }) + let Ctx { pyo3_path, .. } = ctx; + let name = arg.name().unraw().to_string(); + + let extract = if let Some(FromPyWithAttribute { + kw, + value: extractor, + }) = arg.from_py_with() + { + let extractor = quote_spanned! { kw.span => + { let from_py_with: fn(_) -> _ = #extractor; from_py_with } + }; + + quote! { + #pyo3_path::impl_::extract_argument::from_py_with( + unsafe { #pyo3_path::impl_::pymethods::BoundRef::#ref_from_method(py, &#source_ptr).0 }, + #name, + #extractor, + ) + } + } else { + let holder = holders.push_holder(Span::call_site()); + quote! {{ + #[allow(unused_imports)] + use #pyo3_path::impl_::pyclass::Probe as _; + #pyo3_path::impl_::extract_argument::extract_argument( + unsafe { #pyo3_path::impl_::pymethods::BoundRef::#ref_from_method(py, &#source_ptr).0 }, + &mut #holder, + #name + ) + }} + }; + + let extracted = extract_error_mode.handle_error(extract, ctx); + quote!(#extracted) } enum ReturnMode { @@ -1011,21 +1244,29 @@ enum ReturnMode { } impl ReturnMode { - fn return_call_output(&self, call: TokenStream) -> TokenStream { + fn return_call_output(&self, call: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; match self { - ReturnMode::Conversion(conversion) => quote! { - let _result: _pyo3::PyResult<#conversion> = _pyo3::callback::convert(py, #call); - _pyo3::callback::convert(py, _result) - }, - ReturnMode::SpecializedConversion(traits, tag) => quote! { - let _result = #call; - use _pyo3::impl_::pymethods::{#traits}; - (&_result).#tag().convert(py, _result) - }, + ReturnMode::Conversion(conversion) => { + let conversion = TokenGeneratorCtx(*conversion, ctx); + quote! { + let _result: #pyo3_path::PyResult<#conversion> = #pyo3_path::impl_::callback::convert(py, #call); + #pyo3_path::impl_::callback::convert(py, _result) + } + } + ReturnMode::SpecializedConversion(traits, tag) => { + let traits = TokenGeneratorCtx(*traits, ctx); + let tag = TokenGeneratorCtx(*tag, ctx); + quote! { + let _result = #call; + use #pyo3_path::impl_::pymethods::{#traits}; + (&_result).#tag().convert(py, _result) + } + } ReturnMode::ReturnSelf => quote! { - let _result: _pyo3::PyResult<()> = _pyo3::callback::convert(py, #call); + let _result: #pyo3_path::PyResult<()> = #pyo3_path::impl_::callback::convert(py, #call); _result?; - _pyo3::ffi::Py_XINCREF(_raw_slf); + #pyo3_path::ffi::Py_XINCREF(_raw_slf); ::std::result::Result::Ok(_raw_slf) }, } @@ -1101,7 +1342,9 @@ impl SlotDef { cls: &syn::Type, spec: &FnSpec<'_>, method_name: &str, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path, .. } = ctx; let SlotDef { slot, func_ty, @@ -1117,13 +1360,13 @@ impl SlotDef { spec.name.span() => format!("`{}` must be `unsafe fn`", method_name) ); } - let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); + let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type(ctx)).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) .map(|i| format_ident!("arg{}", i)) .collect(); let wrapper_ident = format_ident!("__pymethod_{}__", method_name); - let ret_ty = ret_ty.ffi_type(); - let mut holders = Vec::new(); + let ret_ty = ret_ty.ffi_type(ctx); + let mut holders = Holders::new(); let body = generate_method_body( cls, spec, @@ -1131,36 +1374,39 @@ impl SlotDef { *extract_error_mode, &mut holders, return_mode.as_ref(), + ctx, )?; let name = spec.name; + let holders = holders.init_holders(ctx); let associated_method = quote! { + #[allow(non_snake_case)] unsafe fn #wrapper_ident( - py: _pyo3::Python<'_>, - _raw_slf: *mut _pyo3::ffi::PyObject, + py: #pyo3_path::Python<'_>, + _raw_slf: *mut #pyo3_path::ffi::PyObject, #(#arg_idents: #arg_types),* - ) -> _pyo3::PyResult<#ret_ty> { + ) -> #pyo3_path::PyResult<#ret_ty> { let function = #cls::#name; // Shadow the method name to avoid #3017 let _slf = _raw_slf; - #( #holders )* + #holders #body } }; let slot_def = quote! {{ unsafe extern "C" fn trampoline( - _slf: *mut _pyo3::ffi::PyObject, + _slf: *mut #pyo3_path::ffi::PyObject, #(#arg_idents: #arg_types),* ) -> #ret_ty { - _pyo3::impl_::trampoline:: #func_ty ( + #pyo3_path::impl_::trampoline:: #func_ty ( _slf, #(#arg_idents,)* #cls::#wrapper_ident ) } - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::#slot, - pfunc: trampoline as _pyo3::ffi::#func_ty as _ + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::#slot, + pfunc: trampoline as #pyo3_path::ffi::#func_ty as _ } }}; Ok(MethodAndSlotDef { @@ -1175,17 +1421,30 @@ fn generate_method_body( spec: &FnSpec<'_>, arguments: &[Ty], extract_error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, return_mode: Option<&ReturnMode>, + ctx: &Ctx, ) -> Result { - let self_arg = spec.tp.self_arg(Some(cls), extract_error_mode, holders); + let Ctx { pyo3_path, .. } = ctx; + let self_arg = spec + .tp + .self_arg(Some(cls), extract_error_mode, holders, ctx); let rust_name = spec.name; - let args = extract_proto_arguments(spec, arguments, extract_error_mode, holders)?; + let args = extract_proto_arguments(spec, arguments, extract_error_mode, holders, ctx)?; let call = quote! { #cls::#rust_name(#self_arg #(#args),*) }; - Ok(if let Some(return_mode) = return_mode { - return_mode.return_call_output(call) + let body = if let Some(return_mode) = return_mode { + return_mode.return_call_output(call, ctx) } else { - quote! { _pyo3::callback::convert(py, #call) } + quote! { + let result = #call; + #pyo3_path::impl_::callback::convert(py, result) + } + }; + let warnings = spec.warnings.build_py_warning(ctx); + + Ok(quote! { + #warnings + #body }) } @@ -1216,7 +1475,13 @@ impl SlotFragmentDef { self } - fn generate_pyproto_fragment(&self, cls: &syn::Type, spec: &FnSpec<'_>) -> Result { + fn generate_pyproto_fragment( + &self, + cls: &syn::Type, + spec: &FnSpec<'_>, + ctx: &Ctx, + ) -> Result { + let Ctx { pyo3_path, .. } = ctx; let SlotFragmentDef { fragment, arguments, @@ -1226,11 +1491,11 @@ impl SlotFragmentDef { let fragment_trait = format_ident!("PyClass{}SlotFragment", fragment); let method = syn::Ident::new(fragment, Span::call_site()); let wrapper_ident = format_ident!("__pymethod_{}__", fragment); - let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); + let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type(ctx)).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) .map(|i| format_ident!("arg{}", i)) .collect(); - let mut holders = Vec::new(); + let mut holders = Holders::new(); let body = generate_method_body( cls, spec, @@ -1238,29 +1503,33 @@ impl SlotFragmentDef { *extract_error_mode, &mut holders, None, + ctx, )?; - let ret_ty = ret_ty.ffi_type(); + let ret_ty = ret_ty.ffi_type(ctx); + let holders = holders.init_holders(ctx); Ok(quote! { - impl _pyo3::impl_::pyclass::#fragment_trait<#cls> for _pyo3::impl_::pyclass::PyClassImplCollector<#cls> { + impl #cls { + #[allow(non_snake_case)] + unsafe fn #wrapper_ident( + py: #pyo3_path::Python, + _raw_slf: *mut #pyo3_path::ffi::PyObject, + #(#arg_idents: #arg_types),* + ) -> #pyo3_path::PyResult<#ret_ty> { + let _slf = _raw_slf; + #holders + #body + } + } + + impl #pyo3_path::impl_::pyclass::#fragment_trait<#cls> for #pyo3_path::impl_::pyclass::PyClassImplCollector<#cls> { #[inline] unsafe fn #method( self, - py: _pyo3::Python, - _raw_slf: *mut _pyo3::ffi::PyObject, + py: #pyo3_path::Python, + _raw_slf: *mut #pyo3_path::ffi::PyObject, #(#arg_idents: #arg_types),* - ) -> _pyo3::PyResult<#ret_ty> { - impl #cls { - unsafe fn #wrapper_ident( - py: _pyo3::Python, - _raw_slf: *mut _pyo3::ffi::PyObject, - #(#arg_idents: #arg_types),* - ) -> _pyo3::PyResult<#ret_ty> { - let _slf = _raw_slf; - #( #holders )* - #body - } - } + ) -> #pyo3_path::PyResult<#ret_ty> { #cls::#wrapper_ident(py, _raw_slf, #(#arg_idents),*) } } @@ -1346,19 +1615,20 @@ fn extract_proto_arguments( spec: &FnSpec<'_>, proto_args: &[Ty], extract_error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, + ctx: &Ctx, ) -> Result> { let mut args = Vec::with_capacity(spec.signature.arguments.len()); let mut non_python_args = 0; for arg in &spec.signature.arguments { - if arg.py { + if let FnArg::Py(..) = arg { args.push(quote! { py }); } else { - let ident = syn::Ident::new(&format!("arg{}", non_python_args), Span::call_site()); + let ident = syn::Ident::new(&format!("arg{non_python_args}"), Span::call_site()); let conversions = proto_args.get(non_python_args) - .ok_or_else(|| err_spanned!(arg.ty.span() => format!("Expected at most {} non-python arguments", proto_args.len())))? - .extract(&ident, arg, extract_error_mode, holders); + .ok_or_else(|| err_spanned!(arg.ty().span() => format!("Expected at most {} non-python arguments", proto_args.len())))? + .extract(&ident, arg, extract_error_mode, holders, ctx); non_python_args += 1; args.push(conversions); } @@ -1378,10 +1648,32 @@ impl ToTokens for StaticIdent { } } -struct TokenGenerator(fn() -> TokenStream); +#[derive(Clone, Copy)] +struct TokenGenerator(fn(&Ctx) -> TokenStream); + +struct TokenGeneratorCtx<'ctx>(TokenGenerator, &'ctx Ctx); -impl ToTokens for TokenGenerator { +impl ToTokens for TokenGeneratorCtx<'_> { fn to_tokens(&self, tokens: &mut TokenStream) { - self.0().to_tokens(tokens) + let Self(TokenGenerator(gen), ctx) = self; + (gen)(ctx).to_tokens(tokens) + } +} + +pub fn field_python_name( + field: &Field, + name_attr: Option<&NameAttribute>, + renaming_rule: Option, +) -> Result { + if let Some(name_attr) = name_attr { + return Ok(name_attr.value.0.to_string()); + } + let Some(ident) = &field.ident else { + bail_spanned!(field.span() => "`get` and `set` with tuple struct fields require `name`"); + }; + let mut name = ident.unraw().to_string(); + if let Some(rule) = renaming_rule { + name = utils::apply_renaming_rule(rule, &name); } + Ok(name) } diff --git a/pyo3-macros-backend/src/pyversions.rs b/pyo3-macros-backend/src/pyversions.rs new file mode 100644 index 00000000000..3c5ac47fb84 --- /dev/null +++ b/pyo3-macros-backend/src/pyversions.rs @@ -0,0 +1,11 @@ +use pyo3_build_config::PythonVersion; + +pub fn is_abi3_before(major: u8, minor: u8) -> bool { + let config = pyo3_build_config::get(); + config.abi3 && !config.is_free_threaded() && config.version < PythonVersion { major, minor } +} + +pub fn is_py_before(major: u8, minor: u8) -> bool { + let config = pyo3_build_config::get(); + config.version < PythonVersion { major, minor } +} diff --git a/pyo3-macros-backend/src/quotes.rs b/pyo3-macros-backend/src/quotes.rs index 239036ef3ca..d961b4c0acd 100644 --- a/pyo3-macros-backend/src/quotes.rs +++ b/pyo3-macros-backend/src/quotes.rs @@ -1,21 +1,36 @@ +use crate::utils::Ctx; use proc_macro2::TokenStream; -use quote::quote; +use quote::{quote, quote_spanned}; -pub(crate) fn some_wrap(obj: TokenStream) -> TokenStream { +pub(crate) fn some_wrap(obj: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; quote! { - _pyo3::impl_::wrap::SomeWrap::wrap(#obj) + #pyo3_path::impl_::wrap::SomeWrap::wrap(#obj) } } -pub(crate) fn ok_wrap(obj: TokenStream) -> TokenStream { - quote! { - _pyo3::impl_::wrap::OkWrap::wrap(#obj) - .map_err(::core::convert::Into::<_pyo3::PyErr>::into) - } +pub(crate) fn ok_wrap(obj: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { + pyo3_path, + output_span, + } = ctx; + let pyo3_path = pyo3_path.to_tokens_spanned(*output_span); + quote_spanned! { *output_span => { + let obj = #obj; + #[allow(clippy::useless_conversion)] + #pyo3_path::impl_::wrap::converter(&obj).wrap(obj).map_err(::core::convert::Into::<#pyo3_path::PyErr>::into) + }} } -pub(crate) fn map_result_into_ptr(result: TokenStream) -> TokenStream { - quote! { - _pyo3::impl_::wrap::map_result_into_ptr(py, #result) - } +pub(crate) fn map_result_into_ptr(result: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { + pyo3_path, + output_span, + } = ctx; + let pyo3_path = pyo3_path.to_tokens_spanned(*output_span); + let py = syn::Ident::new("py", proc_macro2::Span::call_site()); + quote_spanned! { *output_span => { + let result = #result; + #pyo3_path::impl_::wrap::converter(&result).map_into_ptr(#py, result) + }} } diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index 9f0f2678476..81228cd8b69 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -1,9 +1,11 @@ +use crate::attributes::{CrateAttribute, RenamingRule}; use proc_macro2::{Span, TokenStream}; -use quote::ToTokens; +use quote::{quote, quote_spanned, ToTokens}; +use std::ffi::CString; +use syn::spanned::Spanned; +use syn::LitCStr; use syn::{punctuated::Punctuated, Token}; -use crate::attributes::{CrateAttribute, RenamingRule}; - /// Macro inspired by `anyhow::anyhow!` to create a compiler error with the given span. macro_rules! err_spanned { ($span:expr => $msg:expr) => { @@ -25,7 +27,20 @@ macro_rules! ensure_spanned { if !($condition) { bail_spanned!($span => $msg); } - } + }; + ($($condition:expr, $span:expr => $msg:expr;)*) => { + if let Some(e) = [$( + (!($condition)).then(|| err_spanned!($span => $msg)), + )*] + .into_iter() + .flatten() + .reduce(|mut acc, e| { + acc.combine(e); + acc + }) { + return Err(e); + } + }; } /// Check if the given type `ty` is `pyo3::Python`. @@ -58,17 +73,29 @@ pub fn option_type_argument(ty: &syn::Type) -> Option<&syn::Type> { /// /// Typically the tokens will just be that string, but if the original docs included macro /// expressions then the tokens will be a concat!("...", "\n", "\0") expression of the strings and -/// macro parts. -/// contents such as parse the string contents. +/// macro parts. contents such as parse the string contents. +#[derive(Clone)] +pub struct PythonDoc(PythonDocKind); + #[derive(Clone)] -pub struct PythonDoc(TokenStream); +enum PythonDocKind { + LitCStr(LitCStr), + // There is currently no way to `concat!` c-string literals, we fallback to the `c_str!` macro in + // this case. + Tokens(TokenStream), +} /// Collects all #[doc = "..."] attributes into a TokenStream evaluating to a null-terminated string. /// /// If this doc is for a callable, the provided `text_signature` can be passed to prepend /// this to the documentation suitable for Python to extract this into the `__text_signature__` /// attribute. -pub fn get_doc(attrs: &[syn::Attribute], mut text_signature: Option) -> PythonDoc { +pub fn get_doc( + attrs: &[syn::Attribute], + mut text_signature: Option, + ctx: &Ctx, +) -> syn::Result { + let Ctx { pyo3_path, .. } = ctx; // insert special divider between `__text_signature__` and doc // (assume text_signature is itself well-formed) if let Some(text_signature) = &mut text_signature { @@ -78,10 +105,15 @@ pub fn get_doc(attrs: &[syn::Attribute], mut text_signature: Option) -> let mut parts = Punctuated::::new(); let mut first = true; let mut current_part = text_signature.unwrap_or_default(); + let mut current_part_span = None; for attr in attrs { if attr.path().is_ident("doc") { if let Ok(nv) = attr.meta.require_name_value() { + current_part_span = match current_part_span { + None => Some(nv.value.span()), + Some(span) => span.join(nv.value.span()), + }; if !first { current_part.push('\n'); } else { @@ -99,7 +131,7 @@ pub fn get_doc(attrs: &[syn::Attribute], mut text_signature: Option) -> } else { // This is probably a macro doc from Rust 1.54, e.g. #[doc = include_str!(...)] // Reset the string buffer, write that part, and then push this macro part too. - parts.push(current_part.to_token_stream()); + parts.push(quote_spanned!(current_part_span.unwrap_or(Span::call_site()) => #current_part)); current_part.clear(); parts.push(nv.value.to_token_stream()); } @@ -110,7 +142,9 @@ pub fn get_doc(attrs: &[syn::Attribute], mut text_signature: Option) -> if !parts.is_empty() { // Doc contained macro pieces - return as `concat!` expression if !current_part.is_empty() { - parts.push(current_part.to_token_stream()); + parts.push( + quote_spanned!(current_part_span.unwrap_or(Span::call_site()) => #current_part), + ); } let mut tokens = TokenStream::new(); @@ -120,20 +154,35 @@ pub fn get_doc(attrs: &[syn::Attribute], mut text_signature: Option) -> syn::token::Bracket(Span::call_site()).surround(&mut tokens, |tokens| { parts.to_tokens(tokens); syn::token::Comma(Span::call_site()).to_tokens(tokens); - syn::LitStr::new("\0", Span::call_site()).to_tokens(tokens); }); - PythonDoc(tokens) + Ok(PythonDoc(PythonDocKind::Tokens( + quote!(#pyo3_path::ffi::c_str!(#tokens)), + ))) } else { // Just a string doc - return directly with nul terminator - current_part.push('\0'); - PythonDoc(current_part.to_token_stream()) + let docs = CString::new(current_part).map_err(|e| { + syn::Error::new( + current_part_span.unwrap_or(Span::call_site()), + format!( + "Python doc may not contain nul byte, found nul at position {}", + e.nul_position() + ), + ) + })?; + Ok(PythonDoc(PythonDocKind::LitCStr(LitCStr::new( + &docs, + current_part_span.unwrap_or(Span::call_site()), + )))) } } impl quote::ToTokens for PythonDoc { fn to_tokens(&self, tokens: &mut TokenStream) { - self.0.to_tokens(tokens) + match &self.0 { + PythonDocKind::LitCStr(lit) => lit.to_tokens(tokens), + PythonDocKind::Tokens(toks) => toks.to_tokens(tokens), + } } } @@ -144,11 +193,60 @@ pub fn unwrap_ty_group(mut ty: &syn::Type) -> &syn::Type { ty } -/// Extract the path to the pyo3 crate, or use the default (`::pyo3`). -pub(crate) fn get_pyo3_crate(attr: &Option) -> syn::Path { - match attr { - Some(attr) => attr.value.0.clone(), - None => syn::parse_str("::pyo3").unwrap(), +pub struct Ctx { + /// Where we can find the pyo3 crate + pub pyo3_path: PyO3CratePath, + + /// If we are in a pymethod or pyfunction, + /// this will be the span of the return type + pub output_span: Span, +} + +impl Ctx { + pub(crate) fn new(attr: &Option, signature: Option<&syn::Signature>) -> Self { + let pyo3_path = match attr { + Some(attr) => PyO3CratePath::Given(attr.value.0.clone()), + None => PyO3CratePath::Default, + }; + + let output_span = if let Some(syn::Signature { + output: syn::ReturnType::Type(_, output_type), + .. + }) = &signature + { + output_type.span() + } else { + Span::call_site() + }; + + Self { + pyo3_path, + output_span, + } + } +} + +#[derive(Clone)] +pub enum PyO3CratePath { + Given(syn::Path), + Default, +} + +impl PyO3CratePath { + pub fn to_tokens_spanned(&self, span: Span) -> TokenStream { + match self { + Self::Given(path) => quote::quote_spanned! { span => #path }, + Self::Default => quote::quote_spanned! { span => ::pyo3 }, + } + } +} + +impl quote::ToTokens for PyO3CratePath { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Given(path) => path.to_tokens(tokens), + Self::Default => quote::quote! { ::pyo3 }.to_tokens(tokens), + } } } @@ -167,6 +265,66 @@ pub fn apply_renaming_rule(rule: RenamingRule, name: &str) -> String { } } -pub(crate) fn is_abi3() -> bool { - pyo3_build_config::get().abi3 +pub(crate) enum IdentOrStr<'a> { + Str(&'a str), + Ident(syn::Ident), +} + +pub(crate) fn has_attribute(attrs: &[syn::Attribute], ident: &str) -> bool { + has_attribute_with_namespace(attrs, None, &[ident]) +} + +pub(crate) fn has_attribute_with_namespace( + attrs: &[syn::Attribute], + crate_path: Option<&PyO3CratePath>, + idents: &[&str], +) -> bool { + let mut segments = vec![]; + if let Some(c) = crate_path { + match c { + PyO3CratePath::Given(paths) => { + for p in &paths.segments { + segments.push(IdentOrStr::Ident(p.ident.clone())); + } + } + PyO3CratePath::Default => segments.push(IdentOrStr::Str("pyo3")), + } + }; + for i in idents { + segments.push(IdentOrStr::Str(i)); + } + + attrs.iter().any(|attr| { + segments + .iter() + .eq(attr.path().segments.iter().map(|v| &v.ident)) + }) +} + +pub fn expr_to_python(expr: &syn::Expr) -> String { + match expr { + // literal values + syn::Expr::Lit(syn::ExprLit { lit, .. }) => match lit { + syn::Lit::Str(s) => s.token().to_string(), + syn::Lit::Char(c) => c.token().to_string(), + syn::Lit::Int(i) => i.base10_digits().to_string(), + syn::Lit::Float(f) => f.base10_digits().to_string(), + syn::Lit::Bool(b) => { + if b.value() { + "True".to_string() + } else { + "False".to_string() + } + } + _ => "...".to_string(), + }, + // None + syn::Expr::Path(syn::ExprPath { qself, path, .. }) + if qself.is_none() && path.is_ident("None") => + { + "None".to_string() + } + // others, unsupported yet so defaults to `...` + _ => "...".to_string(), + } } diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 576c94a2bc1..604a5bbaac7 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.21.0-dev" +version = "0.27.0" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -9,18 +9,21 @@ repository = "https://github.com/pyo3/pyo3" categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" edition = "2021" +rust-version.workspace = true [lib] proc-macro = true [features] multiple-pymethods = [] +experimental-async = ["pyo3-macros-backend/experimental-async"] +experimental-inspect = ["pyo3-macros-backend/experimental-inspect"] [dependencies] -proc-macro2 = { version = "1", default-features = false } +proc-macro2 = { version = "1.0.60", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.0-dev" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.27.0" } [lints] workspace = true diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index d00ede89143..6e4a46ee95e 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -1,16 +1,16 @@ //! This crate declares only the proc macro attributes, as a crate defining proc macro attributes //! must not contain any other public items. -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use pyo3_macros_backend::{ - build_derive_from_pyobject, build_py_class, build_py_enum, build_py_function, build_py_methods, - get_doc, process_functions_in_module, pymodule_impl, PyClassArgs, PyClassMethodsType, - PyFunctionOptions, PyModuleOptions, + build_derive_from_pyobject, build_derive_into_pyobject, build_py_class, build_py_enum, + build_py_function, build_py_methods, pymodule_function_impl, pymodule_module_impl, PyClassArgs, + PyClassMethodsType, PyFunctionOptions, PyModuleOptions, }; use quote::quote; -use syn::{parse::Nothing, parse_macro_input}; +use syn::{parse_macro_input, Item}; /// A proc macro used to implement Python modules. /// @@ -24,6 +24,9 @@ use syn::{parse::Nothing, parse_macro_input}; /// | Annotation | Description | /// | :- | :- | /// | `#[pyo3(name = "...")]` | Defines the name of the module in Python. | +/// | `#[pyo3(submodule)]` | Skips adding a `PyInit_` FFI symbol to the compiled binary. | +/// | `#[pyo3(module = "...")]` | Defines the Python `dotted.path` to the parent module for use in introspection. | +/// | `#[pyo3(crate = "pyo3")]` | Defines the path to PyO3 to use code generated by the macro. | /// /// For more on creating Python modules see the [module section of the guide][1]. /// @@ -32,24 +35,27 @@ use syn::{parse::Nothing, parse_macro_input}; /// `#[pymodule]` implementation generates a hidden module with the same name containing /// metadata about the module, which is used by `wrap_pymodule!`). /// -/// [1]: https://pyo3.rs/latest/module.html +#[doc = concat!("[1]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/module.html")] #[proc_macro_attribute] pub fn pymodule(args: TokenStream, input: TokenStream) -> TokenStream { - parse_macro_input!(args as Nothing); - - let mut ast = parse_macro_input!(input as syn::ItemFn); - let options = match PyModuleOptions::from_attrs(&mut ast.attrs) { - Ok(options) => options, - Err(e) => return e.into_compile_error().into(), - }; - - if let Err(err) = process_functions_in_module(&options, &mut ast) { - return err.into_compile_error().into(); + let options = parse_macro_input!(args as PyModuleOptions); + + let mut ast = parse_macro_input!(input as Item); + let expanded = match &mut ast { + Item::Mod(module) => { + match pymodule_module_impl(module, options) { + // #[pymodule] on a module will rebuild the original ast, so we don't emit it here + Ok(expanded) => return expanded.into(), + Err(e) => Err(e), + } + } + Item::Fn(function) => pymodule_function_impl(function, options), + unsupported => Err(syn::Error::new_spanned( + unsupported, + "#[pymodule] only supports modules and functions.", + )), } - - let doc = get_doc(&ast.attrs, None); - - let expanded = pymodule_impl(&ast.sig.ident, options, doc, &ast.vis); + .unwrap_or_compile_error(); quote!( #ast @@ -60,7 +66,6 @@ pub fn pymodule(args: TokenStream, input: TokenStream) -> TokenStream { #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { - use syn::Item; let item = parse_macro_input!(input as Item); match item { Item::Struct(struct_) => pyclass_impl(attr, struct_, methods_type()), @@ -80,12 +85,12 @@ pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { /// | Annotation | Description | /// | :- | :- | /// | [`#[new]`][4] | Defines the class constructor, like Python's `__new__` method. | -/// | [`#[getter]`][5] and [`#[setter]`][5] | These define getters and setters, similar to Python's `@property` decorator. This is useful for getters/setters that require computation or side effects; if that is not the case consider using [`#[pyo3(get, set)]`][11] on the struct's field(s).| +/// | [`#[getter]`][5] and [`#[setter]`][5] | These define getters and setters, similar to Python's `@property` decorator. This is useful for getters/setters that require computation or side effects; if that is not the case consider using [`#[pyo3(get, set)]`][12] on the struct's field(s).| /// | [`#[staticmethod]`][6]| Defines the method as a staticmethod, like Python's `@staticmethod` decorator.| /// | [`#[classmethod]`][7] | Defines the method as a classmethod, like Python's `@classmethod` decorator.| /// | [`#[classattr]`][9] | Defines a class variable. | /// | [`#[args]`][10] | Deprecated way to define a method's default arguments and allows the function to receive `*args` and `**kwargs`. Use `#[pyo3(signature = (...))]` instead. | -/// | [`#[pyo3( | Any of the `#[pyo3]` options supported on [`macro@pyfunction`]. | +/// | [`#[pyo3( | Any of the `#[pyo3]` options supported on [`macro@pyfunction`]. | /// /// For more on creating class methods, /// see the [class section of the guide][1]. @@ -94,17 +99,18 @@ pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { /// multiple `#[pymethods]` blocks for a single `#[pyclass]`. /// This will add a transitive dependency on the [`inventory`][3] crate. /// -/// [1]: https://pyo3.rs/latest/class.html#instance-methods -/// [2]: https://pyo3.rs/latest/features.html#multiple-pymethods +#[doc = concat!("[1]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#instance-methods")] +#[doc = concat!("[2]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#multiple-pymethods")] /// [3]: https://docs.rs/inventory/ -/// [4]: https://pyo3.rs/latest/class.html#constructor -/// [5]: https://pyo3.rs/latest/class.html#object-properties-using-getter-and-setter -/// [6]: https://pyo3.rs/latest/class.html#static-methods -/// [7]: https://pyo3.rs/latest/class.html#class-methods -/// [8]: https://pyo3.rs/latest/class.html#callable-objects -/// [9]: https://pyo3.rs/latest/class.html#class-attributes -/// [10]: https://pyo3.rs/latest/class.html#method-arguments -/// [11]: https://pyo3.rs/latest/class.html#object-properties-using-pyo3get-set +#[doc = concat!("[4]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#constructor")] +#[doc = concat!("[5]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#object-properties-using-getter-and-setter")] +#[doc = concat!("[6]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#static-methods")] +#[doc = concat!("[7]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#class-methods")] +#[doc = concat!("[8]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#callable-objects")] +#[doc = concat!("[9]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#class-attributes")] +#[doc = concat!("[10]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#method-arguments")] +#[doc = concat!("[11]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/function.html#function-options")] +#[doc = concat!("[12]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/class.html#object-properties-using-pyo3get-set")] #[proc_macro_attribute] pub fn pymethods(attr: TokenStream, input: TokenStream) -> TokenStream { let methods_type = if cfg!(feature = "multiple-pymethods") { @@ -125,6 +131,7 @@ pub fn pymethods(attr: TokenStream, input: TokenStream) -> TokenStream { /// | `#[pyo3(name = "...")]` | Defines the name of the function in Python. | /// | `#[pyo3(text_signature = "...")]` | Defines the `__text_signature__` attribute of the function in Python. | /// | `#[pyo3(pass_module)]` | Passes the module containing the function as a `&PyModule` first argument to the function. | +/// | `#[pyo3(warn(message = "...", category = ...))]` | Generate warning given a message and a category | /// /// For more on exposing functions see the [function section of the guide][1]. /// @@ -133,7 +140,7 @@ pub fn pymethods(attr: TokenStream, input: TokenStream) -> TokenStream { /// `#[pyfunction]` implementation generates a hidden module with the same name containing /// metadata about the function, which is used by `wrap_pyfunction!`). /// -/// [1]: https://pyo3.rs/latest/function.html +#[doc = concat!("[1]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/function.html")] #[proc_macro_attribute] pub fn pyfunction(attr: TokenStream, input: TokenStream) -> TokenStream { let mut ast = parse_macro_input!(input as syn::ItemFn); @@ -148,6 +155,27 @@ pub fn pyfunction(attr: TokenStream, input: TokenStream) -> TokenStream { .into() } +#[proc_macro_derive(IntoPyObject, attributes(pyo3))] +pub fn derive_into_py_object(item: TokenStream) -> TokenStream { + let ast = parse_macro_input!(item as syn::DeriveInput); + let expanded = build_derive_into_pyobject::(&ast).unwrap_or_compile_error(); + quote!( + #expanded + ) + .into() +} + +#[proc_macro_derive(IntoPyObjectRef, attributes(pyo3))] +pub fn derive_into_py_object_ref(item: TokenStream) -> TokenStream { + let ast = parse_macro_input!(item as syn::DeriveInput); + let expanded = + pyo3_macros_backend::build_derive_into_pyobject::(&ast).unwrap_or_compile_error(); + quote!( + #expanded + ) + .into() +} + #[proc_macro_derive(FromPyObject, attributes(pyo3))] pub fn derive_from_py_object(item: TokenStream) -> TokenStream { let ast = parse_macro_input!(item as syn::DeriveInput); @@ -163,7 +191,7 @@ fn pyclass_impl( mut ast: syn::ItemStruct, methods_type: PyClassMethodsType, ) -> TokenStream { - let args = parse_macro_input!(attrs with PyClassArgs::parse_stuct_args); + let args = parse_macro_input!(attrs with PyClassArgs::parse_struct_args); let expanded = build_py_class(&mut ast, args, methods_type).unwrap_or_compile_error(); quote!( diff --git a/pyproject.toml b/pyproject.toml index 866645d2ffc..2b9915ffb45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ -[tool.ruff.extend-per-file-ignores] +[tool.ruff.lint.extend-per-file-ignores] "__init__.py" = ["F403"] [tool.towncrier] filename = "CHANGELOG.md" -version = "0.21.0-dev" +version = "0.27.0" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" @@ -24,3 +24,32 @@ name = "Removed" [tool.towncrier.fragment.fixed] name = "Fixed" + +[tool.rumdl] + +disable = [ + # TODO: what to do about inline HTML, probably allow? + "MD033", + # TODO: {{#PYO3_DOCS_URL}} placeholder confuses rumdl, change syntax perhaps? + "MD051" +] + +exclude = [ + # just has an include to the top-level files + "guide/src/changelog.md", + "guide/src/contributing.md" +] + +[tool.rumdl.per-file-ignores] +"guide/pyclass-parameters.md" = ["MD041"] + +[tool.rumdl.MD004] +style = "dash" + +[tool.rumdl.MD013] +paragraphs = false +code_blocks = false +tables = false +headings = false +reflow = true +reflow_mode = "sentence-per-line" diff --git a/pytests/Cargo.toml b/pytests/Cargo.toml index 255094a6c40..152500792df 100644 --- a/pytests/Cargo.toml +++ b/pytests/Cargo.toml @@ -5,9 +5,13 @@ version = "0.1.0" description = "Python-based tests for PyO3" edition = "2021" publish = false +rust-version = "1.83" + +[features] +experimental-inspect = ["pyo3/experimental-inspect"] [dependencies] -pyo3 = { path = "../", features = ["extension-module"] } +pyo3.path = "../" [build-dependencies] pyo3-build-config = { path = "../pyo3-build-config" } diff --git a/pytests/README.md b/pytests/README.md index 7ced072aa36..1016baa7209 100644 --- a/pytests/README.md +++ b/pytests/README.md @@ -2,6 +2,9 @@ An extension module built using PyO3, used to test and benchmark PyO3 from Python. +The `stubs` directory contains Python stubs used to test the automated stubs introspection. +To test them run `nox -s test-introspection`. + ## Testing This package is intended to be built using `maturin`. Once built, you can run the tests using `pytest`: diff --git a/pytests/conftest.py b/pytests/conftest.py new file mode 100644 index 00000000000..ce729689355 --- /dev/null +++ b/pytests/conftest.py @@ -0,0 +1,22 @@ +import sysconfig +import sys +import pytest + +FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + +gil_enabled_at_start = True +if FREE_THREADED_BUILD: + gil_enabled_at_start = sys._is_gil_enabled() + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + if FREE_THREADED_BUILD and not gil_enabled_at_start and sys._is_gil_enabled(): + tr = terminalreporter + tr.ensure_newline() + tr.section("GIL re-enabled", sep="=", red=True, bold=True) + tr.line("The GIL was re-enabled at runtime during the tests.") + tr.line("") + tr.line("Please ensure all new modules declare support for running") + tr.line("without the GIL. Any new tests that intentionally imports ") + tr.line("code that re-enables the GIL should do so in a subprocess.") + pytest.exit("GIL re-enabled during tests", returncode=1) diff --git a/pytests/noxfile.py b/pytests/noxfile.py index 7c681ab1aa8..2bc9c0366d1 100644 --- a/pytests/noxfile.py +++ b/pytests/noxfile.py @@ -8,12 +8,20 @@ @nox.session def test(session: nox.Session): session.env["MATURIN_PEP517_ARGS"] = "--profile=dev" - session.run_always("python", "-m", "pip", "install", "-v", ".[dev]") - try: - session.install("--only-binary=numpy", "numpy>=1.16") - except CommandFailed: - # No binary wheel for numpy available on this platform - pass + session.install("-v", ".[dev]") + + def try_install_binary(package: str, constraint: str): + try: + session.install("--only-binary=:all:", f"{package}{constraint}") + except CommandFailed: + # No binary wheel available on this platform + pass + + try_install_binary("numpy", ">=1.16") + # https://github.com/zopefoundation/zope.interface/issues/316 + # - is a dependency of gevent + try_install_binary("zope.interface", "<7") + try_install_binary("gevent", ">=22.10.2") ignored_paths = [] if sys.version_info < (3, 10): # Match syntax is only available in Python >= 3.10 diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index 126eaf77b09..32ccec55604 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=1,<2"] +requires = ["maturin>=1.9.4,<2"] build-backend = "maturin" [tool.pytest.ini_options] @@ -20,10 +20,9 @@ classifiers = [ [project.optional-dependencies] dev = [ - "gevent>=22.10.2; implementation_name == 'cpython'", "hypothesis>=3.55", - "pytest-asyncio>=0.21", + "pytest-asyncio>=0.21,<2", "pytest-benchmark>=3.4", - "pytest>=6.0", + "pytest>=7", "typing_extensions>=4.0.0" ] diff --git a/pytests/src/awaitable.rs b/pytests/src/awaitable.rs index e1a70b42bb0..8481e2f6528 100644 --- a/pytests/src/awaitable.rs +++ b/pytests/src/awaitable.rs @@ -2,7 +2,7 @@ //! awaitable protocol. //! //! Both IterAwaitable and FutureAwaitable will return a value immediately -//! when awaited, see guide examples related to pyo3-asyncio for ways +//! when awaited, see guide examples related to pyo3-async-runtimes for ways //! to suspend tasks and await results. use pyo3::exceptions::PyStopIteration; @@ -11,13 +11,13 @@ use pyo3::prelude::*; #[pyclass] #[derive(Debug)] pub(crate) struct IterAwaitable { - result: Option>, + result: Option>>, } #[pymethods] impl IterAwaitable { #[new] - fn new(result: PyObject) -> Self { + fn new(result: Py) -> Self { IterAwaitable { result: Some(Ok(result)), } @@ -31,7 +31,7 @@ impl IterAwaitable { pyself } - fn __next__(&mut self, py: Python<'_>) -> PyResult { + fn __next__(&mut self, py: Python<'_>) -> PyResult> { match self.result.take() { Some(res) => match res { Ok(v) => Err(PyStopIteration::new_err(v)), @@ -46,13 +46,13 @@ impl IterAwaitable { pub(crate) struct FutureAwaitable { #[pyo3(get, set, name = "_asyncio_future_blocking")] py_block: bool, - result: Option>, + result: Option>>, } #[pymethods] impl FutureAwaitable { #[new] - fn new(result: PyObject) -> Self { + fn new(result: Py) -> Self { FutureAwaitable { py_block: false, result: Some(Ok(result)), @@ -78,8 +78,8 @@ impl FutureAwaitable { } } -#[pymodule] -pub fn awaitable(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(gil_used = false)] +pub fn awaitable(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; Ok(()) diff --git a/pytests/src/buf_and_str.rs b/pytests/src/buf_and_str.rs index 23db9f0625e..15230a5e153 100644 --- a/pytests/src/buf_and_str.rs +++ b/pytests/src/buf_and_str.rs @@ -17,38 +17,38 @@ impl BytesExtractor { } #[staticmethod] - pub fn from_bytes(bytes: &PyBytes) -> PyResult { + pub fn from_bytes(bytes: &Bound<'_, PyBytes>) -> PyResult { let byte_vec: Vec = bytes.extract()?; Ok(byte_vec.len()) } #[staticmethod] - pub fn from_str(string: &PyString) -> PyResult { + pub fn from_str(string: &Bound<'_, PyString>) -> PyResult { let rust_string: String = string.extract()?; Ok(rust_string.len()) } #[staticmethod] - pub fn from_str_lossy(string: &PyString) -> usize { + pub fn from_str_lossy(string: &Bound<'_, PyString>) -> usize { let rust_string_lossy: String = string.to_string_lossy().to_string(); rust_string_lossy.len() } #[staticmethod] pub fn from_buffer(buf: &Bound<'_, PyAny>) -> PyResult { - let buf = PyBuffer::::get_bound(buf)?; + let buf = PyBuffer::::get(buf)?; Ok(buf.item_count()) } } #[pyfunction] fn return_memoryview(py: Python<'_>) -> PyResult> { - let bytes = PyBytes::new_bound(py, b"hello world"); - PyMemoryView::from_bound(&bytes) + let bytes = PyBytes::new(py, b"hello world"); + PyMemoryView::from(&bytes) } -#[pymodule] -pub fn buf_and_str(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(gil_used = false)] +pub fn buf_and_str(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_function(wrap_pyfunction!(return_memoryview, m)?)?; Ok(()) diff --git a/pytests/src/comparisons.rs b/pytests/src/comparisons.rs index d8c2f5a6a52..df28b423c3c 100644 --- a/pytests/src/comparisons.rs +++ b/pytests/src/comparisons.rs @@ -1,7 +1,8 @@ +use pyo3::basic::CompareOp; use pyo3::prelude::*; -use pyo3::{types::PyModule, Python}; +use std::fmt; -#[pyclass] +#[pyclass(frozen)] struct Eq(i64); #[pymethods] @@ -20,7 +21,7 @@ impl Eq { } } -#[pyclass] +#[pyclass(frozen)] struct EqDefaultNe(i64); #[pymethods] @@ -35,7 +36,19 @@ impl EqDefaultNe { } } -#[pyclass] +#[pyclass(eq, frozen)] +#[derive(PartialEq, Eq)] +struct EqDerived(i64); + +#[pymethods] +impl EqDerived { + #[new] + fn new(value: i64) -> Self { + Self(value) + } +} + +#[pyclass(frozen)] struct Ordered(i64); #[pymethods] @@ -70,7 +83,40 @@ impl Ordered { } } -#[pyclass] +#[pyclass(frozen)] +struct OrderedRichCmp(i64); + +#[pymethods] +impl OrderedRichCmp { + #[new] + fn new(value: i64) -> Self { + Self(value) + } + + fn __richcmp__(&self, other: &Self, op: CompareOp) -> bool { + op.matches(self.0.cmp(&other.0)) + } +} + +#[pyclass(eq, ord, hash, str, frozen)] +#[derive(PartialEq, Eq, Ord, PartialOrd, Hash)] +struct OrderedDerived(i64); + +impl fmt::Display for OrderedDerived { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[pymethods] +impl OrderedDerived { + #[new] + fn new(value: i64) -> Self { + Self(value) + } +} + +#[pyclass(frozen)] struct OrderedDefaultNe(i64); #[pymethods] @@ -101,11 +147,10 @@ impl OrderedDefaultNe { } } -#[pymodule] -pub fn comparisons(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - Ok(()) +#[pymodule(gil_used = false)] +pub mod comparisons { + #[pymodule_export] + use super::{ + Eq, EqDefaultNe, EqDerived, Ordered, OrderedDefaultNe, OrderedDerived, OrderedRichCmp, + }; } diff --git a/pytests/src/consts.rs b/pytests/src/consts.rs new file mode 100644 index 00000000000..eec09a71819 --- /dev/null +++ b/pytests/src/consts.rs @@ -0,0 +1,21 @@ +use pyo3::pymodule; + +#[pymodule] +pub mod consts { + use pyo3::{pyclass, pymethods}; + + #[pymodule_export] + pub const PI: f64 = std::f64::consts::PI; // Exports PI constant as part of the module + + #[pymodule_export] + pub const SIMPLE: &str = "SIMPLE"; + + #[pyclass] + struct ClassWithConst {} + + #[pymethods] + impl ClassWithConst { + #[classattr] + const INSTANCE: Self = ClassWithConst {}; + } +} diff --git a/pytests/src/datetime.rs b/pytests/src/datetime.rs index 9d8f32a93c9..5162b3508a5 100644 --- a/pytests/src/datetime.rs +++ b/pytests/src/datetime.rs @@ -8,20 +8,24 @@ use pyo3::types::{ #[pyfunction] fn make_date(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult> { - PyDate::new_bound(py, year, month, day) + PyDate::new(py, year, month, day) } #[pyfunction] -fn get_date_tuple<'p>(py: Python<'p>, d: &PyDate) -> Bound<'p, PyTuple> { - PyTuple::new_bound(py, [d.get_year(), d.get_month() as i32, d.get_day() as i32]) +fn get_date_tuple<'py>(d: &Bound<'py, PyDate>) -> PyResult> { + PyTuple::new( + d.py(), + [d.get_year(), d.get_month() as i32, d.get_day() as i32], + ) } #[pyfunction] fn date_from_timestamp(py: Python<'_>, timestamp: i64) -> PyResult> { - PyDate::from_timestamp_bound(py, timestamp) + PyDate::from_timestamp(py, timestamp) } #[pyfunction] +#[pyo3(signature=(hour, minute, second, microsecond, tzinfo=None))] fn make_time<'py>( py: Python<'py>, hour: u8, @@ -30,7 +34,7 @@ fn make_time<'py>( microsecond: u32, tzinfo: Option<&Bound<'py, PyTzInfo>>, ) -> PyResult> { - PyTime::new_bound(py, hour, minute, second, microsecond, tzinfo) + PyTime::new(py, hour, minute, second, microsecond, tzinfo) } #[pyfunction] @@ -44,13 +48,13 @@ fn time_with_fold<'py>( tzinfo: Option<&Bound<'py, PyTzInfo>>, fold: bool, ) -> PyResult> { - PyTime::new_bound_with_fold(py, hour, minute, second, microsecond, tzinfo, fold) + PyTime::new_with_fold(py, hour, minute, second, microsecond, tzinfo, fold) } #[pyfunction] -fn get_time_tuple<'p>(py: Python<'p>, dt: &PyTime) -> Bound<'p, PyTuple> { - PyTuple::new_bound( - py, +fn get_time_tuple<'py>(dt: &Bound<'py, PyTime>) -> PyResult> { + PyTuple::new( + dt.py(), [ dt.get_hour() as u32, dt.get_minute() as u32, @@ -61,9 +65,9 @@ fn get_time_tuple<'p>(py: Python<'p>, dt: &PyTime) -> Bound<'p, PyTuple> { } #[pyfunction] -fn get_time_tuple_fold<'p>(py: Python<'p>, dt: &PyTime) -> Bound<'p, PyTuple> { - PyTuple::new_bound( - py, +fn get_time_tuple_fold<'py>(dt: &Bound<'py, PyTime>) -> PyResult> { + PyTuple::new( + dt.py(), [ dt.get_hour() as u32, dt.get_minute() as u32, @@ -81,12 +85,12 @@ fn make_delta( seconds: i32, microseconds: i32, ) -> PyResult> { - PyDelta::new_bound(py, days, seconds, microseconds, true) + PyDelta::new(py, days, seconds, microseconds, true) } #[pyfunction] -fn get_delta_tuple<'py>(delta: &Bound<'py, PyDelta>) -> Bound<'py, PyTuple> { - PyTuple::new_bound( +fn get_delta_tuple<'py>(delta: &Bound<'py, PyDelta>) -> PyResult> { + PyTuple::new( delta.py(), [ delta.get_days(), @@ -98,6 +102,7 @@ fn get_delta_tuple<'py>(delta: &Bound<'py, PyDelta>) -> Bound<'py, PyTuple> { #[allow(clippy::too_many_arguments)] #[pyfunction] +#[pyo3(signature=(year, month, day, hour, minute, second, microsecond, tzinfo=None))] fn make_datetime<'py>( py: Python<'py>, year: i32, @@ -109,7 +114,7 @@ fn make_datetime<'py>( microsecond: u32, tzinfo: Option<&Bound<'py, PyTzInfo>>, ) -> PyResult> { - PyDateTime::new_bound( + PyDateTime::new( py, year, month, @@ -123,9 +128,9 @@ fn make_datetime<'py>( } #[pyfunction] -fn get_datetime_tuple<'py>(py: Python<'py>, dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTuple> { - PyTuple::new_bound( - py, +fn get_datetime_tuple<'py>(dt: &Bound<'py, PyDateTime>) -> PyResult> { + PyTuple::new( + dt.py(), [ dt.get_year(), dt.get_month() as i32, @@ -139,12 +144,9 @@ fn get_datetime_tuple<'py>(py: Python<'py>, dt: &Bound<'py, PyDateTime>) -> Boun } #[pyfunction] -fn get_datetime_tuple_fold<'py>( - py: Python<'py>, - dt: &Bound<'py, PyDateTime>, -) -> Bound<'py, PyTuple> { - PyTuple::new_bound( - py, +fn get_datetime_tuple_fold<'py>(dt: &Bound<'py, PyDateTime>) -> PyResult> { + PyTuple::new( + dt.py(), [ dt.get_year(), dt.get_month() as i32, @@ -159,22 +161,23 @@ fn get_datetime_tuple_fold<'py>( } #[pyfunction] +#[pyo3(signature=(ts, tz=None))] fn datetime_from_timestamp<'py>( py: Python<'py>, ts: f64, tz: Option<&Bound<'py, PyTzInfo>>, ) -> PyResult> { - PyDateTime::from_timestamp_bound(py, ts, tz) + PyDateTime::from_timestamp(py, ts, tz) } #[pyfunction] fn get_datetime_tzinfo<'py>(dt: &Bound<'py, PyDateTime>) -> Option> { - dt.get_tzinfo_bound() + dt.get_tzinfo() } #[pyfunction] fn get_time_tzinfo<'py>(dt: &Bound<'py, PyTime>) -> Option> { - dt.get_tzinfo_bound() + dt.get_tzinfo() } #[pyclass(extends=PyTzInfo)] @@ -187,12 +190,8 @@ impl TzClass { TzClass {} } - fn utcoffset<'py>( - &self, - py: Python<'py>, - _dt: &Bound<'py, PyDateTime>, - ) -> PyResult> { - PyDelta::new_bound(py, 0, 3600, 0, true) + fn utcoffset<'py>(&self, dt: &Bound<'py, PyDateTime>) -> PyResult> { + PyDelta::new(dt.py(), 0, 3600, 0, true) } fn tzname(&self, _dt: &Bound<'_, PyDateTime>) -> String { @@ -204,8 +203,8 @@ impl TzClass { } } -#[pymodule] -pub fn datetime(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(gil_used = false)] +pub fn datetime(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(make_date, m)?)?; m.add_function(wrap_pyfunction!(get_date_tuple, m)?)?; m.add_function(wrap_pyfunction!(date_from_timestamp, m)?)?; diff --git a/pytests/src/dict_iter.rs b/pytests/src/dict_iter.rs index 5f5992b6efc..c312fbb5f83 100644 --- a/pytests/src/dict_iter.rs +++ b/pytests/src/dict_iter.rs @@ -3,7 +3,7 @@ use pyo3::prelude::*; use pyo3::types::PyDict; #[pymodule] -pub fn dict_iter(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn dict_iter(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } @@ -20,7 +20,7 @@ impl DictSize { DictSize { expected } } - fn iter_dict(&mut self, _py: Python<'_>, dict: &PyDict) -> PyResult { + fn iter_dict(&mut self, _py: Python<'_>, dict: &Bound<'_, PyDict>) -> PyResult { let mut seen = 0u32; for (sym, values) in dict { seen += 1; diff --git a/pytests/src/enums.rs b/pytests/src/enums.rs index 11b592d3563..02431fdd1e8 100644 --- a/pytests/src/enums.rs +++ b/pytests/src/enums.rs @@ -1,15 +1,16 @@ -use pyo3::{pyclass, pyfunction, pymodule, types::PyModule, wrap_pyfunction, PyResult, Python}; +use pyo3::{pyclass, pyfunction, pymodule}; -#[pymodule] -pub fn enums(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_wrapped(wrap_pyfunction!(do_simple_stuff))?; - m.add_wrapped(wrap_pyfunction!(do_complex_stuff))?; - Ok(()) +#[pymodule(gil_used = false)] +pub mod enums { + #[pymodule_export] + use super::{ + do_complex_stuff, do_mixed_complex_stuff, do_simple_stuff, do_tuple_stuff, ComplexEnum, + MixedComplexEnum, SimpleEnum, SimpleTupleEnum, TupleEnum, + }; } -#[pyclass] +#[pyclass(eq, eq_int)] +#[derive(PartialEq)] pub enum SimpleEnum { Sunday, Monday, @@ -35,11 +36,26 @@ pub fn do_simple_stuff(thing: &SimpleEnum) -> SimpleEnum { #[pyclass] pub enum ComplexEnum { - Int { i: i32 }, - Float { f: f64 }, - Str { s: String }, + Int { + i: i32, + }, + Float { + f: f64, + }, + Str { + s: String, + }, EmptyStruct {}, - MultiFieldStruct { a: i32, b: f64, c: bool }, + MultiFieldStruct { + a: i32, + b: f64, + c: bool, + }, + #[pyo3(constructor = (a = 42, b = None))] + VariantWithDefault { + a: i32, + b: Option, + }, } #[pyfunction] @@ -54,5 +70,46 @@ pub fn do_complex_stuff(thing: &ComplexEnum) -> ComplexEnum { b: *b, c: *c, }, + ComplexEnum::VariantWithDefault { a, b } => ComplexEnum::VariantWithDefault { + a: 2 * a, + b: b.as_ref().map(|s| s.to_uppercase()), + }, + } +} + +#[pyclass] +enum SimpleTupleEnum { + Int(i32), + Str(String), +} + +#[pyclass] +pub enum TupleEnum { + #[pyo3(constructor = (_0 = 1, _1 = 1.0, _2 = true))] + FullWithDefault(i32, f64, bool), + Full(i32, f64, bool), + EmptyTuple(), +} + +#[pyfunction] +pub fn do_tuple_stuff(thing: &TupleEnum) -> TupleEnum { + match thing { + TupleEnum::FullWithDefault(a, b, c) => TupleEnum::FullWithDefault(*a, *b, *c), + TupleEnum::Full(a, b, c) => TupleEnum::Full(*a, *b, *c), + TupleEnum::EmptyTuple() => TupleEnum::EmptyTuple(), + } +} + +#[pyclass] +pub enum MixedComplexEnum { + Nothing {}, + Empty(), +} + +#[pyfunction] +pub fn do_mixed_complex_stuff(thing: &MixedComplexEnum) -> MixedComplexEnum { + match thing { + MixedComplexEnum::Nothing {} => MixedComplexEnum::Empty(), + MixedComplexEnum::Empty() => MixedComplexEnum::Nothing {}, } } diff --git a/pytests/src/lib.rs b/pytests/src/lib.rs index e65385bf679..76c07937eed 100644 --- a/pytests/src/lib.rs +++ b/pytests/src/lib.rs @@ -5,6 +5,7 @@ use pyo3::wrap_pymodule; pub mod awaitable; pub mod buf_and_str; pub mod comparisons; +mod consts; pub mod datetime; pub mod dict_iter; pub mod enums; @@ -17,44 +18,53 @@ pub mod pyfunctions; pub mod sequence; pub mod subclassing; -#[pymodule] -fn pyo3_pytests(py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_wrapped(wrap_pymodule!(awaitable::awaitable))?; - #[cfg(not(Py_LIMITED_API))] - m.add_wrapped(wrap_pymodule!(buf_and_str::buf_and_str))?; - m.add_wrapped(wrap_pymodule!(comparisons::comparisons))?; - #[cfg(not(Py_LIMITED_API))] - m.add_wrapped(wrap_pymodule!(datetime::datetime))?; - m.add_wrapped(wrap_pymodule!(dict_iter::dict_iter))?; - m.add_wrapped(wrap_pymodule!(enums::enums))?; - m.add_wrapped(wrap_pymodule!(misc::misc))?; - m.add_wrapped(wrap_pymodule!(objstore::objstore))?; - m.add_wrapped(wrap_pymodule!(othermod::othermod))?; - m.add_wrapped(wrap_pymodule!(path::path))?; - m.add_wrapped(wrap_pymodule!(pyclasses::pyclasses))?; - m.add_wrapped(wrap_pymodule!(pyfunctions::pyfunctions))?; - m.add_wrapped(wrap_pymodule!(sequence::sequence))?; - m.add_wrapped(wrap_pymodule!(subclassing::subclassing))?; +#[pymodule(gil_used = false)] +mod pyo3_pytests { + use super::*; + + #[pymodule_export] + use { + comparisons::comparisons, consts::consts, enums::enums, pyclasses::pyclasses, + pyfunctions::pyfunctions, + }; // Inserting to sys.modules allows importing submodules nicely from Python // e.g. import pyo3_pytests.buf_and_str as bas + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_wrapped(wrap_pymodule!(awaitable::awaitable))?; + #[cfg(not(Py_LIMITED_API))] + m.add_wrapped(wrap_pymodule!(buf_and_str::buf_and_str))?; + #[cfg(not(Py_LIMITED_API))] + m.add_wrapped(wrap_pymodule!(datetime::datetime))?; + m.add_wrapped(wrap_pymodule!(dict_iter::dict_iter))?; + m.add_wrapped(wrap_pymodule!(misc::misc))?; + m.add_wrapped(wrap_pymodule!(objstore::objstore))?; + m.add_wrapped(wrap_pymodule!(othermod::othermod))?; + m.add_wrapped(wrap_pymodule!(path::path))?; + m.add_wrapped(wrap_pymodule!(sequence::sequence))?; + m.add_wrapped(wrap_pymodule!(subclassing::subclassing))?; + + // Inserting to sys.modules allows importing submodules nicely from Python + // e.g. import pyo3_pytests.buf_and_str as bas - let sys = PyModule::import(py, "sys")?; - let sys_modules: &PyDict = sys.getattr("modules")?.downcast()?; - sys_modules.set_item("pyo3_pytests.awaitable", m.getattr("awaitable")?)?; - sys_modules.set_item("pyo3_pytests.buf_and_str", m.getattr("buf_and_str")?)?; - sys_modules.set_item("pyo3_pytests.comparisons", m.getattr("comparisons")?)?; - sys_modules.set_item("pyo3_pytests.datetime", m.getattr("datetime")?)?; - sys_modules.set_item("pyo3_pytests.dict_iter", m.getattr("dict_iter")?)?; - sys_modules.set_item("pyo3_pytests.enums", m.getattr("enums")?)?; - sys_modules.set_item("pyo3_pytests.misc", m.getattr("misc")?)?; - sys_modules.set_item("pyo3_pytests.objstore", m.getattr("objstore")?)?; - sys_modules.set_item("pyo3_pytests.othermod", m.getattr("othermod")?)?; - sys_modules.set_item("pyo3_pytests.path", m.getattr("path")?)?; - sys_modules.set_item("pyo3_pytests.pyclasses", m.getattr("pyclasses")?)?; - sys_modules.set_item("pyo3_pytests.pyfunctions", m.getattr("pyfunctions")?)?; - sys_modules.set_item("pyo3_pytests.sequence", m.getattr("sequence")?)?; - sys_modules.set_item("pyo3_pytests.subclassing", m.getattr("subclassing")?)?; + let sys = PyModule::import(m.py(), "sys")?; + let sys_modules = sys.getattr("modules")?.cast_into::()?; + sys_modules.set_item("pyo3_pytests.awaitable", m.getattr("awaitable")?)?; + sys_modules.set_item("pyo3_pytests.buf_and_str", m.getattr("buf_and_str")?)?; + sys_modules.set_item("pyo3_pytests.comparisons", m.getattr("comparisons")?)?; + sys_modules.set_item("pyo3_pytests.datetime", m.getattr("datetime")?)?; + sys_modules.set_item("pyo3_pytests.dict_iter", m.getattr("dict_iter")?)?; + sys_modules.set_item("pyo3_pytests.enums", m.getattr("enums")?)?; + sys_modules.set_item("pyo3_pytests.misc", m.getattr("misc")?)?; + sys_modules.set_item("pyo3_pytests.objstore", m.getattr("objstore")?)?; + sys_modules.set_item("pyo3_pytests.othermod", m.getattr("othermod")?)?; + sys_modules.set_item("pyo3_pytests.path", m.getattr("path")?)?; + sys_modules.set_item("pyo3_pytests.pyclasses", m.getattr("pyclasses")?)?; + sys_modules.set_item("pyo3_pytests.pyfunctions", m.getattr("pyfunctions")?)?; + sys_modules.set_item("pyo3_pytests.sequence", m.getattr("sequence")?)?; + sys_modules.set_item("pyo3_pytests.subclassing", m.getattr("subclassing")?)?; - Ok(()) + Ok(()) + } } diff --git a/pytests/src/misc.rs b/pytests/src/misc.rs index bd941461e91..811923f971a 100644 --- a/pytests/src/misc.rs +++ b/pytests/src/misc.rs @@ -1,15 +1,39 @@ -use pyo3::{prelude::*, types::PyDict}; -use std::borrow::Cow; +use pyo3::{ + prelude::*, + types::{PyDict, PyString}, +}; #[pyfunction] fn issue_219() { - // issue 219: acquiring GIL inside #[pyfunction] deadlocks. - Python::with_gil(|_| {}); + // issue 219: attaching inside #[pyfunction] deadlocks. + Python::attach(|_| {}); +} + +#[pyclass] +struct LockHolder { + #[allow(unused)] + sender: std::sync::mpsc::Sender<()>, +} + +// This will repeatedly attach and detach from the Python interpreter +// once the LockHolder is dropped. +#[pyfunction] +fn hammer_attaching_in_thread() -> LockHolder { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + receiver.recv().ok(); + // now the interpreter has shut down, so hammer the attach API. In buggy + // versions of PyO3 this will cause a crash. + loop { + Python::try_attach(|_py| ()); + } + }); + LockHolder { sender } } #[pyfunction] -fn get_type_full_name(obj: &PyAny) -> PyResult> { - obj.get_type().name() +fn get_type_fully_qualified_name<'py>(obj: &Bound<'py, PyAny>) -> PyResult> { + obj.get_type().fully_qualified_name() } #[pyfunction] @@ -30,10 +54,11 @@ fn get_item_and_run_callback(dict: Bound<'_, PyDict>, callback: Bound<'_, PyAny> Ok(()) } -#[pymodule] -pub fn misc(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(gil_used = false)] +pub fn misc(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(issue_219, m)?)?; - m.add_function(wrap_pyfunction!(get_type_full_name, m)?)?; + m.add_function(wrap_pyfunction!(hammer_attaching_in_thread, m)?)?; + m.add_function(wrap_pyfunction!(get_type_fully_qualified_name, m)?)?; m.add_function(wrap_pyfunction!(accepts_bool, m)?)?; m.add_function(wrap_pyfunction!(get_item_and_run_callback, m)?)?; Ok(()) diff --git a/pytests/src/objstore.rs b/pytests/src/objstore.rs index f7fc66edb84..89643c3967d 100644 --- a/pytests/src/objstore.rs +++ b/pytests/src/objstore.rs @@ -3,7 +3,7 @@ use pyo3::prelude::*; #[pyclass] #[derive(Default)] pub struct ObjStore { - obj: Vec, + obj: Vec>, } #[pymethods] @@ -13,12 +13,12 @@ impl ObjStore { ObjStore::default() } - fn push(&mut self, py: Python<'_>, obj: &PyAny) { - self.obj.push(obj.to_object(py)); + fn push(&mut self, obj: &Bound<'_, PyAny>) { + self.obj.push(obj.clone().unbind()); } } -#[pymodule] -pub fn objstore(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(gil_used = false)] +pub fn objstore(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::() } diff --git a/pytests/src/othermod.rs b/pytests/src/othermod.rs index 763d38a878f..0de912d7d04 100644 --- a/pytests/src/othermod.rs +++ b/pytests/src/othermod.rs @@ -28,14 +28,14 @@ fn double(x: i32) -> i32 { x * 2 } -#[pymodule] -pub fn othermod(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(gil_used = false)] +pub fn othermod(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(double, m)?)?; m.add_class::()?; - m.add("USIZE_MIN", usize::min_value())?; - m.add("USIZE_MAX", usize::max_value())?; + m.add("USIZE_MIN", usize::MIN)?; + m.add("USIZE_MAX", usize::MAX)?; Ok(()) } diff --git a/pytests/src/path.rs b/pytests/src/path.rs index b3e8f92bacf..b52c038ed34 100644 --- a/pytests/src/path.rs +++ b/pytests/src/path.rs @@ -11,8 +11,8 @@ fn take_pathbuf(path: PathBuf) -> PathBuf { path } -#[pymodule] -pub fn path(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(gil_used = false)] +pub fn path(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(make_path, m)?)?; m.add_function(wrap_pyfunction!(take_pathbuf, m)?)?; diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 9c7b2d250da..631ec4fa0a0 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -1,8 +1,11 @@ +use std::{thread, time}; + use pyo3::exceptions::{PyStopIteration, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyType; #[pyclass] +#[derive(Clone, Default)] struct EmptyClass {} #[pymethods] @@ -11,6 +14,12 @@ impl EmptyClass { fn new() -> Self { EmptyClass {} } + + fn method(&self) {} + + fn __len__(&self) -> usize { + 0 + } } /// This is for demonstrating how to return a value from __next__ @@ -37,6 +46,29 @@ impl PyClassIter { } } +#[pyclass] +#[derive(Default)] +struct PyClassThreadIter { + count: usize, +} + +#[pymethods] +impl PyClassThreadIter { + #[new] + pub fn new() -> Self { + Default::default() + } + + fn __next__(&mut self, py: Python<'_>) -> usize { + let current_count = self.count; + self.count += 1; + if current_count == 0 { + py.detach(|| thread::sleep(time::Duration::from_millis(100))); + } + self.count + } +} + /// Demonstrates a base class which can operate on the relevant subclass in its constructor. #[pyclass(subclass)] #[derive(Clone, Debug)] @@ -49,42 +81,101 @@ impl AssertingBaseClass { fn new(cls: &Bound<'_, PyType>, expected_type: Bound<'_, PyType>) -> PyResult { if !cls.is(&expected_type) { return Err(PyValueError::new_err(format!( - "{:?} != {:?}", - cls, expected_type + "{cls:?} != {expected_type:?}" ))); } Ok(Self) } } -#[pyclass(subclass)] -#[derive(Clone, Debug)] -struct AssertingBaseClassGilRef; +#[pyclass] +struct ClassWithoutConstructor; +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[pyclass(dict)] +struct ClassWithDict; + +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] #[pymethods] -impl AssertingBaseClassGilRef { +impl ClassWithDict { #[new] - #[classmethod] - fn new(cls: &PyType, expected_type: &PyType) -> PyResult { - if !cls.is(expected_type) { - return Err(PyValueError::new_err(format!( - "{:?} != {:?}", - cls, expected_type - ))); - } - Ok(Self) + fn new() -> Self { + ClassWithDict } } #[pyclass] -struct ClassWithoutConstructor; +#[derive(Clone)] +struct ClassWithDecorators { + attr: usize, +} + +#[pymethods] +impl ClassWithDecorators { + #[new] + #[classmethod] + fn new(_cls: Bound<'_, PyType>) -> Self { + Self { attr: 0 } + } + + #[getter] + fn get_attr(&self) -> usize { + self.attr + } + + #[setter] + fn set_attr(&mut self, value: usize) { + self.attr = value; + } + + #[classmethod] + fn cls_method(_cls: &Bound<'_, PyType>) -> usize { + 1 + } + + #[staticmethod] + fn static_method() -> usize { + 2 + } + + #[classattr] + fn cls_attribute() -> usize { + 3 + } +} + +#[pyclass(get_all, set_all)] +struct PlainObject { + foo: String, + bar: usize, +} + +#[derive(FromPyObject, IntoPyObject)] +enum AClass { + NewType(EmptyClass), + Tuple(EmptyClass, EmptyClass), + Struct { + f: EmptyClass, + #[pyo3(item(42))] + g: EmptyClass, + #[pyo3(default)] + h: EmptyClass, + }, +} + +#[pyfunction] +fn map_a_class(cls: AClass) -> AClass { + cls +} -#[pymodule] -pub fn pyclasses(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - Ok(()) +#[pymodule(gil_used = false)] +pub mod pyclasses { + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + #[pymodule_export] + use super::ClassWithDict; + #[pymodule_export] + use super::{ + map_a_class, AssertingBaseClass, ClassWithDecorators, ClassWithoutConstructor, EmptyClass, + PlainObject, PyClassIter, PyClassThreadIter, + }; } diff --git a/pytests/src/pyfunctions.rs b/pytests/src/pyfunctions.rs index 1eef970430e..5cb88f976ab 100644 --- a/pytests/src/pyfunctions.rs +++ b/pytests/src/pyfunctions.rs @@ -4,72 +4,144 @@ use pyo3::types::{PyDict, PyTuple}; #[pyfunction(signature = ())] fn none() {} +type Any<'py> = Bound<'py, PyAny>; +type Dict<'py> = Bound<'py, PyDict>; +type Tuple<'py> = Bound<'py, PyTuple>; + #[pyfunction(signature = (a, b = None, *, c = None))] -fn simple<'a>( - a: &'a PyAny, - b: Option<&'a PyAny>, - c: Option<&'a PyAny>, -) -> (&'a PyAny, Option<&'a PyAny>, Option<&'a PyAny>) { +fn simple<'py>( + a: Any<'py>, + b: Option>, + c: Option>, +) -> (Any<'py>, Option>, Option>) { (a, b, c) } #[pyfunction(signature = (a, b = None, *args, c = None))] -fn simple_args<'a>( - a: &'a PyAny, - b: Option<&'a PyAny>, - args: &'a PyTuple, - c: Option<&'a PyAny>, -) -> (&'a PyAny, Option<&'a PyAny>, &'a PyTuple, Option<&'a PyAny>) { +fn simple_args<'py>( + a: Any<'py>, + b: Option>, + args: Tuple<'py>, + c: Option>, +) -> (Any<'py>, Option>, Tuple<'py>, Option>) { (a, b, args, c) } #[pyfunction(signature = (a, b = None, c = None, **kwargs))] -fn simple_kwargs<'a>( - a: &'a PyAny, - b: Option<&'a PyAny>, - c: Option<&'a PyAny>, - kwargs: Option<&'a PyDict>, +fn simple_kwargs<'py>( + a: Any<'py>, + b: Option>, + c: Option>, + kwargs: Option>, ) -> ( - &'a PyAny, - Option<&'a PyAny>, - Option<&'a PyAny>, - Option<&'a PyDict>, + Any<'py>, + Option>, + Option>, + Option>, ) { (a, b, c, kwargs) } #[pyfunction(signature = (a, b = None, *args, c = None, **kwargs))] -fn simple_args_kwargs<'a>( - a: &'a PyAny, - b: Option<&'a PyAny>, - args: &'a PyTuple, - c: Option<&'a PyAny>, - kwargs: Option<&'a PyDict>, +fn simple_args_kwargs<'py>( + a: Any<'py>, + b: Option>, + args: Tuple<'py>, + c: Option>, + kwargs: Option>, ) -> ( - &'a PyAny, - Option<&'a PyAny>, - &'a PyTuple, - Option<&'a PyAny>, - Option<&'a PyDict>, + Any<'py>, + Option>, + Tuple<'py>, + Option>, + Option>, ) { (a, b, args, c, kwargs) } #[pyfunction(signature = (*args, **kwargs))] -fn args_kwargs<'a>( - args: &'a PyTuple, - kwargs: Option<&'a PyDict>, -) -> (&'a PyTuple, Option<&'a PyDict>) { +fn args_kwargs<'py>( + args: Tuple<'py>, + kwargs: Option>, +) -> (Tuple<'py>, Option>) { (args, kwargs) } +#[pyfunction(signature = (a, /, b))] +fn positional_only<'py>(a: Any<'py>, b: Any<'py>) -> (Any<'py>, Any<'py>) { + (a, b) +} + +#[pyfunction(signature = (a = false, b = 0, c = 0.0, d = ""))] +fn with_typed_args(a: bool, b: u64, c: f64, d: &str) -> (bool, u64, f64, &str) { + (a, b, c, d) +} + +#[cfg(feature = "experimental-inspect")] +#[pyfunction(signature = (a: "int", *_args: "str", _b: "int | None" = None, **_kwargs: "bool") -> "int")] +fn with_custom_type_annotations<'py>( + a: Any<'py>, + _args: Tuple<'py>, + _b: Option>, + _kwargs: Option>, +) -> Any<'py> { + a +} + +#[allow(clippy::too_many_arguments)] +#[pyfunction( + signature = ( + *, + ant = None, + bear = None, + cat = None, + dog = None, + elephant = None, + fox = None, + goat = None, + horse = None, + iguana = None, + jaguar = None, + koala = None, + lion = None, + monkey = None, + newt = None, + owl = None, + penguin = None + ) +)] +fn many_keyword_arguments<'py>( + ant: Option<&'_ Bound<'py, PyAny>>, + bear: Option<&'_ Bound<'py, PyAny>>, + cat: Option<&'_ Bound<'py, PyAny>>, + dog: Option<&'_ Bound<'py, PyAny>>, + elephant: Option<&'_ Bound<'py, PyAny>>, + fox: Option<&'_ Bound<'py, PyAny>>, + goat: Option<&'_ Bound<'py, PyAny>>, + horse: Option<&'_ Bound<'py, PyAny>>, + iguana: Option<&'_ Bound<'py, PyAny>>, + jaguar: Option<&'_ Bound<'py, PyAny>>, + koala: Option<&'_ Bound<'py, PyAny>>, + lion: Option<&'_ Bound<'py, PyAny>>, + monkey: Option<&'_ Bound<'py, PyAny>>, + newt: Option<&'_ Bound<'py, PyAny>>, + owl: Option<&'_ Bound<'py, PyAny>>, + penguin: Option<&'_ Bound<'py, PyAny>>, +) { + std::hint::black_box(( + ant, bear, cat, dog, elephant, fox, goat, horse, iguana, jaguar, koala, lion, monkey, newt, + owl, penguin, + )); +} + #[pymodule] -pub fn pyfunctions(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(none, m)?)?; - m.add_function(wrap_pyfunction!(simple, m)?)?; - m.add_function(wrap_pyfunction!(simple_args, m)?)?; - m.add_function(wrap_pyfunction!(simple_kwargs, m)?)?; - m.add_function(wrap_pyfunction!(simple_args_kwargs, m)?)?; - m.add_function(wrap_pyfunction!(args_kwargs, m)?)?; - Ok(()) +pub mod pyfunctions { + #[cfg(feature = "experimental-inspect")] + #[pymodule_export] + use super::with_custom_type_annotations; + #[pymodule_export] + use super::{ + args_kwargs, many_keyword_arguments, none, positional_only, simple, simple_args, + simple_args_kwargs, simple_kwargs, with_typed_args, + }; } diff --git a/pytests/src/sequence.rs b/pytests/src/sequence.rs index 5916414ee8f..175f5fba8aa 100644 --- a/pytests/src/sequence.rs +++ b/pytests/src/sequence.rs @@ -12,12 +12,12 @@ fn array_to_array_i32(arr: [i32; 3]) -> [i32; 3] { } #[pyfunction] -fn vec_to_vec_pystring(vec: Vec<&PyString>) -> Vec<&PyString> { +fn vec_to_vec_pystring(vec: Vec>) -> Vec> { vec } -#[pymodule] -pub fn sequence(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(gil_used = false)] +pub fn sequence(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(vec_to_vec_i32, m)?)?; m.add_function(wrap_pyfunction!(array_to_array_i32, m)?)?; m.add_function(wrap_pyfunction!(vec_to_vec_pystring, m)?)?; diff --git a/pytests/src/subclassing.rs b/pytests/src/subclassing.rs index 0033114ccea..0f00e74c19d 100644 --- a/pytests/src/subclassing.rs +++ b/pytests/src/subclassing.rs @@ -17,8 +17,8 @@ impl Subclassable { } } -#[pymodule] -pub fn subclassing(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +#[pymodule(gil_used = false)] +pub fn subclassing(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/pytests/stubs/__init__.pyi b/pytests/stubs/__init__.pyi new file mode 100644 index 00000000000..0f6820f054e --- /dev/null +++ b/pytests/stubs/__init__.pyi @@ -0,0 +1,3 @@ +from _typeshed import Incomplete + +def __getattr__(name: str) -> Incomplete: ... diff --git a/pytests/stubs/comparisons.pyi b/pytests/stubs/comparisons.pyi new file mode 100644 index 00000000000..f5e44130057 --- /dev/null +++ b/pytests/stubs/comparisons.pyi @@ -0,0 +1,50 @@ +class Eq: + def __eq__(self, /, other: Eq) -> bool: ... + def __ne__(self, /, other: Eq) -> bool: ... + def __new__(cls, /, value: int) -> None: ... + +class EqDefaultNe: + def __eq__(self, /, other: EqDefaultNe) -> bool: ... + def __new__(cls, /, value: int) -> None: ... + +class EqDerived: + def __eq__(self, /, other: EqDerived) -> bool: ... + def __ne__(self, /, other: EqDerived) -> bool: ... + def __new__(cls, /, value: int) -> None: ... + +class Ordered: + def __eq__(self, /, other: Ordered) -> bool: ... + def __ge__(self, /, other: Ordered) -> bool: ... + def __gt__(self, /, other: Ordered) -> bool: ... + def __le__(self, /, other: Ordered) -> bool: ... + def __lt__(self, /, other: Ordered) -> bool: ... + def __ne__(self, /, other: Ordered) -> bool: ... + def __new__(cls, /, value: int) -> None: ... + +class OrderedDefaultNe: + def __eq__(self, /, other: OrderedDefaultNe) -> bool: ... + def __ge__(self, /, other: OrderedDefaultNe) -> bool: ... + def __gt__(self, /, other: OrderedDefaultNe) -> bool: ... + def __le__(self, /, other: OrderedDefaultNe) -> bool: ... + def __lt__(self, /, other: OrderedDefaultNe) -> bool: ... + def __new__(cls, /, value: int) -> None: ... + +class OrderedDerived: + def __eq__(self, /, other: OrderedDerived) -> bool: ... + def __ge__(self, /, other: OrderedDerived) -> bool: ... + def __gt__(self, /, other: OrderedDerived) -> bool: ... + def __hash__(self, /) -> int: ... + def __le__(self, /, other: OrderedDerived) -> bool: ... + def __lt__(self, /, other: OrderedDerived) -> bool: ... + def __ne__(self, /, other: OrderedDerived) -> bool: ... + def __new__(cls, /, value: int) -> None: ... + def __str__(self, /) -> str: ... + +class OrderedRichCmp: + def __eq__(self, /, other: OrderedRichCmp) -> bool: ... + def __ge__(self, /, other: OrderedRichCmp) -> bool: ... + def __gt__(self, /, other: OrderedRichCmp) -> bool: ... + def __le__(self, /, other: OrderedRichCmp) -> bool: ... + def __lt__(self, /, other: OrderedRichCmp) -> bool: ... + def __ne__(self, /, other: OrderedRichCmp) -> bool: ... + def __new__(cls, /, value: int) -> None: ... diff --git a/pytests/stubs/consts.pyi b/pytests/stubs/consts.pyi new file mode 100644 index 00000000000..66b0672c8a5 --- /dev/null +++ b/pytests/stubs/consts.pyi @@ -0,0 +1,7 @@ +from typing import Final + +PI: Final[float] +SIMPLE: Final = "SIMPLE" + +class ClassWithConst: + INSTANCE: Final[ClassWithConst] diff --git a/pytests/stubs/enums.pyi b/pytests/stubs/enums.pyi new file mode 100644 index 00000000000..95eaabac451 --- /dev/null +++ b/pytests/stubs/enums.pyi @@ -0,0 +1,14 @@ +class ComplexEnum: ... +class MixedComplexEnum: ... + +class SimpleEnum: + def __eq__(self, /, other: SimpleEnum | int) -> bool: ... + def __ne__(self, /, other: SimpleEnum | int) -> bool: ... + +class SimpleTupleEnum: ... +class TupleEnum: ... + +def do_complex_stuff(thing: ComplexEnum) -> ComplexEnum: ... +def do_mixed_complex_stuff(thing: MixedComplexEnum) -> MixedComplexEnum: ... +def do_simple_stuff(thing: SimpleEnum) -> SimpleEnum: ... +def do_tuple_stuff(thing: TupleEnum) -> TupleEnum: ... diff --git a/pytests/stubs/pyclasses.pyi b/pytests/stubs/pyclasses.pyi new file mode 100644 index 00000000000..e2d1f4d006b --- /dev/null +++ b/pytests/stubs/pyclasses.pyi @@ -0,0 +1,51 @@ +from _typeshed import Incomplete +from typing import Any + +class AssertingBaseClass: + def __new__(cls, /, expected_type: Any) -> None: ... + +class ClassWithDecorators: + def __new__(cls, /) -> None: ... + @property + def attr(self, /) -> int: ... + @attr.setter + def attr(self, /, value: int) -> None: ... + @classmethod + @property + def cls_attribute(cls, /) -> int: ... + @classmethod + def cls_method(cls, /) -> int: ... + @staticmethod + def static_method() -> int: ... + +class ClassWithDict: + def __new__(cls, /) -> None: ... + +class ClassWithoutConstructor: ... + +class EmptyClass: + def __len__(self, /) -> int: ... + def __new__(cls, /) -> None: ... + def method(self, /) -> None: ... + +class PlainObject: + @property + def bar(self, /) -> int: ... + @bar.setter + def bar(self, /, value: int) -> None: ... + @property + def foo(self, /) -> str: ... + @foo.setter + def foo(self, /, value: str) -> None: ... + +class PyClassIter: + def __new__(cls, /) -> None: ... + def __next__(self, /) -> int: ... + +class PyClassThreadIter: + def __new__(cls, /) -> None: ... + def __next__(self, /) -> int: ... + +def map_a_class( + cls: EmptyClass | tuple[EmptyClass, EmptyClass] | Incomplete, +) -> EmptyClass | tuple[EmptyClass, EmptyClass] | Incomplete: ... diff --git a/pytests/stubs/pyfunctions.pyi b/pytests/stubs/pyfunctions.pyi new file mode 100644 index 00000000000..9513072d023 --- /dev/null +++ b/pytests/stubs/pyfunctions.pyi @@ -0,0 +1,38 @@ +from typing import Any + +def args_kwargs(*args, **kwargs) -> Any: ... +def many_keyword_arguments( + *, + ant: Any | None = None, + bear: Any | None = None, + cat: Any | None = None, + dog: Any | None = None, + elephant: Any | None = None, + fox: Any | None = None, + goat: Any | None = None, + horse: Any | None = None, + iguana: Any | None = None, + jaguar: Any | None = None, + koala: Any | None = None, + lion: Any | None = None, + monkey: Any | None = None, + newt: Any | None = None, + owl: Any | None = None, + penguin: Any | None = None, +) -> None: ... +def none() -> None: ... +def positional_only(a: Any, /, b: Any) -> Any: ... +def simple(a: Any, b: Any | None = None, *, c: Any | None = None) -> Any: ... +def simple_args(a: Any, b: Any | None = None, *args, c: Any | None = None) -> Any: ... +def simple_args_kwargs( + a: Any, b: Any | None = None, *args, c: Any | None = None, **kwargs +) -> Any: ... +def simple_kwargs( + a: Any, b: Any | None = None, c: Any | None = None, **kwargs +) -> Any: ... +def with_custom_type_annotations( + a: int, *_args: str, _b: int | None = None, **_kwargs: bool +) -> int: ... +def with_typed_args( + a: bool = False, b: int = 0, c: float = 0.0, d: str = "" +) -> Any: ... diff --git a/pytests/tests/test_awaitable.py b/pytests/tests/test_awaitable.py index 2bada317517..40f1e1cc01d 100644 --- a/pytests/tests/test_awaitable.py +++ b/pytests/tests/test_awaitable.py @@ -1,13 +1,22 @@ import pytest +import sys from pyo3_pytests.awaitable import IterAwaitable, FutureAwaitable +@pytest.mark.skipif( + sys.implementation.name == "graalpy", + reason="GraalPy's asyncio module has a bug with native classes, see oracle/graalpython#365", +) @pytest.mark.asyncio async def test_iter_awaitable(): assert await IterAwaitable(5) == 5 +@pytest.mark.skipif( + sys.implementation.name == "graalpy", + reason="GraalPy's asyncio module has a bug with native classes, see oracle/graalpython#365", +) @pytest.mark.asyncio async def test_future_awaitable(): assert await FutureAwaitable(5) == 5 diff --git a/pytests/tests/test_comparisons.py b/pytests/tests/test_comparisons.py index 508cdeb2465..2c889be4189 100644 --- a/pytests/tests/test_comparisons.py +++ b/pytests/tests/test_comparisons.py @@ -1,7 +1,16 @@ from typing import Type, Union +import sys import pytest -from pyo3_pytests.comparisons import Eq, EqDefaultNe, Ordered, OrderedDefaultNe +from pyo3_pytests.comparisons import ( + Eq, + EqDefaultNe, + EqDerived, + Ordered, + OrderedDefaultNe, + OrderedDerived, + OrderedRichCmp, +) from typing_extensions import Self @@ -9,15 +18,28 @@ class PyEq: def __init__(self, x: int) -> None: self.x = x - def __eq__(self, other: Self) -> bool: - return self.x == other.x + def __eq__(self, other: object) -> bool: + if isinstance(other, self.__class__): + return self.x == other.x + else: + return NotImplemented def __ne__(self, other: Self) -> bool: - return self.x != other.x + if isinstance(other, self.__class__): + return self.x != other.x + else: + return NotImplemented -@pytest.mark.parametrize("ty", (Eq, PyEq), ids=("rust", "python")) -def test_eq(ty: Type[Union[Eq, PyEq]]): +@pytest.mark.skipif( + sys.implementation.name == "graalpy" + and __graalpython__.get_graalvm_version().startswith("24.1"), # noqa: F821 + reason="Bug in GraalPy 24.1", +) +@pytest.mark.parametrize( + "ty", (Eq, EqDerived, PyEq), ids=("rust", "rust-derived", "python") +) +def test_eq(ty: Type[Union[Eq, EqDerived, PyEq]]): a = ty(0) b = ty(0) c = ty(1) @@ -32,6 +54,13 @@ def test_eq(ty: Type[Union[Eq, PyEq]]): assert b != c assert not (b == c) + assert not a == 0 + assert a != 0 + assert not b == 0 + assert b != 1 + assert not c == 1 + assert c != 1 + with pytest.raises(TypeError): assert a <= b @@ -105,8 +134,12 @@ def __ge__(self, other: Self) -> bool: return self.x >= other.x -@pytest.mark.parametrize("ty", (Ordered, PyOrdered), ids=("rust", "python")) -def test_ordered(ty: Type[Union[Ordered, PyOrdered]]): +@pytest.mark.parametrize( + "ty", + (Ordered, OrderedDerived, OrderedRichCmp, PyOrdered), + ids=("rust", "rust-derived", "rust-richcmp", "python"), +) +def test_ordered(ty: Type[Union[Ordered, OrderedDerived, OrderedRichCmp, PyOrdered]]): a = ty(0) b = ty(0) c = ty(1) diff --git a/pytests/tests/test_datetime.py b/pytests/tests/test_datetime.py index c81d13a929a..6484b926a96 100644 --- a/pytests/tests/test_datetime.py +++ b/pytests/tests/test_datetime.py @@ -56,11 +56,11 @@ def tzname(self, dt): IS_WINDOWS = sys.platform == "win32" if IS_WINDOWS: - MIN_DATETIME = pdt.datetime(1971, 1, 2, 0, 0) + MIN_DATETIME = pdt.datetime(1970, 1, 1, 0, 0, 0) if IS_32_BIT: - MAX_DATETIME = pdt.datetime(3001, 1, 19, 4, 59, 59) + MAX_DATETIME = pdt.datetime(2038, 1, 18, 23, 59, 59) else: - MAX_DATETIME = pdt.datetime(3001, 1, 19, 7, 59, 59) + MAX_DATETIME = pdt.datetime(3000, 12, 31, 23, 59, 59) else: if IS_32_BIT: # TS ±2147483648 (2**31) @@ -93,11 +93,21 @@ def test_invalid_date_fails(): @given(d=st.dates(MIN_DATETIME.date(), MAX_DATETIME.date())) def test_date_from_timestamp(d): - if PYPY and d < pdt.date(1900, 1, 1): - pytest.xfail("pdt.datetime.timestamp will raise on PyPy with dates before 1900") - - ts = pdt.datetime.timestamp(pdt.datetime.combine(d, pdt.time(0))) - assert rdt.date_from_timestamp(int(ts)) == pdt.date.fromtimestamp(ts) + try: + ts = pdt.datetime.timestamp(d) + except Exception: + # out of range for timestamp + return + + try: + expected = pdt.date.fromtimestamp(ts) + except Exception as pdt_fail: + # date from timestamp failed; expect the same from Rust binding + with pytest.raises(type(pdt_fail)) as exc_info: + rdt.date_from_timestamp(ts) + assert str(exc_info.value) == str(pdt_fail) + else: + assert rdt.date_from_timestamp(int(ts)) == expected @pytest.mark.parametrize( @@ -229,11 +239,21 @@ def test_datetime_typeerror(): @given(dt=st.datetimes(MIN_DATETIME, MAX_DATETIME)) @example(dt=pdt.datetime(1971, 1, 2, 0, 0)) def test_datetime_from_timestamp(dt): - if PYPY and dt < pdt.datetime(1900, 1, 1): - pytest.xfail("pdt.datetime.timestamp will raise on PyPy with dates before 1900") - - ts = pdt.datetime.timestamp(dt) - assert rdt.datetime_from_timestamp(ts) == pdt.datetime.fromtimestamp(ts) + try: + ts = pdt.datetime.timestamp(dt) + except Exception: + # out of range for timestamp + return + + try: + expected = pdt.datetime.fromtimestamp(ts) + except Exception as pdt_fail: + # datetime from timestamp failed; expect the same from Rust binding + with pytest.raises(type(pdt_fail)) as exc_info: + rdt.datetime_from_timestamp(ts) + assert str(exc_info.value) == str(pdt_fail) + else: + assert rdt.datetime_from_timestamp(ts) == expected def test_datetime_from_timestamp_tzinfo(): diff --git a/pytests/tests/test_enums.py b/pytests/tests/test_enums.py index 04b0cdca431..cd4f7e124c9 100644 --- a/pytests/tests/test_enums.py +++ b/pytests/tests/test_enums.py @@ -18,6 +18,12 @@ def test_complex_enum_variant_constructors(): multi_field_struct_variant = enums.ComplexEnum.MultiFieldStruct(42, 3.14, True) assert isinstance(multi_field_struct_variant, enums.ComplexEnum.MultiFieldStruct) + variant_with_default_1 = enums.ComplexEnum.VariantWithDefault() + assert isinstance(variant_with_default_1, enums.ComplexEnum.VariantWithDefault) + + variant_with_default_2 = enums.ComplexEnum.VariantWithDefault(25, "Hello") + assert isinstance(variant_with_default_2, enums.ComplexEnum.VariantWithDefault) + @pytest.mark.parametrize( "variant", @@ -27,6 +33,7 @@ def test_complex_enum_variant_constructors(): enums.ComplexEnum.Str("hello"), enums.ComplexEnum.EmptyStruct(), enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + enums.ComplexEnum.VariantWithDefault(), ], ) def test_complex_enum_variant_subclasses(variant: enums.ComplexEnum): @@ -48,6 +55,10 @@ def test_complex_enum_field_getters(): assert multi_field_struct_variant.b == 3.14 assert multi_field_struct_variant.c is True + variant_with_default = enums.ComplexEnum.VariantWithDefault() + assert variant_with_default.a == 42 + assert variant_with_default.b is None + @pytest.mark.parametrize( "variant", @@ -57,6 +68,7 @@ def test_complex_enum_field_getters(): enums.ComplexEnum.Str("hello"), enums.ComplexEnum.EmptyStruct(), enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + enums.ComplexEnum.VariantWithDefault(), ], ) def test_complex_enum_desugared_match(variant: enums.ComplexEnum): @@ -78,6 +90,11 @@ def test_complex_enum_desugared_match(variant: enums.ComplexEnum): assert x == 42 assert y == 3.14 assert z is True + elif isinstance(variant, enums.ComplexEnum.VariantWithDefault): + x = variant.a + y = variant.b + assert x == 42 + assert y is None else: assert False @@ -90,6 +107,7 @@ def test_complex_enum_desugared_match(variant: enums.ComplexEnum): enums.ComplexEnum.Str("hello"), enums.ComplexEnum.EmptyStruct(), enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + enums.ComplexEnum.VariantWithDefault(b="hello"), ], ) def test_complex_enum_pyfunction_in_out_desugared_match(variant: enums.ComplexEnum): @@ -112,5 +130,74 @@ def test_complex_enum_pyfunction_in_out_desugared_match(variant: enums.ComplexEn assert x == 42 assert y == 3.14 assert z is True + elif isinstance(variant, enums.ComplexEnum.VariantWithDefault): + x = variant.a + y = variant.b + assert x == 84 + assert y == "HELLO" else: assert False + + +def test_tuple_enum_variant_constructors(): + tuple_variant = enums.TupleEnum.Full(42, 3.14, False) + assert isinstance(tuple_variant, enums.TupleEnum.Full) + + empty_tuple_variant = enums.TupleEnum.EmptyTuple() + assert isinstance(empty_tuple_variant, enums.TupleEnum.EmptyTuple) + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.FullWithDefault(), + enums.TupleEnum.Full(42, 3.14, False), + enums.TupleEnum.EmptyTuple(), + ], +) +def test_tuple_enum_variant_subclasses(variant: enums.TupleEnum): + assert isinstance(variant, enums.TupleEnum) + + +def test_tuple_enum_defaults(): + variant = enums.TupleEnum.FullWithDefault() + assert variant._0 == 1 + assert variant._1 == 1.0 + assert variant._2 is True + + +def test_tuple_enum_field_getters(): + tuple_variant = enums.TupleEnum.Full(42, 3.14, False) + assert tuple_variant._0 == 42 + assert tuple_variant._1 == 3.14 + assert tuple_variant._2 is False + + +def test_tuple_enum_index_getter(): + tuple_variant = enums.TupleEnum.Full(42, 3.14, False) + assert len(tuple_variant) == 3 + assert tuple_variant[0] == 42 + + +@pytest.mark.parametrize( + "variant", + [enums.MixedComplexEnum.Nothing()], +) +def test_mixed_complex_enum_pyfunction_instance_nothing( + variant: enums.MixedComplexEnum, +): + assert isinstance(variant, enums.MixedComplexEnum.Nothing) + assert isinstance( + enums.do_mixed_complex_stuff(variant), enums.MixedComplexEnum.Empty + ) + + +@pytest.mark.parametrize( + "variant", + [enums.MixedComplexEnum.Empty()], +) +def test_mixed_complex_enum_pyfunction_instance_empty(variant: enums.MixedComplexEnum): + assert isinstance(variant, enums.MixedComplexEnum.Empty) + assert isinstance( + enums.do_mixed_complex_stuff(variant), enums.MixedComplexEnum.Nothing + ) diff --git a/pytests/tests/test_enums_match.py b/pytests/tests/test_enums_match.py index 4d55bbbe351..6c4b5f6aa07 100644 --- a/pytests/tests/test_enums_match.py +++ b/pytests/tests/test_enums_match.py @@ -57,3 +57,102 @@ def test_complex_enum_pyfunction_in_out(variant: enums.ComplexEnum): assert z is True case _: assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + ], +) +def test_complex_enum_partial_match(variant: enums.ComplexEnum): + match variant: + case enums.ComplexEnum.MultiFieldStruct(a): + assert a == 42 + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.Full(42, 3.14, True), + enums.TupleEnum.EmptyTuple(), + ], +) +def test_tuple_enum_match_statement(variant: enums.TupleEnum): + match variant: + case enums.TupleEnum.Full(_0=x, _1=y, _2=z): + assert x == 42 + assert y == 3.14 + assert z is True + case enums.TupleEnum.EmptyTuple(): + assert True + case _: + print(variant) + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.SimpleTupleEnum.Int(42), + enums.SimpleTupleEnum.Str("hello"), + ], +) +def test_simple_tuple_enum_match_statement(variant: enums.SimpleTupleEnum): + match variant: + case enums.SimpleTupleEnum.Int(x): + assert x == 42 + case enums.SimpleTupleEnum.Str(x): + assert x == "hello" + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.Full(42, 3.14, True), + ], +) +def test_tuple_enum_match_match_args(variant: enums.TupleEnum): + match variant: + case enums.TupleEnum.Full(x, y, z): + assert x == 42 + assert y == 3.14 + assert z is True + assert True + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.Full(42, 3.14, True), + ], +) +def test_tuple_enum_partial_match(variant: enums.TupleEnum): + match variant: + case enums.TupleEnum.Full(a): + assert a == 42 + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.MixedComplexEnum.Nothing(), + enums.MixedComplexEnum.Empty(), + ], +) +def test_mixed_complex_enum_match_statement(variant: enums.MixedComplexEnum): + match variant: + case enums.MixedComplexEnum.Nothing(): + assert True + case enums.MixedComplexEnum.Empty(): + assert True + case _: + assert False diff --git a/pytests/tests/test_hammer_attaching_in_thread.py b/pytests/tests/test_hammer_attaching_in_thread.py new file mode 100644 index 00000000000..15ec8953bc6 --- /dev/null +++ b/pytests/tests/test_hammer_attaching_in_thread.py @@ -0,0 +1,28 @@ +import sysconfig + +import pytest + +from pyo3_pytests import misc + + +def make_loop(): + # create a reference loop that will only be destroyed when the GC is called at the end + # of execution + start = [] + cur = [start] + for _ in range(1000 * 1000 * 10): + cur = [cur] + start.append(cur) + return start + + +# set a bomb that will explode when modules are cleaned up +loopy = [make_loop()] + + +@pytest.mark.skipif( + sysconfig.get_config_var("Py_DEBUG"), + reason="causes a crash on debug builds, see discussion in https://github.com/PyO3/pyo3/pull/4874", +) +def test_hammer_attaching_in_thread(): + loopy.append(misc.hammer_attaching_in_thread()) diff --git a/pytests/tests/test_misc.py b/pytests/tests/test_misc.py index 6645f942f1a..dd7f8007e81 100644 --- a/pytests/tests/test_misc.py +++ b/pytests/tests/test_misc.py @@ -5,6 +5,11 @@ import pyo3_pytests.misc import pytest +if sys.version_info >= (3, 13): + subinterpreters = pytest.importorskip("_interpreters") +else: + subinterpreters = pytest.importorskip("_xxsubinterpreters") + def test_issue_219(): # Should not deadlock @@ -27,33 +32,43 @@ def test_multiple_imports_same_interpreter_ok(): reason="Cannot identify subinterpreters on Python older than 3.9", ) @pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="PyPy does not support subinterpreters", + platform.python_implementation() in ("PyPy", "GraalVM"), + reason="PyPy and GraalPy do not support subinterpreters", ) def test_import_in_subinterpreter_forbidden(): - import _xxsubinterpreters - + sub_interpreter = subinterpreters.create() if sys.version_info < (3, 12): expected_error = "PyO3 modules do not yet support subinterpreters, see https://github.com/PyO3/pyo3/issues/576" else: expected_error = "module pyo3_pytests.pyo3_pytests does not support loading in subinterpreters" - sub_interpreter = _xxsubinterpreters.create() - with pytest.raises( - _xxsubinterpreters.RunFailedError, - match=expected_error, - ): - _xxsubinterpreters.run_string( + if sys.version_info < (3, 13): + # Python 3.12 subinterpreters had a special error for this + with pytest.raises( + subinterpreters.RunFailedError, + match=expected_error, + ): + subinterpreters.run_string( + sub_interpreter, "import pyo3_pytests.pyo3_pytests" + ) + else: + res = subinterpreters.run_string( sub_interpreter, "import pyo3_pytests.pyo3_pytests" ) + assert res.type.__name__ == "ImportError" + assert res.msg == expected_error - _xxsubinterpreters.destroy(sub_interpreter) + subinterpreters.destroy(sub_interpreter) -def test_type_full_name_includes_module(): +def test_type_fully_qualified_name_includes_module(): numpy = pytest.importorskip("numpy") - assert pyo3_pytests.misc.get_type_full_name(numpy.bool_(True)) == "numpy.bool_" + # For numpy 1.x and 2.x + assert pyo3_pytests.misc.get_type_fully_qualified_name(numpy.bool_(True)) in [ + "numpy.bool", + "numpy.bool_", + ] def test_accepts_numpy_bool(): diff --git a/pytests/tests/test_objstore.py b/pytests/tests/test_objstore.py index bfd8bad84df..3f0d23fa97c 100644 --- a/pytests/tests/test_objstore.py +++ b/pytests/tests/test_objstore.py @@ -12,6 +12,11 @@ def test_objstore_doesnot_leak_memory(): # check refcount on PyPy getrefcount = getattr(sys, "getrefcount", lambda obj: 0) + if sys.implementation.name == "graalpy": + # GraalPy has an incomplete sys.getrefcount implementation + def getrefcount(obj): + return 0 + before = getrefcount(message) store = ObjStore() for _ in range(N): diff --git a/pytests/tests/test_othermod.py b/pytests/tests/test_othermod.py index ff67bba435c..f2dd9ad8fd2 100644 --- a/pytests/tests/test_othermod.py +++ b/pytests/tests/test_othermod.py @@ -3,11 +3,16 @@ from pyo3_pytests import othermod -INTEGER32_ST = st.integers(min_value=(-(2**31)), max_value=(2**31 - 1)) +INTEGER31_ST = st.integers(min_value=(-(2**30)), max_value=(2**30 - 1)) USIZE_ST = st.integers(min_value=othermod.USIZE_MIN, max_value=othermod.USIZE_MAX) -@given(x=INTEGER32_ST) +# If the full 32 bits are used here, then you can get failures that look like this: +# hypothesis.errors.FailedHealthCheck: It looks like your strategy is filtering out a lot of data. +# Health check found 50 filtered examples but only 7 good ones. +# +# Limit the range to 31 bits to avoid this problem. +@given(x=INTEGER31_ST) def test_double(x): expected = x * 2 assume(-(2**31) <= expected <= (2**31 - 1)) diff --git a/pytests/tests/test_path.py b/pytests/tests/test_path.py index 21240187356..d1d6eb83924 100644 --- a/pytests/tests/test_path.py +++ b/pytests/tests/test_path.py @@ -7,21 +7,21 @@ def test_make_path(): p = rpath.make_path() - assert p == "/root" + assert p == pathlib.Path("/root") def test_take_pathbuf(): p = "/root" - assert rpath.take_pathbuf(p) == p + assert rpath.take_pathbuf(p) == pathlib.Path(p) def test_take_pathlib(): p = pathlib.Path("/root") - assert rpath.take_pathbuf(p) == str(p) + assert rpath.take_pathbuf(p) == p def test_take_pathlike(): - assert rpath.take_pathbuf(PathLike("/root")) == "/root" + assert rpath.take_pathbuf(PathLike("/root")) == pathlib.Path("/root") def test_take_invalid_pathlike(): diff --git a/pytests/tests/test_pyclasses.py b/pytests/tests/test_pyclasses.py index 9a9b44b52e8..04430736733 100644 --- a/pytests/tests/test_pyclasses.py +++ b/pytests/tests/test_pyclasses.py @@ -1,3 +1,5 @@ +import platform +import sys from typing import Type import pytest @@ -8,14 +10,38 @@ def test_empty_class_init(benchmark): benchmark(pyclasses.EmptyClass) +def test_method_call(benchmark): + obj = pyclasses.EmptyClass() + assert benchmark(obj.method) is None + + +def test_proto_call(benchmark): + obj = pyclasses.EmptyClass() + assert benchmark(len, obj) == 0 + + class EmptyClassPy: - pass + def method(self): + pass + + def __len__(self) -> int: + return 0 def test_empty_class_init_py(benchmark): benchmark(EmptyClassPy) +def test_method_call_py(benchmark): + obj = EmptyClassPy() + assert benchmark(obj.method) == pyclasses.EmptyClass().method() + + +def test_proto_call_py(benchmark): + obj = EmptyClassPy() + assert benchmark(len, obj) == len(pyclasses.EmptyClass()) + + def test_iter(): i = pyclasses.PyClassIter() assert next(i) == 1 @@ -29,22 +55,31 @@ def test_iter(): assert excinfo.value.value == "Ended" -class AssertingSubClass(pyclasses.AssertingBaseClass): - pass +@pytest.mark.skipif( + platform.machine() in ["wasm32", "wasm64"], + reason="not supporting threads in CI for WASM yet", +) +def test_parallel_iter(): + import concurrent.futures + i = pyclasses.PyClassThreadIter() -def test_new_classmethod(): - # The `AssertingBaseClass` constructor errors if it is not passed the - # relevant subclass. - _ = AssertingSubClass(expected_type=AssertingSubClass) - with pytest.raises(ValueError): - _ = AssertingSubClass(expected_type=str) + # the second thread attempts to borrow a reference to the instance's + # state while the first thread is still sleeping, so we trigger a + # runtime borrow-check error + with pytest.raises(RuntimeError, match="Already borrowed"): + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as tpe: + # should never reach 100 iterations, should error out as soon + # as the borrow error occurs + for _ in tpe.map(lambda _: next(i), range(100)): + pass -def test_new_classmethod_gil_ref(): - class AssertingSubClass(pyclasses.AssertingBaseClassGilRef): - pass +class AssertingSubClass(pyclasses.AssertingBaseClass): + pass + +def test_new_classmethod(): # The `AssertingBaseClass` constructor errors if it is not passed the # relevant subclass. _ = AssertingSubClass(expected_type=AssertingSubClass) @@ -52,22 +87,80 @@ class AssertingSubClass(pyclasses.AssertingBaseClassGilRef): _ = AssertingSubClass(expected_type=str) -class ClassWithoutConstructorPy: +class ClassWithoutConstructor: def __new__(cls): - raise TypeError("No constructor defined") + raise TypeError( + f"cannot create '{cls.__module__}.{cls.__qualname__}' instances" + ) +@pytest.mark.xfail( + platform.python_implementation() == "PyPy" and sys.version_info[:2] == (3, 11), + reason="broken on PyPy 3.11 due to https://github.com/pypy/pypy/issues/5319, waiting for next release", +) @pytest.mark.parametrize( - "cls", [pyclasses.ClassWithoutConstructor, ClassWithoutConstructorPy] + "cls, exc_message", + [ + ( + pyclasses.ClassWithoutConstructor, + "cannot create 'builtins.ClassWithoutConstructor' instances", + ), + ( + ClassWithoutConstructor, + "cannot create 'test_pyclasses.ClassWithoutConstructor' instances", + ), + ], ) -def test_no_constructor_defined_propagates_cause(cls: Type): +def test_no_constructor_defined_propagates_cause(cls: Type, exc_message: str): original_error = ValueError("Original message") with pytest.raises(Exception) as exc_info: try: raise original_error except Exception: - cls() # should raise TypeError("No constructor defined") + cls() # should raise TypeError("No constructor defined for ...") assert exc_info.type is TypeError - assert exc_info.value.args == ("No constructor defined",) + assert exc_info.value.args == (exc_message,) assert exc_info.value.__context__ is original_error + + +def test_dict(): + try: + ClassWithDict = pyclasses.ClassWithDict + except AttributeError: + pytest.skip("not defined using abi3 < 3.9") + + d = ClassWithDict() + assert d.__dict__ == {} + + d.foo = 42 + assert d.__dict__ == {"foo": 42} + + +def test_getter(benchmark): + obj = pyclasses.ClassWithDecorators() + benchmark(lambda: obj.attr) + + +def test_setter(benchmark): + obj = pyclasses.ClassWithDecorators() + + def set_attr(): + obj.attr = 42 + + benchmark(set_attr) + + +def test_class_attribute(benchmark): + cls = pyclasses.ClassWithDecorators + benchmark(lambda: cls.cls_attribute) + + +def test_class_method(benchmark): + cls = pyclasses.ClassWithDecorators + benchmark(lambda: cls.cls_method()) + + +def test_static_method(benchmark): + cls = pyclasses.ClassWithDecorators + benchmark(lambda: cls.static_method()) diff --git a/pytests/tests/test_pyfunctions.py b/pytests/tests/test_pyfunctions.py index c6fb448248b..e33d4d6f102 100644 --- a/pytests/tests/test_pyfunctions.py +++ b/pytests/tests/test_pyfunctions.py @@ -1,3 +1,5 @@ +from typing import Any, Tuple + from pyo3_pytests import pyfunctions @@ -58,7 +60,7 @@ def test_simple_kwargs_rs(benchmark): def simple_args_kwargs_py(a, b=None, *args, c=None, **kwargs): - return (a, b, args, c, kwargs) + return a, b, args, c, kwargs def test_simple_args_kwargs_py(benchmark): @@ -72,7 +74,7 @@ def test_simple_args_kwargs_rs(benchmark): def args_kwargs_py(*args, **kwargs): - return (args, kwargs) + return args, kwargs def test_args_kwargs_py(benchmark): @@ -83,3 +85,88 @@ def test_args_kwargs_rs(benchmark): rust = benchmark(pyfunctions.args_kwargs, 1, "foo", {1: 2}, bar=4, foo=10) py = args_kwargs_py(1, "foo", {1: 2}, bar=4, foo=10) assert rust == py + + +# TODO: the second argument should be positional-only +# but can't be without breaking tests on Python 3.7. +# See gh-5095. +def positional_only_py(a, b): + return a, b + + +def test_positional_only_py(benchmark): + benchmark(positional_only_py, 1, "foo") + + +def test_positional_only_rs(benchmark): + rust = benchmark(pyfunctions.positional_only, 1, "foo") + py = positional_only_py(1, "foo") + assert rust == py + + +def with_typed_args_py( + a: bool, b: int, c: float, d: str +) -> Tuple[bool, int, float, str]: + return a, b, c, d + + +def test_with_typed_args_py(benchmark): + benchmark(with_typed_args_py, True, 1, 1.2, "foo") + + +def test_with_typed_args_rs(benchmark): + rust = benchmark(pyfunctions.with_typed_args, True, 1, 1.2, "foo") + py = with_typed_args_py(True, 1, 1.2, "foo") + assert rust == py + + +def many_keyword_arguments_py( + *, + ant: Any = None, + bear: Any = None, + cat: Any = None, + dog: Any = None, + elephant: Any = None, + fox: Any = None, + goat: Any = None, + horse: Any = None, + iguana: Any = None, + jaguar: Any = None, + koala: Any = None, + lion: Any = None, + monkey: Any = None, + newt: Any = None, + owl: Any = None, + penguin: Any = None, +): ... + + +def call_with_many_keyword_arguments(f) -> Any: + return f( + ant=True, + bear=1, + cat=1.2, + dog="foo", + elephant=None, + fox=8, + goat=9, + horse=10, + iguana=None, + jaguar=None, + koala=None, + lion=11, + owl=None, + penguin=None, + ) + + +def test_many_keyword_arguments_py(benchmark): + benchmark(call_with_many_keyword_arguments, many_keyword_arguments_py) + + +def test_many_keyword_arguments_rs(benchmark): + rust = benchmark( + call_with_many_keyword_arguments, pyfunctions.many_keyword_arguments + ) + py = call_with_many_keyword_arguments(many_keyword_arguments_py) + assert rust == py diff --git a/src/buffer.rs b/src/buffer.rs index 84b08289771..081f520e795 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -19,37 +19,51 @@ //! `PyBuffer` implementation use crate::{err, exceptions::PyBufferError, ffi, FromPyObject, PyAny, PyResult, Python}; -use crate::{Bound, PyNativeType}; -use std::marker::PhantomData; -use std::os::raw; +use crate::{Borrowed, Bound, PyErr}; +use std::ffi::{ + c_char, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, c_ulonglong, + c_ushort, c_void, +}; +use std::marker::{PhantomData, PhantomPinned}; use std::pin::Pin; use std::{cell, mem, ptr, slice}; use std::{ffi::CStr, fmt::Debug}; /// Allows access to the underlying buffer used by a python object such as `bytes`, `bytearray` or `array.array`. -// use Pin because Python expects that the Py_buffer struct has a stable memory address #[repr(transparent)] -pub struct PyBuffer(Pin>, PhantomData); +pub struct PyBuffer( + // It is common for exporters filling `Py_buffer` struct to make it self-referential, e.g. see + // implementation of + // [`PyBuffer_FillInfo`](https://github.com/python/cpython/blob/2fd43a1ffe4ff1f6c46f6045bc327d6085c40fbf/Objects/abstract.c#L798-L802). + // + // Therefore we use `Pin>` to ensure that the memory address of the `Py_buffer` is stable + Pin>, + PhantomData, +); + +/// Wrapper around `ffi::Py_buffer` to be `!Unpin`. +#[repr(transparent)] +struct RawBuffer(ffi::Py_buffer, PhantomPinned); // PyBuffer is thread-safe: the shape of the buffer is immutable while a Py_buffer exists. -// Accessing the buffer contents is protected using the GIL. unsafe impl Send for PyBuffer {} unsafe impl Sync for PyBuffer {} impl Debug for PyBuffer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let raw = self.raw(); f.debug_struct("PyBuffer") - .field("buf", &self.0.buf) - .field("obj", &self.0.obj) - .field("len", &self.0.len) - .field("itemsize", &self.0.itemsize) - .field("readonly", &self.0.readonly) - .field("ndim", &self.0.ndim) - .field("format", &self.0.format) - .field("shape", &self.0.shape) - .field("strides", &self.0.strides) - .field("suboffsets", &self.0.suboffsets) - .field("internal", &self.0.internal) + .field("buf", &raw.buf) + .field("obj", &raw.obj) + .field("len", &raw.len) + .field("itemsize", &raw.itemsize) + .field("readonly", &raw.readonly) + .field("ndim", &raw.ndim) + .field("format", &self.format()) + .field("shape", &self.shape()) + .field("strides", &self.strides()) + .field("suboffsets", &self.suboffsets()) + .field("internal", &raw.internal) .finish() } } @@ -96,38 +110,38 @@ fn native_element_type_from_type_char(type_char: u8) -> ElementType { use self::ElementType::*; match type_char { b'c' => UnsignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'b' => SignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'B' => UnsignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'?' => Bool, b'h' => SignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'H' => UnsignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'i' => SignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'I' => UnsignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'l' => SignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'L' => UnsignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'q' => SignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'Q' => UnsignedInteger { - bytes: mem::size_of::(), + bytes: mem::size_of::(), }, b'n' => SignedInteger { bytes: mem::size_of::(), @@ -182,51 +196,45 @@ pub unsafe trait Element: Copy { fn is_compatible_format(format: &CStr) -> bool; } -impl<'py, T: Element> FromPyObject<'py> for PyBuffer { - fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult> { - Self::get_bound(obj) +impl FromPyObject<'_, '_> for PyBuffer { + type Error = PyErr; + + fn extract(obj: Borrowed<'_, '_, PyAny>) -> Result, Self::Error> { + Self::get(&obj) } } impl PyBuffer { - /// Deprecated form of [`PyBuffer::get_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyBuffer::get` will be replaced by `PyBuffer::get_bound` in a future PyO3 version" - ) - )] - pub fn get(obj: &PyAny) -> PyResult> { - Self::get_bound(&obj.as_borrowed()) - } - /// Gets the underlying buffer from the specified python object. - pub fn get_bound(obj: &Bound<'_, PyAny>) -> PyResult> { - // TODO: use nightly API Box::new_uninit() once stable - let mut buf = Box::new(mem::MaybeUninit::uninit()); - let buf: Box = { + pub fn get(obj: &Bound<'_, PyAny>) -> PyResult> { + let mut buf = Box::::new_uninit(); + let buf: Box = { err::error_on_minusone(obj.py(), unsafe { - ffi::PyObject_GetBuffer(obj.as_ptr(), buf.as_mut_ptr(), ffi::PyBUF_FULL_RO) + ffi::PyObject_GetBuffer( + obj.as_ptr(), + // SAFETY: RawBuffer is `#[repr(transparent)]` around FFI struct + buf.as_mut_ptr().cast::(), + ffi::PyBUF_FULL_RO, + ) })?; // Safety: buf is initialized by PyObject_GetBuffer. - // TODO: use nightly API Box::assume_init() once stable - unsafe { mem::transmute(buf) } + unsafe { buf.assume_init() } }; // Create PyBuffer immediately so that if validation checks fail, the PyBuffer::drop code // will call PyBuffer_Release (thus avoiding any leaks). let buf = PyBuffer(Pin::from(buf), PhantomData); + let raw = buf.raw(); - if buf.0.shape.is_null() { + if raw.shape.is_null() { Err(PyBufferError::new_err("shape is null")) - } else if buf.0.strides.is_null() { + } else if raw.strides.is_null() { Err(PyBufferError::new_err("strides is null")) } else if mem::size_of::() != buf.item_size() || !T::is_compatible_format(buf.format()) { Err(PyBufferError::new_err(format!( "buffer contents are not compatible with {}", std::any::type_name::() ))) - } else if buf.0.buf.align_offset(mem::align_of::()) != 0 { + } else if raw.buf.align_offset(mem::align_of::()) != 0 { Err(PyBufferError::new_err(format!( "buffer contents are insufficiently aligned for {}", std::any::type_name::() @@ -235,20 +243,27 @@ impl PyBuffer { Ok(buf) } } +} +impl PyBuffer { /// Gets the pointer to the start of the buffer memory. /// - /// Warning: the buffer memory might be mutated by other Python functions, - /// and thus may only be accessed while the GIL is held. + /// Warning: the buffer memory can be mutated by other code (including + /// other Python functions, if the GIL is released, or other extension + /// modules even if the GIL is held). You must either access memory + /// atomically, or ensure there are no data races yourself. See + /// [this blog post] for more details. + /// + /// [this blog post]: https://alexgaynor.net/2022/oct/23/buffers-on-the-edge/ #[inline] - pub fn buf_ptr(&self) -> *mut raw::c_void { - self.0.buf + pub fn buf_ptr(&self) -> *mut c_void { + self.raw().buf } /// Gets a pointer to the specified item. /// /// If `indices.len() < self.dimensions()`, returns the start address of the sub-array at the specified dimension. - pub fn get_ptr(&self, indices: &[usize]) -> *mut raw::c_void { + pub fn get_ptr(&self, indices: &[usize]) -> *mut c_void { let shape = &self.shape()[..indices.len()]; for i in 0..indices.len() { assert!(indices[i] < shape[i]); @@ -256,14 +271,14 @@ impl PyBuffer { unsafe { ffi::PyBuffer_GetPointer( #[cfg(Py_3_11)] - &*self.0, + self.raw(), #[cfg(not(Py_3_11))] { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + self.raw() as *const ffi::Py_buffer as *mut ffi::Py_buffer }, #[cfg(Py_3_11)] { - indices.as_ptr() as *const ffi::Py_ssize_t + indices.as_ptr().cast() }, #[cfg(not(Py_3_11))] { @@ -276,20 +291,20 @@ impl PyBuffer { /// Gets whether the underlying buffer is read-only. #[inline] pub fn readonly(&self) -> bool { - self.0.readonly != 0 + self.raw().readonly != 0 } /// Gets the size of a single element, in bytes. /// Important exception: when requesting an unformatted buffer, item_size still has the value #[inline] pub fn item_size(&self) -> usize { - self.0.itemsize as usize + self.raw().itemsize as usize } /// Gets the total number of items. #[inline] pub fn item_count(&self) -> usize { - (self.0.len as usize) / (self.0.itemsize as usize) + (self.raw().len as usize) / (self.raw().itemsize as usize) } /// `item_size() * item_count()`. @@ -297,7 +312,7 @@ impl PyBuffer { /// For non-contiguous arrays, it is the length that the logical structure would have if it were copied to a contiguous representation. #[inline] pub fn len_bytes(&self) -> usize { - self.0.len as usize + self.raw().len as usize } /// Gets the number of dimensions. @@ -305,7 +320,7 @@ impl PyBuffer { /// May be 0 to indicate a single scalar value. #[inline] pub fn dimensions(&self) -> usize { - self.0.ndim as usize + self.raw().ndim as usize } /// Returns an array of length `dimensions`. `shape()[i]` is the length of the array in dimension number `i`. @@ -317,7 +332,7 @@ impl PyBuffer { /// However, dimensions of length 0 are possible and might need special attention. #[inline] pub fn shape(&self) -> &[usize] { - unsafe { slice::from_raw_parts(self.0.shape as *const usize, self.0.ndim as usize) } + unsafe { slice::from_raw_parts(self.raw().shape.cast(), self.raw().ndim as usize) } } /// Returns an array that holds, for each dimension, the number of bytes to skip to get to the next element in the dimension. @@ -326,7 +341,7 @@ impl PyBuffer { /// but a consumer MUST be able to handle the case `strides[n] <= 0`. #[inline] pub fn strides(&self) -> &[isize] { - unsafe { slice::from_raw_parts(self.0.strides, self.0.ndim as usize) } + unsafe { slice::from_raw_parts(self.raw().strides, self.raw().ndim as usize) } } /// An array of length ndim. @@ -337,12 +352,12 @@ impl PyBuffer { #[inline] pub fn suboffsets(&self) -> Option<&[isize]> { unsafe { - if self.0.suboffsets.is_null() { + if self.raw().suboffsets.is_null() { None } else { Some(slice::from_raw_parts( - self.0.suboffsets, - self.0.ndim as usize, + self.raw().suboffsets, + self.raw().ndim as usize, )) } } @@ -351,35 +366,31 @@ impl PyBuffer { /// A NUL terminated string in struct module style syntax describing the contents of a single item. #[inline] pub fn format(&self) -> &CStr { - if self.0.format.is_null() { - CStr::from_bytes_with_nul(b"B\0").unwrap() + if self.raw().format.is_null() { + c"B" } else { - unsafe { CStr::from_ptr(self.0.format) } + unsafe { CStr::from_ptr(self.raw().format) } } } /// Gets whether the buffer is contiguous in C-style order (last index varies fastest when visiting items in order of memory address). #[inline] pub fn is_c_contiguous(&self) -> bool { - unsafe { - ffi::PyBuffer_IsContiguous( - &*self.0 as *const ffi::Py_buffer, - b'C' as std::os::raw::c_char, - ) != 0 - } + unsafe { ffi::PyBuffer_IsContiguous(self.raw(), b'C' as std::ffi::c_char) != 0 } } /// Gets whether the buffer is contiguous in Fortran-style order (first index varies fastest when visiting items in order of memory address). #[inline] pub fn is_fortran_contiguous(&self) -> bool { - unsafe { - ffi::PyBuffer_IsContiguous( - &*self.0 as *const ffi::Py_buffer, - b'F' as std::os::raw::c_char, - ) != 0 - } + unsafe { ffi::PyBuffer_IsContiguous(self.raw(), b'F' as std::ffi::c_char) != 0 } + } + + fn raw(&self) -> &ffi::Py_buffer { + &self.0 .0 } +} +impl PyBuffer { /// Gets the buffer memory as a slice. /// /// This function succeeds if: @@ -393,7 +404,7 @@ impl PyBuffer { if self.is_c_contiguous() { unsafe { Some(slice::from_raw_parts( - self.0.buf as *mut ReadOnlyCell, + self.raw().buf as *mut ReadOnlyCell, self.item_count(), )) } @@ -416,7 +427,7 @@ impl PyBuffer { if !self.readonly() && self.is_c_contiguous() { unsafe { Some(slice::from_raw_parts( - self.0.buf as *mut cell::Cell, + self.raw().buf as *mut cell::Cell, self.item_count(), )) } @@ -438,7 +449,7 @@ impl PyBuffer { if mem::size_of::() == self.item_size() && self.is_fortran_contiguous() { unsafe { Some(slice::from_raw_parts( - self.0.buf as *mut ReadOnlyCell, + self.raw().buf as *mut ReadOnlyCell, self.item_count(), )) } @@ -461,7 +472,7 @@ impl PyBuffer { if !self.readonly() && self.is_fortran_contiguous() { unsafe { Some(slice::from_raw_parts( - self.0.buf as *mut cell::Cell, + self.raw().buf as *mut cell::Cell, self.item_count(), )) } @@ -509,13 +520,13 @@ impl PyBuffer { ffi::PyBuffer_ToContiguous( target.as_mut_ptr().cast(), #[cfg(Py_3_11)] - &*self.0, + self.raw(), #[cfg(not(Py_3_11))] { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + self.raw() as *const ffi::Py_buffer as *mut ffi::Py_buffer }, - self.0.len, - fort as std::os::raw::c_char, + self.raw().len, + fort as std::ffi::c_char, ) }) } @@ -544,15 +555,15 @@ impl PyBuffer { // Due to T:Copy, we don't need to be concerned with Drop impls. err::error_on_minusone(py, unsafe { ffi::PyBuffer_ToContiguous( - vec.as_ptr() as *mut raw::c_void, + vec.as_ptr() as *mut c_void, #[cfg(Py_3_11)] - &*self.0, + self.raw(), #[cfg(not(Py_3_11))] { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + self.raw() as *const ffi::Py_buffer as *mut ffi::Py_buffer }, - self.0.len, - fort as std::os::raw::c_char, + self.raw().len, + fort as std::ffi::c_char, ) })?; // set vector length to mark the now-initialized space as usable @@ -602,21 +613,21 @@ impl PyBuffer { err::error_on_minusone(py, unsafe { ffi::PyBuffer_FromContiguous( #[cfg(Py_3_11)] - &*self.0, + self.raw(), #[cfg(not(Py_3_11))] { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + self.raw() as *const ffi::Py_buffer as *mut ffi::Py_buffer }, #[cfg(Py_3_11)] { - source.as_ptr() as *const raw::c_void + source.as_ptr().cast() }, #[cfg(not(Py_3_11))] { - source.as_ptr() as *mut raw::c_void + source.as_ptr() as *mut c_void }, - self.0.len, - fort as std::os::raw::c_char, + self.raw().len, + fort as std::ffi::c_char, ) }) } @@ -627,24 +638,52 @@ impl PyBuffer { /// This will automatically be called on drop. pub fn release(self, _py: Python<'_>) { // First move self into a ManuallyDrop, so that PyBuffer::drop will - // never be called. (It would acquire the GIL and call PyBuffer_Release + // never be called. (It would attach to the interpreter and call PyBuffer_Release // again.) let mut mdself = mem::ManuallyDrop::new(self); unsafe { // Next, make the actual PyBuffer_Release call. - ffi::PyBuffer_Release(&mut *mdself.0); + // Fine to get a mutable reference to the inner ffi::Py_buffer here, as we're destroying it. + mdself.0.release(); // Finally, drop the contained Pin> in place, to free the // Box memory. - let inner: *mut Pin> = &mut mdself.0; - ptr::drop_in_place(inner); + ptr::drop_in_place::>>(&mut mdself.0); + } + } +} + +impl RawBuffer { + /// Release the contents of this pinned buffer. + /// + /// # Safety + /// + /// - The buffer must not be used after calling this function. + /// - This function can only be called once. + /// - Must be attached to the interpreter. + /// + unsafe fn release(self: &mut Pin>) { + unsafe { + ffi::PyBuffer_Release(&mut Pin::get_unchecked_mut(self.as_mut()).0); } } } impl Drop for PyBuffer { fn drop(&mut self) { - Python::with_gil(|_| unsafe { ffi::PyBuffer_Release(&mut *self.0) }); + fn inner(buf: &mut Pin>) { + if Python::try_attach(|_| unsafe { buf.release() }).is_none() + && crate::internal::state::is_in_gc_traversal() + { + eprintln!("Warning: PyBuffer dropped while in GC traversal, this is a bug and will leak memory."); + } + // If `try_attach` failed and `is_in_gc_traversal()` is false, then probably the interpreter has + // already finalized and we can just assume that the underlying memory has already been freed. + // + // So we don't handle that case here. + } + + inner(&mut self.0); } } @@ -699,30 +738,28 @@ impl_element!(f64, Float); #[cfg(test)] mod tests { - use super::PyBuffer; + use super::*; + use crate::ffi; use crate::types::any::PyAnyMethods; + use crate::types::PyBytes; use crate::Python; #[test] fn test_debug() { - Python::with_gil(|py| { - let bytes = py.eval_bound("b'abcde'", None, None).unwrap(); - let buffer: PyBuffer = PyBuffer::get_bound(&bytes).unwrap(); + Python::attach(|py| { + let bytes = PyBytes::new(py, b"abcde"); + let buffer: PyBuffer = PyBuffer::get(&bytes).unwrap(); let expected = format!( concat!( "PyBuffer {{ buf: {:?}, obj: {:?}, ", "len: 5, itemsize: 1, readonly: 1, ", - "ndim: 1, format: {:?}, shape: {:?}, ", - "strides: {:?}, suboffsets: {:?}, internal: {:?} }}", + "ndim: 1, format: \"B\", shape: [5], ", + "strides: [1], suboffsets: None, internal: {:?} }}", ), - buffer.0.buf, - buffer.0.obj, - buffer.0.format, - buffer.0.shape, - buffer.0.strides, - buffer.0.suboffsets, - buffer.0.internal + buffer.raw().buf, + buffer.raw().obj, + buffer.raw().internal ); let debug_repr = format!("{:?}", buffer); assert_eq!(debug_repr, expected); @@ -731,129 +768,125 @@ mod tests { #[test] fn test_element_type_from_format() { - use super::ElementType; use super::ElementType::*; - use std::ffi::CStr; use std::mem::size_of; - use std::os::raw; - for (cstr, expected) in &[ + for (cstr, expected) in [ // @ prefix goes to native_element_type_from_type_char ( - "@b\0", + c"@b", SignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@c\0", + c"@c", UnsignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@b\0", + c"@b", SignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@B\0", + c"@B", UnsignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), - ("@?\0", Bool), + (c"@?", Bool), ( - "@h\0", + c"@h", SignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@H\0", + c"@H", UnsignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@i\0", + c"@i", SignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@I\0", + c"@I", UnsignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@l\0", + c"@l", SignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@L\0", + c"@L", UnsignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@q\0", + c"@q", SignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@Q\0", + c"@Q", UnsignedInteger { - bytes: size_of::(), + bytes: size_of::(), }, ), ( - "@n\0", + c"@n", SignedInteger { bytes: size_of::(), }, ), ( - "@N\0", + c"@N", UnsignedInteger { bytes: size_of::(), }, ), - ("@e\0", Float { bytes: 2 }), - ("@f\0", Float { bytes: 4 }), - ("@d\0", Float { bytes: 8 }), - ("@z\0", Unknown), + (c"@e", Float { bytes: 2 }), + (c"@f", Float { bytes: 4 }), + (c"@d", Float { bytes: 8 }), + (c"@z", Unknown), // = prefix goes to standard_element_type_from_type_char - ("=b\0", SignedInteger { bytes: 1 }), - ("=c\0", UnsignedInteger { bytes: 1 }), - ("=B\0", UnsignedInteger { bytes: 1 }), - ("=?\0", Bool), - ("=h\0", SignedInteger { bytes: 2 }), - ("=H\0", UnsignedInteger { bytes: 2 }), - ("=l\0", SignedInteger { bytes: 4 }), - ("=l\0", SignedInteger { bytes: 4 }), - ("=I\0", UnsignedInteger { bytes: 4 }), - ("=L\0", UnsignedInteger { bytes: 4 }), - ("=q\0", SignedInteger { bytes: 8 }), - ("=Q\0", UnsignedInteger { bytes: 8 }), - ("=e\0", Float { bytes: 2 }), - ("=f\0", Float { bytes: 4 }), - ("=d\0", Float { bytes: 8 }), - ("=z\0", Unknown), - ("=0\0", Unknown), + (c"=b", SignedInteger { bytes: 1 }), + (c"=c", UnsignedInteger { bytes: 1 }), + (c"=B", UnsignedInteger { bytes: 1 }), + (c"=?", Bool), + (c"=h", SignedInteger { bytes: 2 }), + (c"=H", UnsignedInteger { bytes: 2 }), + (c"=l", SignedInteger { bytes: 4 }), + (c"=l", SignedInteger { bytes: 4 }), + (c"=I", UnsignedInteger { bytes: 4 }), + (c"=L", UnsignedInteger { bytes: 4 }), + (c"=q", SignedInteger { bytes: 8 }), + (c"=Q", UnsignedInteger { bytes: 8 }), + (c"=e", Float { bytes: 2 }), + (c"=f", Float { bytes: 4 }), + (c"=d", Float { bytes: 8 }), + (c"=z", Unknown), + (c"=0", Unknown), // unknown prefix -> Unknown - (":b\0", Unknown), + (c":b", Unknown), ] { assert_eq!( - ElementType::from_format(CStr::from_bytes_with_nul(cstr.as_bytes()).unwrap()), - *expected, - "element from format &Cstr: {:?}", - cstr, + ElementType::from_format(cstr), + expected, + "element from format &Cstr: {cstr:?}", ); } } @@ -869,9 +902,9 @@ mod tests { #[test] fn test_bytes_buffer() { - Python::with_gil(|py| { - let bytes = py.eval_bound("b'abcde'", None, None).unwrap(); - let buffer = PyBuffer::get_bound(&bytes).unwrap(); + Python::attach(|py| { + let bytes = PyBytes::new(py, b"abcde"); + let buffer = PyBuffer::get(&bytes).unwrap(); assert_eq!(buffer.dimensions(), 1); assert_eq!(buffer.item_count(), 5); assert_eq!(buffer.format().to_str().unwrap(), "B"); @@ -901,19 +934,19 @@ mod tests { #[test] fn test_array_buffer() { - Python::with_gil(|py| { + Python::attach(|py| { let array = py - .import_bound("array") + .import("array") .unwrap() .call_method("array", ("f", (1.0, 1.5, 2.0, 2.5)), None) .unwrap(); - let buffer = PyBuffer::get_bound(&array).unwrap(); + let buffer = PyBuffer::get(&array).unwrap(); assert_eq!(buffer.dimensions(), 1); assert_eq!(buffer.item_count(), 4); assert_eq!(buffer.format().to_str().unwrap(), "f"); assert_eq!(buffer.shape(), [4]); - // array creates a 1D contiguious buffer, so it's both C and F contiguous. This would + // array creates a 1D contiguous buffer, so it's both C and F contiguous. This would // be more interesting if we can come up with a 2D buffer but I think it would need a // third-party lib or a custom class. @@ -937,7 +970,7 @@ mod tests { assert_eq!(buffer.to_vec(py).unwrap(), [10.0, 11.0, 12.0, 13.0]); // F-contiguous fns - let buffer = PyBuffer::get_bound(&array).unwrap(); + let buffer = PyBuffer::get(&array).unwrap(); let slice = buffer.as_fortran_slice(py).unwrap(); assert_eq!(slice.len(), 4); assert_eq!(slice[1].get(), 11.0); diff --git a/src/call.rs b/src/call.rs new file mode 100644 index 00000000000..51e67246ef6 --- /dev/null +++ b/src/call.rs @@ -0,0 +1,312 @@ +//! Defines how Python calls are dispatched, see [`PyCallArgs`].for more information. + +use crate::ffi_ptr_ext::FfiPtrExt as _; +use crate::types::{PyAnyMethods as _, PyDict, PyString, PyTuple}; +use crate::{ffi, Borrowed, Bound, IntoPyObjectExt as _, Py, PyAny, PyResult}; + +pub(crate) mod private { + use super::*; + + pub trait Sealed {} + + impl Sealed for () {} + impl Sealed for Bound<'_, PyTuple> {} + impl Sealed for &'_ Bound<'_, PyTuple> {} + impl Sealed for Py {} + impl Sealed for &'_ Py {} + impl Sealed for Borrowed<'_, '_, PyTuple> {} + pub struct Token; +} + +/// This trait marks types that can be used as arguments to Python function +/// calls. +/// +/// This trait is currently implemented for Rust tuple (up to a size of 12), +/// [`Bound<'py, PyTuple>`] and [`Py`]. Custom types that are +/// convertable to `PyTuple` via `IntoPyObject` need to do so before passing it +/// to `call`. +/// +/// This trait is not intended to used by downstream crates directly. As such it +/// has no publicly available methods and cannot be implemented outside of +/// `pyo3`. The corresponding public API is available through [`call`] +/// ([`call0`], [`call1`] and friends) on [`PyAnyMethods`]. +/// +/// # What is `PyCallArgs` used for? +/// `PyCallArgs` is used internally in `pyo3` to dispatch the Python calls in +/// the most optimal way for the current build configuration. Certain types, +/// such as Rust tuples, do allow the usage of a faster calling convention of +/// the Python interpreter (if available). More types that may take advantage +/// from this may be added in the future. +/// +/// [`call0`]: crate::types::PyAnyMethods::call0 +/// [`call1`]: crate::types::PyAnyMethods::call1 +/// [`call`]: crate::types::PyAnyMethods::call +/// [`PyAnyMethods`]: crate::types::PyAnyMethods +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot used as a Python `call` argument", + note = "`PyCallArgs` is implemented for Rust tuples, `Bound<'py, PyTuple>` and `Py`", + note = "if your type is convertable to `PyTuple` via `IntoPyObject`, call `.into_pyobject(py)` manually", + note = "if you meant to pass the type as a single argument, wrap it in a 1-tuple, `(,)`" +)] +pub trait PyCallArgs<'py>: Sized + private::Sealed { + #[doc(hidden)] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult>; + + #[doc(hidden)] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult>; + + #[doc(hidden)] + fn call_method_positional( + self, + object: Borrowed<'_, 'py, PyAny>, + method_name: Borrowed<'_, 'py, PyString>, + _: private::Token, + ) -> PyResult> { + object + .getattr(method_name) + .and_then(|method| method.call1(self)) + } +} + +impl<'py> PyCallArgs<'py> for () { + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + let args = self.into_pyobject_or_pyerr(function.py())?; + args.call(function, kwargs, token) + } + + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + let args = self.into_pyobject_or_pyerr(function.py())?; + args.call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for Bound<'py, PyTuple> { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + self.as_borrowed().call(function, kwargs, token) + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + self.as_borrowed().call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for &'_ Bound<'py, PyTuple> { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + self.as_borrowed().call(function, kwargs, token) + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + self.as_borrowed().call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for Py { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + self.bind_borrowed(function.py()) + .call(function, kwargs, token) + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + self.bind_borrowed(function.py()) + .call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for &'_ Py { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + self.bind_borrowed(function.py()) + .call(function, kwargs, token) + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + self.bind_borrowed(function.py()) + .call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for Borrowed<'_, 'py, PyTuple> { + #[inline] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + _: private::Token, + ) -> PyResult> { + unsafe { + ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), kwargs.as_ptr()) + .assume_owned_or_err(function.py()) + } + } + + #[inline] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + _: private::Token, + ) -> PyResult> { + unsafe { + ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), std::ptr::null_mut()) + .assume_owned_or_err(function.py()) + } + } +} + +#[cfg(test)] +#[cfg(feature = "macros")] +mod tests { + use crate::{ + pyfunction, + types::{PyDict, PyTuple}, + Py, + }; + + #[pyfunction(signature = (*args, **kwargs), crate = "crate")] + fn args_kwargs( + args: Py, + kwargs: Option>, + ) -> (Py, Option>) { + (args, kwargs) + } + + #[test] + fn test_call() { + use crate::{ + types::{IntoPyDict, PyAnyMethods, PyDict, PyTuple}, + wrap_pyfunction, Py, Python, + }; + + Python::attach(|py| { + let f = wrap_pyfunction!(args_kwargs, py).unwrap(); + + let args = PyTuple::new(py, [1, 2, 3]).unwrap(); + let kwargs = &[("foo", 1), ("bar", 2)].into_py_dict(py).unwrap(); + + macro_rules! check_call { + ($args:expr, $kwargs:expr) => { + let (a, k): (Py, Py) = f + .call(args.clone(), Some(kwargs)) + .unwrap() + .extract() + .unwrap(); + assert!(a.is(&args)); + assert!(k.is(kwargs)); + }; + } + + // Bound<'py, PyTuple> + check_call!(args.clone(), kwargs); + + // &Bound<'py, PyTuple> + check_call!(&args, kwargs); + + // Py + check_call!(args.clone().unbind(), kwargs); + + // &Py + check_call!(&args.as_unbound(), kwargs); + + // Borrowed<'_, '_, PyTuple> + check_call!(args.as_borrowed(), kwargs); + }) + } + + #[test] + fn test_call_positional() { + use crate::{ + types::{PyAnyMethods, PyNone, PyTuple}, + wrap_pyfunction, Py, Python, + }; + + Python::attach(|py| { + let f = wrap_pyfunction!(args_kwargs, py).unwrap(); + + let args = PyTuple::new(py, [1, 2, 3]).unwrap(); + + macro_rules! check_call { + ($args:expr, $kwargs:expr) => { + let (a, k): (Py, Py) = + f.call1(args.clone()).unwrap().extract().unwrap(); + assert!(a.is(&args)); + assert!(k.is_none(py)); + }; + } + + // Bound<'py, PyTuple> + check_call!(args.clone(), kwargs); + + // &Bound<'py, PyTuple> + check_call!(&args, kwargs); + + // Py + check_call!(args.clone().unbind(), kwargs); + + // &Py + check_call!(args.as_unbound(), kwargs); + + // Borrowed<'_, '_, PyTuple> + check_call!(args.as_borrowed(), kwargs); + }) + } +} diff --git a/src/conversion.rs b/src/conversion.rs index d9536aa9445..d4f5c513173 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -1,576 +1,575 @@ //! Defines conversions between Rust and Python types. -use crate::err::{self, PyDowncastError, PyResult}; +use crate::err::PyResult; +use crate::impl_::pyclass::ExtractPyClassWithClone; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; +#[cfg(feature = "experimental-inspect")] +use crate::inspect::TypeHint; use crate::pyclass::boolean_struct::False; -use crate::type_object::PyTypeInfo; +use crate::pyclass::{PyClassGuardError, PyClassGuardMutError}; use crate::types::PyTuple; use crate::{ - ffi, gil, Bound, Py, PyAny, PyCell, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, + Borrowed, Bound, BoundObject, Py, PyAny, PyClass, PyClassGuard, PyErr, PyRef, PyRefMut, Python, }; -use std::cell::Cell; -use std::ptr::NonNull; +use std::convert::Infallible; +use std::marker::PhantomData; + +/// Defines a conversion from a Rust type to a Python object, which may fail. +/// +/// This trait has `#[derive(IntoPyObject)]` to automatically implement it for simple types and +/// `#[derive(IntoPyObjectRef)]` to implement the same for references. +/// +/// It functions similarly to std's [`TryInto`] trait, but requires a [`Python<'py>`] token +/// as an argument. +/// +/// The [`into_pyobject`][IntoPyObject::into_pyobject] method is designed for maximum flexibility and efficiency; it +/// - allows for a concrete Python type to be returned (the [`Target`][IntoPyObject::Target] associated type) +/// - allows for the smart pointer containing the Python object to be either `Bound<'py, Self::Target>` or `Borrowed<'a, 'py, Self::Target>` +/// to avoid unnecessary reference counting overhead +/// - allows for a custom error type to be returned in the event of a conversion error to avoid +/// unnecessarily creating a Python exception +/// +/// # See also +/// +/// - The [`IntoPyObjectExt`] trait, which provides convenience methods for common usages of +/// `IntoPyObject` which erase type information and convert errors to `PyErr`. +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be converted to a Python object", + note = "`IntoPyObject` is automatically implemented by the `#[pyclass]` macro", + note = "if you do not wish to have a corresponding Python type, implement it manually", + note = "if you do not own `{Self}` you can perform a manual conversion to one of the types in `pyo3::types::*`" +)] +pub trait IntoPyObject<'py>: Sized { + /// The Python output type + type Target; + /// The smart pointer type to use. + /// + /// This will usually be [`Bound<'py, Target>`], but in special cases [`Borrowed<'a, 'py, Target>`] can be + /// used to minimize reference counting overhead. + type Output: BoundObject<'py, Self::Target>; + /// The type returned in the event of a conversion error. + type Error: Into; -/// Returns a borrowed pointer to a Python object. -/// -/// The returned pointer will be valid for as long as `self` is. It may be null depending on the -/// implementation. -/// -/// # Examples -/// -/// ```rust -/// use pyo3::prelude::*; -/// use pyo3::types::PyString; -/// use pyo3::ffi; -/// -/// Python::with_gil(|py| { -/// let s: Py = "foo".into_py(py); -/// let ptr = s.as_ptr(); -/// -/// let is_really_a_pystring = unsafe { ffi::PyUnicode_CheckExact(ptr) }; -/// assert_eq!(is_really_a_pystring, 1); -/// }); -/// ``` -/// -/// # Safety -/// -/// For callers, it is your responsibility to make sure that the underlying Python object is not dropped too -/// early. For example, the following code will cause undefined behavior: -/// -/// ```rust,no_run -/// # use pyo3::prelude::*; -/// # use pyo3::ffi; -/// # -/// Python::with_gil(|py| { -/// let ptr: *mut ffi::PyObject = 0xabad1dea_u32.into_py(py).as_ptr(); -/// -/// let isnt_a_pystring = unsafe { -/// // `ptr` is dangling, this is UB -/// ffi::PyUnicode_CheckExact(ptr) -/// }; -/// # assert_eq!(isnt_a_pystring, 0); -/// }); -/// ``` -/// -/// This happens because the pointer returned by `as_ptr` does not carry any lifetime information -/// and the Python object is dropped immediately after the `0xabad1dea_u32.into_py(py).as_ptr()` -/// expression is evaluated. To fix the problem, bind Python object to a local variable like earlier -/// to keep the Python object alive until the end of its scope. -/// -/// Implementors must ensure this returns a valid pointer to a Python object, which borrows a reference count from `&self`. -pub unsafe trait AsPyPointer { - /// Returns the underlying FFI pointer as a borrowed pointer. - fn as_ptr(&self) -> *mut ffi::PyObject; + /// Extracts the type hint information for this type when it appears as a return value. + /// + /// For example, `Vec` would return `List[int]`. + /// The default implementation returns `Any`, which is correct for any type. + /// + /// For most types, the return value for this method will be identical to that of [`FromPyObject::INPUT_TYPE`]. + /// It may be different for some types, such as `Dict`, to allow duck-typing: functions return `Dict` but take `Mapping` as argument. + #[cfg(feature = "experimental-inspect")] + const OUTPUT_TYPE: TypeHint = TypeHint::module_attr("typing", "Any"); + + /// Performs the conversion. + fn into_pyobject(self, py: Python<'py>) -> Result; + + /// Extracts the type hint information for this type when it appears as a return value. + /// + /// For example, `Vec` would return `List[int]`. + /// The default implementation returns `Any`, which is correct for any type. + /// + /// For most types, the return value for this method will be identical to that of [`FromPyObject::type_input`]. + /// It may be different for some types, such as `Dict`, to allow duck-typing: functions return `Dict` but take `Mapping` as argument. + #[cfg(feature = "experimental-inspect")] + fn type_output() -> TypeInfo { + TypeInfo::Any + } + + /// Converts sequence of Self into a Python object. Used to specialize `Vec`, `[u8; N]` + /// and `SmallVec<[u8; N]>` as a sequence of bytes into a `bytes` object. + #[doc(hidden)] + fn owned_sequence_into_pyobject( + iter: I, + py: Python<'py>, + _: private::Token, + ) -> Result, PyErr> + where + I: IntoIterator + AsRef<[Self]>, + I::IntoIter: ExactSizeIterator, + { + let mut iter = iter.into_iter().map(|e| e.into_bound_py_any(py)); + let list = crate::types::list::try_new_from_iter(py, &mut iter); + list.map(Bound::into_any) + } + + /// Converts sequence of Self into a Python object. Used to specialize `&[u8]` and `Cow<[u8]>` + /// as a sequence of bytes into a `bytes` object. + #[doc(hidden)] + fn borrowed_sequence_into_pyobject( + iter: I, + py: Python<'py>, + _: private::Token, + ) -> Result, PyErr> + where + Self: private::Reference, + I: IntoIterator + AsRef<[::BaseType]>, + I::IntoIter: ExactSizeIterator, + { + let mut iter = iter.into_iter().map(|e| e.into_bound_py_any(py)); + let list = crate::types::list::try_new_from_iter(py, &mut iter); + list.map(Bound::into_any) + } +} + +pub(crate) mod private { + pub struct Token; + + pub trait Reference { + type BaseType; + } + + impl Reference for &'_ T { + type BaseType = T; + } +} + +impl<'py, T> IntoPyObject<'py> for Bound<'py, T> { + type Target = T; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, _py: Python<'py>) -> Result { + Ok(self) + } +} + +impl<'a, 'py, T> IntoPyObject<'py> for &'a Bound<'py, T> { + type Target = T; + type Output = Borrowed<'a, 'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, _py: Python<'py>) -> Result { + Ok(self.as_borrowed()) + } +} + +impl<'a, 'py, T> IntoPyObject<'py> for Borrowed<'a, 'py, T> { + type Target = T; + type Output = Borrowed<'a, 'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, _py: Python<'py>) -> Result { + Ok(self) + } +} + +impl<'a, 'py, T> IntoPyObject<'py> for &Borrowed<'a, 'py, T> { + type Target = T; + type Output = Borrowed<'a, 'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, _py: Python<'py>) -> Result { + Ok(*self) + } +} + +impl<'py, T> IntoPyObject<'py> for Py { + type Target = T; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.into_bound(py)) + } } -/// Convert `None` into a null pointer. -unsafe impl AsPyPointer for Option +impl<'a, 'py, T> IntoPyObject<'py> for &'a Py { + type Target = T; + type Output = Borrowed<'a, 'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.bind_borrowed(py)) + } +} + +impl<'a, 'py, T> IntoPyObject<'py> for &&'a T where - T: AsPyPointer, + &'a T: IntoPyObject<'py>, { + type Target = <&'a T as IntoPyObject<'py>>::Target; + type Output = <&'a T as IntoPyObject<'py>>::Output; + type Error = <&'a T as IntoPyObject<'py>>::Error; + + #[cfg(feature = "experimental-inspect")] + const OUTPUT_TYPE: TypeHint = <&'a T as IntoPyObject<'py>>::OUTPUT_TYPE; + #[inline] - fn as_ptr(&self) -> *mut ffi::PyObject { - self.as_ref() - .map_or_else(std::ptr::null_mut, |t| t.as_ptr()) + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } -/// Conversion trait that allows various objects to be converted into `PyObject`. -pub trait ToPyObject { - /// Converts self into a Python object. - fn to_object(&self, py: Python<'_>) -> PyObject; +mod into_pyobject_ext { + pub trait Sealed {} + impl<'py, T> Sealed for T where T: super::IntoPyObject<'py> {} +} + +/// Convenience methods for common usages of [`IntoPyObject`]. Every type that implements +/// [`IntoPyObject`] also implements this trait. +/// +/// These methods: +/// - Drop type information from the output, returning a `PyAny` object. +/// - Always convert the `Error` type to `PyErr`, which may incur a performance penalty but it +/// more convenient in contexts where the `?` operator would produce a `PyErr` anyway. +pub trait IntoPyObjectExt<'py>: IntoPyObject<'py> + into_pyobject_ext::Sealed { + /// Converts `self` into an owned Python object, dropping type information. + #[inline] + fn into_bound_py_any(self, py: Python<'py>) -> PyResult> { + match self.into_pyobject(py) { + Ok(obj) => Ok(obj.into_any().into_bound()), + Err(err) => Err(err.into()), + } + } + + /// Converts `self` into an owned Python object, dropping type information and unbinding it + /// from the `'py` lifetime. + #[inline] + fn into_py_any(self, py: Python<'py>) -> PyResult> { + match self.into_pyobject(py) { + Ok(obj) => Ok(obj.into_any().unbind()), + Err(err) => Err(err.into()), + } + } + + /// Converts `self` into a Python object. + /// + /// This is equivalent to calling [`into_pyobject`][IntoPyObject::into_pyobject] followed + /// with `.map_err(Into::into)` to convert the error type to [`PyErr`]. This is helpful + /// for generic code which wants to make use of the `?` operator. + #[inline] + fn into_pyobject_or_pyerr(self, py: Python<'py>) -> PyResult { + match self.into_pyobject(py) { + Ok(obj) => Ok(obj), + Err(err) => Err(err.into()), + } + } } -/// Defines a conversion from a Rust type to a Python object. +impl<'py, T> IntoPyObjectExt<'py> for T where T: IntoPyObject<'py> {} + +/// Extract a type from a Python object. +/// /// -/// It functions similarly to std's [`Into`] trait, but requires a [GIL token](Python) -/// as an argument. Many functions and traits internal to PyO3 require this trait as a bound, -/// so a lack of this trait can manifest itself in different error messages. +/// Normal usage is through the `extract` methods on [`Bound`], [`Borrowed`] and [`Py`], which +/// forward to this trait. /// /// # Examples -/// ## With `#[pyclass]` -/// The easiest way to implement `IntoPy` is by exposing a struct as a native Python object -/// by annotating it with [`#[pyclass]`](crate::prelude::pyclass). /// /// ```rust /// use pyo3::prelude::*; +/// use pyo3::types::PyString; /// -/// #[pyclass] -/// struct Number { -/// #[pyo3(get, set)] -/// value: i32, -/// } +/// # fn main() -> PyResult<()> { +/// Python::attach(|py| { +/// // Calling `.extract()` on a `Bound` smart pointer +/// let obj: Bound<'_, PyString> = PyString::new(py, "blah"); +/// let s: String = obj.extract()?; +/// # assert_eq!(s, "blah"); +/// +/// // Calling `.extract(py)` on a `Py` smart pointer +/// let obj: Py = obj.unbind(); +/// let s: String = obj.extract(py)?; +/// # assert_eq!(s, "blah"); +/// # Ok(()) +/// }) +/// # } /// ``` -/// Python code will see this as an instance of the `Number` class with a `value` attribute. /// -/// ## Conversion to a Python object +/// Note: Depending on the Python version and implementation, some [`FromPyObject`] implementations +/// may produce a result that borrows into the Python type. This is described by the input lifetime +/// `'a` of `obj`. /// -/// However, it may not be desirable to expose the existence of `Number` to Python code. -/// `IntoPy` allows us to define a conversion to an appropriate Python object. -/// ```rust -/// use pyo3::prelude::*; +/// Types that must not borrow from the input can use [`FromPyObjectOwned`] as a restriction. This +/// is most often the case for collection types. See its documentation for more details. /// -/// struct Number { -/// value: i32, -/// } +/// # How to implement [`FromPyObject`]? +/// ## `#[derive(FromPyObject)]` +/// The simplest way to implement [`FromPyObject`] for a custom type is to make use of our derive +/// macro. +/// ```rust,no_run +/// # #![allow(dead_code)] +/// use pyo3::prelude::*; /// -/// impl IntoPy for Number { -/// fn into_py(self, py: Python<'_>) -> PyObject { -/// // delegates to i32's IntoPy implementation. -/// self.value.into_py(py) -/// } +/// #[derive(FromPyObject)] +/// struct MyObject { +/// msg: String, +/// list: Vec /// } +/// # fn main() {} /// ``` -/// Python code will see this as an `int` object. -/// -/// ## Dynamic conversion into Python objects. -/// It is also possible to return a different Python object depending on some condition. -/// This is useful for types like enums that can carry different types. -/// -/// ```rust +/// By default this will try to extract each field from the Python object by attribute access, but +/// this can be customized. For more information about the derive macro, its configuration as well +/// as its working principle for other types, take a look at the [guide]. +/// +/// In case the derive macro is not sufficient or can not be used for some other reason, +/// [`FromPyObject`] can be implemented manually. In the following types without lifetime parameters +/// are handled first, because they are a little bit simpler. Types with lifetime parameters are +/// explained below. +/// +/// ## Manual implementation for types without lifetime +/// Types that do not contain lifetime parameters are unable to borrow from the Python object, so +/// the lifetimes of [`FromPyObject`] can be elided: +/// ```rust,no_run +/// # #![allow(dead_code)] /// use pyo3::prelude::*; /// -/// enum Value { -/// Integer(i32), -/// String(String), -/// None, +/// struct MyObject { +/// msg: String, +/// list: Vec /// } /// -/// impl IntoPy for Value { -/// fn into_py(self, py: Python<'_>) -> PyObject { -/// match self { -/// Self::Integer(val) => val.into_py(py), -/// Self::String(val) => val.into_py(py), -/// Self::None => py.None(), -/// } +/// impl FromPyObject<'_, '_> for MyObject { +/// type Error = PyErr; +/// +/// fn extract(obj: Borrowed<'_, '_, PyAny>) -> Result { +/// Ok(MyObject { +/// msg: obj.getattr("msg")?.extract()?, +/// list: obj.getattr("list")?.extract()?, +/// }) /// } /// } -/// # fn main() { -/// # Python::with_gil(|py| { -/// # let v = Value::Integer(73).into_py(py); -/// # let v = v.extract::(py).unwrap(); -/// # -/// # let v = Value::String("foo".into()).into_py(py); -/// # let v = v.extract::(py).unwrap(); -/// # -/// # let v = Value::None.into_py(py); -/// # let v = v.extract::>>(py).unwrap(); -/// # }); -/// # } -/// ``` -/// Python code will see this as any of the `int`, `string` or `None` objects. -#[doc(alias = "IntoPyCallbackOutput")] -pub trait IntoPy: Sized { - /// Performs the conversion. - fn into_py(self, py: Python<'_>) -> T; - - /// Extracts the type hint information for this type when it appears as a return value. - /// - /// For example, `Vec` would return `List[int]`. - /// The default implementation returns `Any`, which is correct for any type. - /// - /// For most types, the return value for this method will be identical to that of [`FromPyObject::type_input`]. - /// It may be different for some types, such as `Dict`, to allow duck-typing: functions return `Dict` but take `Mapping` as argument. - #[cfg(feature = "experimental-inspect")] - fn type_output() -> TypeInfo { - TypeInfo::Any - } -} - -/// Extract a type from a Python object. -/// /// -/// Normal usage is through the `extract` methods on [`Py`] and [`PyAny`], which forward to this trait. +/// # fn main() {} +/// ``` +/// This is basically what the derive macro above expands to. /// -/// # Examples +/// ## Manual implementation for types with lifetime paramaters +/// For types that contain lifetimes, these lifetimes need to be bound to the corresponding +/// [`FromPyObject`] lifetime. This is roughly how the extraction of a typed [`Bound`] is +/// implemented within PyO3. /// -/// ```rust +/// ```rust,no_run +/// # #![allow(dead_code)] /// use pyo3::prelude::*; /// use pyo3::types::PyString; /// -/// # fn main() -> PyResult<()> { -/// Python::with_gil(|py| { -/// let obj: Py = PyString::new_bound(py, "blah").into(); +/// struct MyObject<'py>(Bound<'py, PyString>); /// -/// // Straight from an owned reference -/// let s: &str = obj.extract(py)?; -/// # assert_eq!(s, "blah"); +/// impl<'py> FromPyObject<'_, 'py> for MyObject<'py> { +/// type Error = PyErr; /// -/// // Or from a borrowed reference -/// let obj: &PyString = obj.as_ref(py); -/// let s: &str = obj.extract()?; -/// # assert_eq!(s, "blah"); -/// # Ok(()) -/// }) -/// # } +/// fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result { +/// Ok(MyObject(obj.cast()?.to_owned())) +/// } +/// } +/// +/// # fn main() {} /// ``` /// -/// Note: depending on the implementation, the lifetime of the extracted result may -/// depend on the lifetime of the `obj` or the `prepared` variable. -/// -/// For example, when extracting `&str` from a Python byte string, the resulting string slice will -/// point to the existing string data (lifetime: `'py`). -/// On the other hand, when extracting `&str` from a Python Unicode string, the preparation step -/// will convert the string to UTF-8, and the resulting string slice will have lifetime `'prepared`. -/// Since which case applies depends on the runtime type of the Python object, -/// both the `obj` and `prepared` variables must outlive the resulting string slice. -/// -/// During the migration of PyO3 from the "GIL Refs" API to the `Bound` smart pointer, this trait -/// has two methods `extract` and `extract_bound` which are defaulted to call each other. To avoid -/// infinite recursion, implementors must implement at least one of these methods. The recommendation -/// is to implement `extract_bound` and leave `extract` as the default implementation. -pub trait FromPyObject<'py>: Sized { - /// Extracts `Self` from the source GIL Ref `obj`. +/// # Details +/// [`Cow<'a, str>`] is an example of an output type that may or may not borrow from the input +/// lifetime `'a`. Which variant will be produced depends on the runtime type of the Python object. +/// For a Python byte string, the existing string data can be borrowed for `'a` into a +/// [`Cow::Borrowed`]. For a Python Unicode string, the data may have to be reencoded to UTF-8, and +/// copied into a [`Cow::Owned`]. It does _not_ depend on the Python lifetime `'py`. +/// +/// The output type may also depend on the Python lifetime `'py`. This allows the output type to +/// keep interacting with the Python interpreter. See also [`Bound<'py, T>`]. +/// +/// [`Cow<'a, str>`]: std::borrow::Cow +/// [`Cow::Borrowed`]: std::borrow::Cow::Borrowed +/// [`Cow::Owned`]: std::borrow::Cow::Owned +/// [guide]: https://pyo3.rs/latest/conversions/traits.html#deriving-frompyobject +pub trait FromPyObject<'a, 'py>: Sized { + /// The type returned in the event of a conversion error. /// - /// Implementors are encouraged to implement `extract_bound` and leave this method as the - /// default implementation, which will forward calls to `extract_bound`. - fn extract(ob: &'py PyAny) -> PyResult { - Self::extract_bound(&ob.as_borrowed()) - } + /// For most use cases defaulting to [PyErr] here is perfectly acceptable. Using a custom error + /// type can be used to avoid having to create a Python exception object in the case where that + /// exception never reaches Python. This may lead to slightly better performance under certain + /// conditions. + /// + /// # Note + /// Unfortunately `Try` and thus `?` is based on [`From`], not [`Into`], so implementations may + /// need to use `.map_err(Into::into)` sometimes to convert a generic `Error` into a [`PyErr`]. + type Error: Into; + + /// Provides the type hint information for this type when it appears as an argument. + /// + /// For example, `Vec` would be `collections.abc.Sequence[int]`. + /// The default value is `typing.Any`, which is correct for any type. + #[cfg(feature = "experimental-inspect")] + const INPUT_TYPE: TypeHint = TypeHint::module_attr("typing", "Any"); /// Extracts `Self` from the bound smart pointer `obj`. /// - /// Implementors are encouraged to implement this method and leave `extract` defaulted, as - /// this will be most compatible with PyO3's future API. - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - Self::extract(ob.clone().into_gil_ref()) - } + /// Users are advised against calling this method directly: instead, use this via + /// [`Bound<'_, PyAny>::extract`](crate::types::any::PyAnyMethods::extract) or [`Py::extract`]. + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result; /// Extracts the type hint information for this type when it appears as an argument. /// /// For example, `Vec` would return `Sequence[int]`. /// The default implementation returns `Any`, which is correct for any type. /// - /// For most types, the return value for this method will be identical to that of [`IntoPy::type_output`]. - /// It may be different for some types, such as `Dict`, to allow duck-typing: functions return `Dict` but take `Mapping` as argument. + /// For most types, the return value for this method will be identical to that of + /// [`IntoPyObject::type_output`]. It may be different for some types, such as `Dict`, + /// to allow duck-typing: functions return `Dict` but take `Mapping` as argument. #[cfg(feature = "experimental-inspect")] fn type_input() -> TypeInfo { TypeInfo::Any } -} -/// Identity conversion: allows using existing `PyObject` instances where -/// `T: ToPyObject` is expected. -impl ToPyObject for &'_ T { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - ::to_object(*self, py) - } -} + /// Specialization hook for extracting sequences for types like `Vec` and `[u8; N]`, + /// where the bytes can be directly copied from some python objects without going through + /// iteration. + #[doc(hidden)] + #[inline(always)] + fn sequence_extractor( + _obj: Borrowed<'_, 'py, PyAny>, + _: private::Token, + ) -> Option> { + struct NeverASequence(PhantomData); -/// `Option::Some` is converted like `T`. -/// `Option::None` is converted to Python `None`. -impl ToPyObject for Option -where - T: ToPyObject, -{ - fn to_object(&self, py: Python<'_>) -> PyObject { - self.as_ref() - .map_or_else(|| py.None(), |val| val.to_object(py)) - } -} + impl FromPyObjectSequence for NeverASequence { + type Target = T; -impl IntoPy for Option -where - T: IntoPy, -{ - fn into_py(self, py: Python<'_>) -> PyObject { - self.map_or_else(|| py.None(), |val| val.into_py(py)) - } -} + fn to_vec(&self) -> Vec { + unreachable!() + } -impl IntoPy for &'_ PyAny { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - unsafe { PyObject::from_borrowed_ptr(py, self.as_ptr()) } + fn to_array(&self) -> PyResult<[Self::Target; N]> { + unreachable!() + } + } + + Option::>::None } -} -impl IntoPy for &'_ T -where - T: AsRef, -{ + /// Helper used to make a specialized path in extracting `DateTime` where `Tz` is + /// `chrono::Local`, which will accept "naive" datetime objects as being in the local timezone. + #[cfg(feature = "chrono-local")] #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - unsafe { PyObject::from_borrowed_ptr(py, self.as_ref().as_ptr()) } + fn as_local_tz(_: private::Token) -> Option { + None } } -impl ToPyObject for Cell { - fn to_object(&self, py: Python<'_>) -> PyObject { - self.get().to_object(py) - } -} +mod from_py_object_sequence { + use crate::PyResult; -impl> IntoPy for Cell { - fn into_py(self, py: Python<'_>) -> PyObject { - self.get().into_py(py) - } -} + /// Private trait for implementing specialized sequence extraction for `Vec` and `[u8; N]` + #[doc(hidden)] + pub trait FromPyObjectSequence { + type Target; -impl<'py, T: FromPyObject<'py>> FromPyObject<'py> for Cell { - fn extract(ob: &'py PyAny) -> PyResult { - T::extract(ob).map(Cell::new) - } -} + fn to_vec(&self) -> Vec; -impl<'py, T> FromPyObject<'py> for &'py PyCell -where - T: PyClass, -{ - fn extract(obj: &'py PyAny) -> PyResult { - obj.downcast().map_err(Into::into) + fn to_array(&self) -> PyResult<[Self::Target; N]>; } } -impl FromPyObject<'_> for T +// Only reachable / implementable inside PyO3 itself. +pub(crate) use from_py_object_sequence::FromPyObjectSequence; + +/// A data structure that can be extracted without borrowing any data from the input. +/// +/// This is primarily useful for trait bounds. For example a [`FromPyObject`] implementation of a +/// wrapper type may be able to borrow data from the input, but a [`FromPyObject`] implementation of +/// a collection type may only extract owned data. +/// +/// For example [`PyList`] will not hand out references tied to its own lifetime, but "owned" +/// references independent of it. (Similar to [`Vec>`] where you clone the [`Arc`] out). +/// This makes it impossible to collect borrowed types in a collection, since they would not borrow +/// from the original [`PyList`], but the much shorter lived element reference. See the example +/// below. +/// +/// ```,no_run +/// # use pyo3::prelude::*; +/// # #[allow(dead_code)] +/// pub struct MyWrapper(T); +/// +/// impl<'a, 'py, T> FromPyObject<'a, 'py> for MyWrapper +/// where +/// T: FromPyObject<'a, 'py> +/// { +/// type Error = T::Error; +/// +/// fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { +/// obj.extract().map(MyWrapper) +/// } +/// } +/// +/// # #[allow(dead_code)] +/// pub struct MyVec(Vec); +/// +/// impl<'py, T> FromPyObject<'_, 'py> for MyVec +/// where +/// T: FromPyObjectOwned<'py> // 👈 can only extract owned values, because each `item` below +/// // is a temporary short lived owned reference +/// { +/// type Error = PyErr; +/// +/// fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result { +/// let mut v = MyVec(Vec::new()); +/// for item in obj.try_iter()? { +/// v.0.push(item?.extract::().map_err(Into::into)?); +/// } +/// Ok(v) +/// } +/// } +/// ``` +/// +/// [`PyList`]: crate::types::PyList +/// [`Arc`]: std::sync::Arc +pub trait FromPyObjectOwned<'py>: for<'a> FromPyObject<'a, 'py> {} +impl<'py, T> FromPyObjectOwned<'py> for T where T: for<'a> FromPyObject<'a, 'py> {} + +impl<'a, 'py, T> FromPyObject<'a, 'py> for T where - T: PyClass + Clone, + T: PyClass + Clone + ExtractPyClassWithClone, { - fn extract(obj: &PyAny) -> PyResult { - let cell: &PyCell = obj.downcast()?; - Ok(unsafe { cell.try_borrow_unguarded()?.clone() }) + type Error = PyClassGuardError<'a, 'py>; + + #[cfg(feature = "experimental-inspect")] + const INPUT_TYPE: TypeHint = ::TYPE_HINT; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + Ok(obj.extract::>()?.clone()) } } -impl<'py, T> FromPyObject<'py> for PyRef<'py, T> +impl<'a, 'py, T> FromPyObject<'a, 'py> for PyRef<'py, T> where T: PyClass, { - fn extract(obj: &'py PyAny) -> PyResult { - let cell: &PyCell = obj.downcast()?; - cell.try_borrow().map_err(Into::into) - } -} + type Error = PyClassGuardError<'a, 'py>; -impl<'py, T> FromPyObject<'py> for PyRefMut<'py, T> -where - T: PyClass, -{ - fn extract(obj: &'py PyAny) -> PyResult { - let cell: &PyCell = obj.downcast()?; - cell.try_borrow_mut().map_err(Into::into) + #[cfg(feature = "experimental-inspect")] + const INPUT_TYPE: TypeHint = ::TYPE_HINT; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + obj.cast::() + .map_err(|e| PyClassGuardError(Some(e)))? + .try_borrow() + .map_err(|_| PyClassGuardError(None)) } } -impl<'py, T> FromPyObject<'py> for Option +impl<'a, 'py, T> FromPyObject<'a, 'py> for PyRefMut<'py, T> where - T: FromPyObject<'py>, + T: PyClass, { - fn extract(obj: &'py PyAny) -> PyResult { - if obj.as_ptr() == unsafe { ffi::Py_None() } { - Ok(None) - } else { - T::extract(obj).map(Some) - } - } -} - -/// Trait implemented by Python object types that allow a checked downcast. -/// If `T` implements `PyTryFrom`, we can convert `&PyAny` to `&T`. -/// -/// This trait is similar to `std::convert::TryFrom` -#[deprecated(since = "0.21.0")] -pub trait PyTryFrom<'v>: Sized + PyNativeType { - /// Cast from a concrete Python object type to PyObject. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast::()` instead of `T::try_from(value)`" - )] - fn try_from>(value: V) -> Result<&'v Self, PyDowncastError<'v>>; - - /// Cast from a concrete Python object type to PyObject. With exact type check. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast_exact::()` instead of `T::try_from_exact(value)`" - )] - fn try_from_exact>(value: V) -> Result<&'v Self, PyDowncastError<'v>>; - - /// Cast a PyAny to a specific type of PyObject. The caller must - /// have already verified the reference is for this type. - /// - /// # Safety - /// - /// Callers must ensure that the type is valid or risk type confusion. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast_unchecked::()` instead of `T::try_from_unchecked(value)`" - )] - unsafe fn try_from_unchecked>(value: V) -> &'v Self; -} - -/// Trait implemented by Python object types that allow a checked downcast. -/// This trait is similar to `std::convert::TryInto` -#[deprecated(since = "0.21.0")] -pub trait PyTryInto: Sized { - /// Cast from PyObject to a concrete Python object type. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast()` instead of `value.try_into()`" - )] - fn try_into(&self) -> Result<&T, PyDowncastError<'_>>; - - /// Cast from PyObject to a concrete Python object type. With exact type check. - #[deprecated( - since = "0.21.0", - note = "use `value.downcast()` instead of `value.try_into_exact()`" - )] - fn try_into_exact(&self) -> Result<&T, PyDowncastError<'_>>; -} - -#[allow(deprecated)] -mod implementations { - use super::*; - - // TryFrom implies TryInto - impl PyTryInto for PyAny - where - U: for<'v> PyTryFrom<'v>, - { - fn try_into(&self) -> Result<&U, PyDowncastError<'_>> { - >::try_from(self) - } - fn try_into_exact(&self) -> Result<&U, PyDowncastError<'_>> { - U::try_from_exact(self) - } - } - - impl<'v, T> PyTryFrom<'v> for T - where - T: PyTypeInfo + PyNativeType, - { - fn try_from>(value: V) -> Result<&'v Self, PyDowncastError<'v>> { - value.into().downcast() - } + type Error = PyClassGuardMutError<'a, 'py>; - fn try_from_exact>(value: V) -> Result<&'v Self, PyDowncastError<'v>> { - value.into().downcast_exact() - } - - #[inline] - unsafe fn try_from_unchecked>(value: V) -> &'v Self { - value.into().downcast_unchecked() - } - } - - impl<'v, T> PyTryFrom<'v> for PyCell - where - T: 'v + PyClass, - { - fn try_from>(value: V) -> Result<&'v Self, PyDowncastError<'v>> { - value.into().downcast() - } - fn try_from_exact>(value: V) -> Result<&'v Self, PyDowncastError<'v>> { - let value = value.into(); - unsafe { - if T::is_exact_type_of(value) { - Ok(Self::try_from_unchecked(value)) - } else { - Err(PyDowncastError::new(value, T::NAME)) - } - } - } - #[inline] - unsafe fn try_from_unchecked>(value: V) -> &'v Self { - value.into().downcast_unchecked() - } - } -} + #[cfg(feature = "experimental-inspect")] + const INPUT_TYPE: TypeHint = ::TYPE_HINT; -/// Converts `()` to an empty Python tuple. -impl IntoPy> for () { - fn into_py(self, py: Python<'_>) -> Py { - PyTuple::empty_bound(py).unbind() + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + obj.cast::() + .map_err(|e| PyClassGuardMutError(Some(e)))? + .try_borrow_mut() + .map_err(|_| PyClassGuardMutError(None)) } } -/// Raw level conversion between `*mut ffi::PyObject` and PyO3 types. -/// -/// # Safety -/// -/// See safety notes on individual functions. -pub unsafe trait FromPyPointer<'p>: Sized { - /// Convert from an arbitrary `PyObject`. - /// - /// # Safety - /// - /// Implementations must ensure the object does not get freed during `'p` - /// and ensure that `ptr` is of the correct type. - /// Note that it must be safe to decrement the reference count of `ptr`. - unsafe fn from_owned_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self>; - /// Convert from an arbitrary `PyObject` or panic. - /// - /// # Safety - /// - /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). - unsafe fn from_owned_ptr_or_panic(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { - Self::from_owned_ptr_or_opt(py, ptr).unwrap_or_else(|| err::panic_after_error(py)) - } - /// Convert from an arbitrary `PyObject` or panic. - /// - /// # Safety - /// - /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). - unsafe fn from_owned_ptr(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { - Self::from_owned_ptr_or_panic(py, ptr) - } - /// Convert from an arbitrary `PyObject`. - /// - /// # Safety - /// - /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). - unsafe fn from_owned_ptr_or_err(py: Python<'p>, ptr: *mut ffi::PyObject) -> PyResult<&'p Self> { - Self::from_owned_ptr_or_opt(py, ptr).ok_or_else(|| err::PyErr::fetch(py)) - } - /// Convert from an arbitrary borrowed `PyObject`. - /// - /// # Safety - /// - /// Implementations must ensure the object does not get freed during `'p` and avoid type confusion. - unsafe fn from_borrowed_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) - -> Option<&'p Self>; - /// Convert from an arbitrary borrowed `PyObject`. - /// - /// # Safety - /// - /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). - unsafe fn from_borrowed_ptr_or_panic(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { - Self::from_borrowed_ptr_or_opt(py, ptr).unwrap_or_else(|| err::panic_after_error(py)) - } - /// Convert from an arbitrary borrowed `PyObject`. - /// - /// # Safety - /// - /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). - unsafe fn from_borrowed_ptr(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { - Self::from_borrowed_ptr_or_panic(py, ptr) - } - /// Convert from an arbitrary borrowed `PyObject`. - /// - /// # Safety - /// - /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). - unsafe fn from_borrowed_ptr_or_err( - py: Python<'p>, - ptr: *mut ffi::PyObject, - ) -> PyResult<&'p Self> { - Self::from_borrowed_ptr_or_opt(py, ptr).ok_or_else(|| err::PyErr::fetch(py)) - } -} +impl<'py> IntoPyObject<'py> for () { + type Target = PyTuple; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; -unsafe impl<'p, T> FromPyPointer<'p> for T -where - T: 'p + crate::PyNativeType, -{ - unsafe fn from_owned_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self> { - gil::register_owned(py, NonNull::new(ptr)?); - Some(&*(ptr as *mut Self)) - } - unsafe fn from_borrowed_ptr_or_opt( - _py: Python<'p>, - ptr: *mut ffi::PyObject, - ) -> Option<&'p Self> { - NonNull::new(ptr as *mut Self).map(|p| &*p.as_ptr()) + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(PyTuple::empty(py)) } } @@ -584,7 +583,7 @@ where /// /// let t = TestClass { num: 10 }; /// -/// Python::with_gil(|py| { +/// Python::attach(|py| { /// let pyvalue = Py::new(py, t).unwrap().to_object(py); /// let t: TestClass = pyvalue.extract(py).unwrap(); /// }) @@ -593,67 +592,56 @@ mod test_no_clone {} #[cfg(test)] mod tests { - use crate::{PyObject, Python}; - - #[allow(deprecated)] - mod deprecated { - use super::super::PyTryFrom; - use crate::types::{IntoPyDict, PyAny, PyDict, PyList}; - use crate::{Python, ToPyObject}; - - #[test] - fn test_try_from() { - Python::with_gil(|py| { - let list: &PyAny = vec![3, 6, 5, 4, 7].to_object(py).into_ref(py); - let dict: &PyAny = vec![("reverse", true)].into_py_dict(py).as_ref(); - - assert!(>::try_from(list).is_ok()); - assert!(>::try_from(dict).is_ok()); - - assert!(>::try_from(list).is_ok()); - assert!(>::try_from(dict).is_ok()); - }); - } + #[test] + #[cfg(feature = "macros")] + fn test_pyclass_skip_from_py_object() { + use crate::{types::PyAnyMethods, FromPyObject, IntoPyObject, PyErr, Python}; - #[test] - fn test_try_from_exact() { - Python::with_gil(|py| { - let list: &PyAny = vec![3, 6, 5, 4, 7].to_object(py).into_ref(py); - let dict: &PyAny = vec![("reverse", true)].into_py_dict(py).as_ref(); + #[crate::pyclass(crate = "crate", skip_from_py_object)] + #[derive(Clone)] + struct Foo(i32); - assert!(PyList::try_from_exact(list).is_ok()); - assert!(PyDict::try_from_exact(dict).is_ok()); + impl<'py> FromPyObject<'_, 'py> for Foo { + type Error = PyErr; - assert!(PyAny::try_from_exact(list).is_err()); - assert!(PyAny::try_from_exact(dict).is_err()); - }); + fn extract(obj: crate::Borrowed<'_, 'py, crate::PyAny>) -> Result { + if let Ok(obj) = obj.cast::() { + Ok(obj.borrow().clone()) + } else { + obj.extract::().map(Self) + } + } } + Python::attach(|py| { + let foo1 = 42i32.into_pyobject(py)?; + assert_eq!(foo1.extract::()?.0, 42); - #[test] - fn test_try_from_unchecked() { - Python::with_gil(|py| { - let list = PyList::new(py, [1, 2, 3]); - let val = unsafe { ::try_from_unchecked(list.as_ref()) }; - assert!(list.is(val)); - }); - } + let foo2 = Foo(0).into_pyobject(py)?; + assert_eq!(foo2.extract::()?.0, 0); + + Ok::<_, PyErr>(()) + }) + .unwrap(); } #[test] - fn test_option_as_ptr() { - Python::with_gil(|py| { - use crate::AsPyPointer; - let mut option: Option = None; - assert_eq!(option.as_ptr(), std::ptr::null_mut()); + #[cfg(feature = "macros")] + fn test_pyclass_from_py_object() { + use crate::{types::PyAnyMethods, IntoPyObject, PyErr, Python}; + + #[crate::pyclass(crate = "crate", from_py_object)] + #[derive(Clone)] + struct Foo(i32); - let none = py.None(); - option = Some(none.clone()); + Python::attach(|py| { + let foo1 = 42i32.into_pyobject(py)?; + assert!(foo1.extract::().is_err()); - let ref_cnt = none.get_refcnt(py); - assert_eq!(option.as_ptr(), none.as_ptr()); + let foo2 = Foo(0).into_pyobject(py)?; + assert_eq!(foo2.extract::()?.0, 0); - // Ensure ref count not changed by as_ptr call - assert_eq!(none.get_refcnt(py), ref_cnt); - }); + Ok::<_, PyErr>(()) + }) + .unwrap(); } } diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index 453799c6e8b..af6bf515fee 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -1,9 +1,6 @@ #![cfg(feature = "anyhow")] -//! A conversion from -//! [anyhow](https://docs.rs/anyhow/ "A trait object based error system for easy idiomatic error handling in Rust applications.")’s -//! [`Error`](https://docs.rs/anyhow/latest/anyhow/struct.Error.html "Anyhows `Error` type, a wrapper around a dynamic error type") -//! type to [`PyErr`]. +//! A conversion from [anyhow]’s [`Error`][anyhow-error] type to [`PyErr`]. //! //! Use of an error handling library like [anyhow] is common in application code and when you just //! want error handling to be easy. If you are writing a library or you need more control over your @@ -35,7 +32,6 @@ //! //! ```rust //! use pyo3::prelude::*; -//! use pyo3::wrap_pyfunction; //! use std::path::PathBuf; //! //! // A wrapper around a Rust function. @@ -47,7 +43,7 @@ //! } //! //! fn main() { -//! let error = Python::with_gil(|py| -> PyResult> { +//! let error = Python::attach(|py| -> PyResult> { //! let fun = wrap_pyfunction!(py_open, py)?; //! let text = fun.call1(("foo.txt",))?.extract::>()?; //! Ok(text) @@ -74,10 +70,10 @@ //! // An arbitrary example of a Python api you //! // could call inside an application... //! // This might return a `PyErr`. -//! let res = Python::with_gil(|py| { +//! let res = Python::attach(|py| { //! let zlib = PyModule::import(py, "zlib")?; //! let decompress = zlib.getattr("decompress")?; -//! let bytes = PyBytes::new_bound(py, bytes); +//! let bytes = PyBytes::new(py, bytes); //! let value = decompress.call1((bytes,))?; //! value.extract::>() //! })?; @@ -100,6 +96,8 @@ //! } //! ``` //! +//! [anyhow]: https://docs.rs/anyhow/ "A trait object based error system for easy idiomatic error handling in Rust applications." +//! [anyhow-error]: https://docs.rs/anyhow/latest/anyhow/struct.Error.html "Anyhows `Error` type, a wrapper around a dynamic error type" //! [`RuntimeError`]: https://docs.python.org/3/library/exceptions.html#RuntimeError "Built-in Exceptions — Python documentation" //! [Error handling]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html "Recoverable Errors with Result - The Rust Programming Language" @@ -115,7 +113,7 @@ impl From for PyErr { Err(error) => error, }; } - PyRuntimeError::new_err(format!("{:?}", error)) + PyRuntimeError::new_err(format!("{error:?}")) } } @@ -143,12 +141,12 @@ mod test_anyhow { #[test] fn test_pyo3_exception_contents() { let err = h().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); - Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict_bound(py); - let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); + Python::attach(|py| { + let locals = [("err", pyerr)].into_py_dict(py).unwrap(); + let pyerr = py.run(c"raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } @@ -160,12 +158,12 @@ mod test_anyhow { #[test] fn test_pyo3_exception_contents2() { let err = k().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); - Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict_bound(py); - let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); + Python::attach(|py| { + let locals = [("err", pyerr)].into_py_dict(py).unwrap(); + let pyerr = py.run(c"raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } @@ -175,7 +173,7 @@ mod test_anyhow { let origin_exc = PyValueError::new_err("Value Error"); let err: anyhow::Error = origin_exc.into(); let converted: PyErr = err.into(); - assert!(Python::with_gil( + assert!(Python::attach( |py| converted.is_instance_of::(py) )) } @@ -185,7 +183,7 @@ mod test_anyhow { let mut err: anyhow::Error = origin_exc.into(); err = err.context("Context"); let converted: PyErr = err.into(); - assert!(Python::with_gil( + assert!(Python::attach( |py| converted.is_instance_of::(py) )) } diff --git a/src/conversions/bigdecimal.rs b/src/conversions/bigdecimal.rs new file mode 100644 index 00000000000..70783d0b2b1 --- /dev/null +++ b/src/conversions/bigdecimal.rs @@ -0,0 +1,254 @@ +#![cfg(feature = "bigdecimal")] +//! Conversions to and from [bigdecimal](https://docs.rs/bigdecimal)'s [`BigDecimal`] type. +//! +//! This is useful for converting Python's decimal.Decimal into and from a native Rust type. +//! +//! # Setup +//! +//! To use this feature, add to your **`Cargo.toml`**: +//! +//! ```toml +//! [dependencies] +#![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"bigdecimal\"] }")] +//! bigdecimal = "0.4" +//! ``` +//! +//! Note that you must use a compatible version of bigdecimal and PyO3. +//! The required bigdecimal version may vary based on the version of PyO3. +//! +//! # Example +//! +//! Rust code to create a function that adds one to a BigDecimal +//! +//! ```rust +//! use bigdecimal::BigDecimal; +//! use pyo3::prelude::*; +//! +//! #[pyfunction] +//! fn add_one(d: BigDecimal) -> BigDecimal { +//! d + 1 +//! } +//! +//! #[pymodule] +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { +//! m.add_function(wrap_pyfunction!(add_one, m)?)?; +//! Ok(()) +//! } +//! ``` +//! +//! Python code that validates the functionality +//! +//! +//! ```python +//! from my_module import add_one +//! from decimal import Decimal +//! +//! d = Decimal("2") +//! value = add_one(d) +//! +//! assert d + 1 == value +//! ``` + +use std::str::FromStr; + +use crate::types::PyTuple; +use crate::{ + exceptions::PyValueError, + sync::PyOnceLock, + types::{PyAnyMethods, PyStringMethods, PyType}, + Borrowed, Bound, FromPyObject, IntoPyObject, Py, PyAny, PyErr, PyResult, Python, +}; +use bigdecimal::BigDecimal; +use num_bigint::Sign; + +fn get_decimal_cls(py: Python<'_>) -> PyResult<&Bound<'_, PyType>> { + static DECIMAL_CLS: PyOnceLock> = PyOnceLock::new(); + DECIMAL_CLS.import(py, "decimal", "Decimal") +} + +fn get_invalid_operation_error_cls(py: Python<'_>) -> PyResult<&Bound<'_, PyType>> { + static INVALID_OPERATION_CLS: PyOnceLock> = PyOnceLock::new(); + INVALID_OPERATION_CLS.import(py, "decimal", "InvalidOperation") +} + +impl FromPyObject<'_, '_> for BigDecimal { + type Error = PyErr; + + fn extract(obj: Borrowed<'_, '_, PyAny>) -> PyResult { + let py_str = &obj.str()?; + let rs_str = &py_str.to_cow()?; + BigDecimal::from_str(rs_str).map_err(|e| PyValueError::new_err(e.to_string())) + } +} + +impl<'py> IntoPyObject<'py> for BigDecimal { + type Target = PyAny; + + type Output = Bound<'py, Self::Target>; + + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let cls = get_decimal_cls(py)?; + let (bigint, scale) = self.into_bigint_and_scale(); + if scale == 0 { + return cls.call1((bigint,)); + } + let exponent = scale.checked_neg().ok_or_else(|| { + get_invalid_operation_error_cls(py) + .map_or_else(|err| err, |cls| PyErr::from_type(cls.clone(), ())) + })?; + let (sign, digits) = bigint.to_radix_be(10); + let signed = matches!(sign, Sign::Minus).into_pyobject(py)?; + let digits = PyTuple::new(py, digits)?; + + cls.call1(((signed, digits, exponent),)) + } +} + +#[cfg(test)] +mod test_bigdecimal { + use super::*; + use crate::types::dict::PyDictMethods; + use crate::types::PyDict; + use std::ffi::CString; + + use bigdecimal::{One, Zero}; + #[cfg(not(target_arch = "wasm32"))] + use proptest::prelude::*; + + macro_rules! convert_constants { + ($name:ident, $rs:expr, $py:literal) => { + #[test] + fn $name() { + Python::attach(|py| { + let rs_orig = $rs; + let rs_dec = rs_orig.clone().into_pyobject(py).unwrap(); + let locals = PyDict::new(py); + locals.set_item("rs_dec", &rs_dec).unwrap(); + // Checks if BigDecimal -> Python Decimal conversion is correct + py.run( + &CString::new(format!( + "import decimal\npy_dec = decimal.Decimal(\"{}\")\nassert py_dec == rs_dec", + $py + )) + .unwrap(), + None, + Some(&locals), + ) + .unwrap(); + // Checks if Python Decimal -> BigDecimal conversion is correct + let py_dec = locals.get_item("py_dec").unwrap().unwrap(); + let py_result: BigDecimal = py_dec.extract().unwrap(); + assert_eq!(rs_orig, py_result); + }) + } + }; + } + + convert_constants!(convert_zero, BigDecimal::zero(), "0"); + convert_constants!(convert_one, BigDecimal::one(), "1"); + convert_constants!(convert_neg_one, -BigDecimal::one(), "-1"); + convert_constants!(convert_two, BigDecimal::from(2), "2"); + convert_constants!(convert_ten, BigDecimal::from_str("10").unwrap(), "10"); + convert_constants!( + convert_one_hundred_point_one, + BigDecimal::from_str("100.1").unwrap(), + "100.1" + ); + convert_constants!( + convert_one_thousand, + BigDecimal::from_str("1000").unwrap(), + "1000" + ); + convert_constants!( + convert_scientific, + BigDecimal::from_str("1e10").unwrap(), + "1e10" + ); + + #[cfg(not(target_arch = "wasm32"))] + proptest! { + #[test] + fn test_roundtrip( + number in 0..28u32 + ) { + let num = BigDecimal::from(number); + Python::attach(|py| { + let rs_dec = num.clone().into_pyobject(py).unwrap(); + let locals = PyDict::new(py); + locals.set_item("rs_dec", &rs_dec).unwrap(); + py.run( + &CString::new(format!( + "import decimal\npy_dec = decimal.Decimal(\"{num}\")\nassert py_dec == rs_dec")).unwrap(), + None, Some(&locals)).unwrap(); + let roundtripped: BigDecimal = rs_dec.extract().unwrap(); + assert_eq!(num, roundtripped); + }) + } + + #[test] + fn test_integers(num in any::()) { + Python::attach(|py| { + let py_num = num.into_pyobject(py).unwrap(); + let roundtripped: BigDecimal = py_num.extract().unwrap(); + let rs_dec = BigDecimal::from(num); + assert_eq!(rs_dec, roundtripped); + }) + } + } + + #[test] + fn test_nan() { + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c"import decimal\npy_dec = decimal.Decimal(\"NaN\")", + None, + Some(&locals), + ) + .unwrap(); + let py_dec = locals.get_item("py_dec").unwrap().unwrap(); + let roundtripped: Result = py_dec.extract(); + assert!(roundtripped.is_err()); + }) + } + + #[test] + fn test_infinity() { + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c"import decimal\npy_dec = decimal.Decimal(\"Infinity\")", + None, + Some(&locals), + ) + .unwrap(); + let py_dec = locals.get_item("py_dec").unwrap().unwrap(); + let roundtripped: Result = py_dec.extract(); + assert!(roundtripped.is_err()); + }) + } + + #[test] + fn test_no_precision_loss() { + Python::attach(|py| { + let src = "1e4"; + let expected = get_decimal_cls(py) + .unwrap() + .call1((src,)) + .unwrap() + .call_method0("as_tuple") + .unwrap(); + let actual = src + .parse::() + .unwrap() + .into_pyobject(py) + .unwrap() + .call_method0("as_tuple") + .unwrap(); + + assert!(actual.eq(expected).unwrap()); + }); + } +} diff --git a/src/conversions/bytes.rs b/src/conversions/bytes.rs new file mode 100644 index 00000000000..fb656579a2d --- /dev/null +++ b/src/conversions/bytes.rs @@ -0,0 +1,132 @@ +#![cfg(feature = "bytes")] + +//! Conversions to and from [bytes](https://docs.rs/bytes/latest/bytes/)'s [`Bytes`]. +//! +//! This is useful for efficiently converting Python's `bytes` types efficiently. +//! While `bytes` will be directly borrowed, converting from `bytearray` will result in a copy. +//! +//! When converting `Bytes` back into Python, this will do a copy, just like `&[u8]` and `Vec`. +//! +//! # When to use `Bytes` +//! +//! Unless you specifically need [`Bytes`] for ref-counted ownership and sharing, +//! you may find that using `&[u8]`, `Vec`, [`Bound`], or [`PyBackedBytes`] +//! is simpler for most use cases. +//! +//! # Setup +//! +//! To use this feature, add in your **`Cargo.toml`**: +//! +//! ```toml +//! [dependencies] +//! bytes = "1.10" +#![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"bytes\"] }")] +//! ``` +//! +//! Note that you must use compatible versions of bytes and PyO3. +//! +//! # Example +//! +//! Rust code to create functions which return `Bytes` or take `Bytes` as arguments: +//! +//! ```rust,no_run +//! use pyo3::prelude::*; +//! use bytes::Bytes; +//! +//! #[pyfunction] +//! fn get_message_bytes() -> Bytes { +//! Bytes::from_static(b"Hello Python!") +//! } +//! +//! #[pyfunction] +//! fn num_bytes(bytes: Bytes) -> usize { +//! bytes.len() +//! } +//! +//! #[pymodule] +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { +//! m.add_function(wrap_pyfunction!(get_message_bytes, m)?)?; +//! m.add_function(wrap_pyfunction!(num_bytes, m)?)?; +//! Ok(()) +//! } +//! ``` +//! +//! Python code that calls these functions: +//! +//! ```python +//! from my_module import get_message_bytes, num_bytes +//! +//! message = get_message_bytes() +//! assert message == b"Hello Python!" +//! +//! size = num_bytes(message) +//! assert size == 13 +//! ``` +use bytes::Bytes; + +use crate::conversion::IntoPyObject; +use crate::instance::Bound; +use crate::pybacked::PyBackedBytes; +use crate::types::PyBytes; +use crate::{Borrowed, CastError, FromPyObject, PyAny, PyErr, Python}; + +impl<'a, 'py> FromPyObject<'a, 'py> for Bytes { + type Error = CastError<'a, 'py>; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + Ok(Bytes::from_owner(obj.extract::()?)) + } +} + +impl<'py> IntoPyObject<'py> for Bytes { + type Target = PyBytes; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(PyBytes::new(py, &self)) + } +} + +impl<'py> IntoPyObject<'py> for &Bytes { + type Target = PyBytes; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(PyBytes::new(py, self)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::{PyAnyMethods, PyByteArray, PyByteArrayMethods, PyBytes}; + use crate::Python; + + #[test] + fn test_bytes() { + Python::attach(|py| { + let py_bytes = PyBytes::new(py, b"foobar"); + let bytes: Bytes = py_bytes.extract().unwrap(); + assert_eq!(&*bytes, b"foobar"); + + let bytes = Bytes::from_static(b"foobar").into_pyobject(py).unwrap(); + assert!(bytes.is_instance_of::()); + }); + } + + #[test] + fn test_bytearray() { + Python::attach(|py| { + let py_bytearray = PyByteArray::new(py, b"foobar"); + let bytes: Bytes = py_bytearray.extract().unwrap(); + assert_eq!(&*bytes, b"foobar"); + + // Editing the bytearray should not change extracted Bytes + unsafe { py_bytearray.as_bytes_mut()[0] = b'x' }; + assert_eq!(&bytes, "foobar"); + assert_eq!(&py_bytearray.extract::>().unwrap(), b"xoobar"); + }); + } +} diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index d67818e645c..7d67f504ad5 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -20,52 +20,61 @@ //! //! ```rust //! use chrono::{DateTime, Duration, TimeZone, Utc}; -//! use pyo3::{Python, ToPyObject}; +//! use pyo3::{Python, PyResult, IntoPyObject, types::PyAnyMethods}; //! -//! fn main() { -//! pyo3::prepare_freethreaded_python(); -//! Python::with_gil(|py| { +//! fn main() -> PyResult<()> { +//! Python::initialize(); +//! Python::attach(|py| { //! // Build some chrono values //! let chrono_datetime = Utc.with_ymd_and_hms(2022, 1, 1, 12, 0, 0).unwrap(); //! let chrono_duration = Duration::seconds(1); //! // Convert them to Python -//! let py_datetime = chrono_datetime.to_object(py); -//! let py_timedelta = chrono_duration.to_object(py); +//! let py_datetime = chrono_datetime.into_pyobject(py)?; +//! let py_timedelta = chrono_duration.into_pyobject(py)?; //! // Do an operation in Python -//! let py_sum = py_datetime.call_method1(py, "__add__", (py_timedelta,)).unwrap(); +//! let py_sum = py_datetime.call_method1("__add__", (py_timedelta,))?; //! // Convert back to Rust -//! let chrono_sum: DateTime = py_sum.extract(py).unwrap(); +//! let chrono_sum: DateTime = py_sum.extract()?; //! println!("DateTime: {}", chrono_datetime); -//! }); +//! Ok(()) +//! }) //! } //! ``` + +use crate::conversion::{FromPyObjectOwned, IntoPyObject}; use crate::exceptions::{PyTypeError, PyUserWarning, PyValueError}; -#[cfg(Py_LIMITED_API)] -use crate::sync::GILOnceCell; +use crate::intern; use crate::types::any::PyAnyMethods; +use crate::types::PyNone; +use crate::types::{PyDate, PyDateTime, PyDelta, PyTime, PyTzInfo, PyTzInfoAccess}; #[cfg(not(Py_LIMITED_API))] -use crate::types::datetime::timezone_from_offset; -#[cfg(not(Py_LIMITED_API))] -use crate::types::{ - timezone_utc_bound, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, - PyTimeAccess, PyTzInfo, PyTzInfoAccess, -}; -#[cfg(Py_LIMITED_API)] -use crate::{intern, DowncastError}; +use crate::types::{PyDateAccess, PyDeltaAccess, PyTimeAccess}; +#[cfg(feature = "chrono-local")] use crate::{ - Bound, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, ToPyObject, + exceptions::PyRuntimeError, + sync::PyOnceLock, + types::{PyString, PyStringMethods}, + Py, }; +use crate::{Borrowed, Bound, FromPyObject, IntoPyObjectExt, PyAny, PyErr, PyResult, Python}; use chrono::offset::{FixedOffset, Utc}; +#[cfg(feature = "chrono-local")] +use chrono::Local; use chrono::{ - DateTime, Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, + DateTime, Datelike, Duration, LocalResult, NaiveDate, NaiveDateTime, NaiveTime, Offset, + TimeZone, Timelike, }; -impl ToPyObject for Duration { - fn to_object(&self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for Duration { + type Target = PyDelta; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { // Total number of days let days = self.num_days(); // Remainder of seconds - let secs_dur = *self - Duration::days(days); + let secs_dur = self - Duration::days(days); let secs = secs_dur.num_seconds(); // Fractional part of the microseconds let micros = (secs_dur - Duration::seconds(secs_dur.num_seconds())) @@ -73,49 +82,43 @@ impl ToPyObject for Duration { // This should never panic since we are just getting the fractional // part of the total microseconds, which should never overflow. .unwrap(); - - #[cfg(not(Py_LIMITED_API))] - { - // We do not need to check the days i64 to i32 cast from rust because - // python will panic with OverflowError. - // We pass true as the `normalize` parameter since we'd need to do several checks here to - // avoid that, and it shouldn't have a big performance impact. - // The seconds and microseconds cast should never overflow since it's at most the number of seconds per day - PyDelta::new_bound( - py, - days.try_into().unwrap_or(i32::MAX), - secs.try_into().unwrap(), - micros.try_into().unwrap(), - true, - ) - .expect("failed to construct delta") - .into() - } - #[cfg(Py_LIMITED_API)] - { - DatetimeTypes::get(py) - .timedelta - .call1(py, (days, secs, micros)) - .expect("failed to construct datetime.timedelta") - } + // We do not need to check the days i64 to i32 cast from rust because + // python will panic with OverflowError. + // We pass true as the `normalize` parameter since we'd need to do several checks here to + // avoid that, and it shouldn't have a big performance impact. + // The seconds and microseconds cast should never overflow since it's at most the number of seconds per day + PyDelta::new( + py, + days.try_into().unwrap_or(i32::MAX), + secs.try_into()?, + micros.try_into()?, + true, + ) } } -impl IntoPy for Duration { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &Duration { + type Target = PyDelta; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } -impl FromPyObject<'_> for Duration { - fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { +impl FromPyObject<'_, '_> for Duration { + type Error = PyErr; + + fn extract(ob: Borrowed<'_, '_, PyAny>) -> Result { + let delta = ob.cast::()?; // Python size are much lower than rust size so we do not need bound checks. // 0 <= microseconds < 1000000 // 0 <= seconds < 3600*24 // -999999999 <= days <= 999999999 #[cfg(not(Py_LIMITED_API))] let (days, seconds, microseconds) = { - let delta = ob.downcast::()?; ( delta.get_days().into(), delta.get_seconds().into(), @@ -124,11 +127,11 @@ impl FromPyObject<'_> for Duration { }; #[cfg(Py_LIMITED_API)] let (days, seconds, microseconds) = { - check_type(ob, &DatetimeTypes::get(ob.py()).timedelta, "PyDelta")?; + let py = delta.py(); ( - ob.getattr(intern!(ob.py(), "days"))?.extract()?, - ob.getattr(intern!(ob.py(), "seconds"))?.extract()?, - ob.getattr(intern!(ob.py(), "microseconds"))?.extract()?, + delta.getattr(intern!(py, "days"))?.extract()?, + delta.getattr(intern!(py, "seconds"))?.extract()?, + delta.getattr(intern!(py, "microseconds"))?.extract()?, ) }; Ok( @@ -139,118 +142,127 @@ impl FromPyObject<'_> for Duration { } } -impl ToPyObject for NaiveDate { - fn to_object(&self, py: Python<'_>) -> PyObject { - let DateArgs { year, month, day } = self.into(); - #[cfg(not(Py_LIMITED_API))] - { - PyDate::new_bound(py, year, month, day) - .expect("failed to construct date") - .into() - } - #[cfg(Py_LIMITED_API)] - { - DatetimeTypes::get(py) - .date - .call1(py, (year, month, day)) - .expect("failed to construct datetime.date") - } +impl<'py> IntoPyObject<'py> for NaiveDate { + type Target = PyDate; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let DateArgs { year, month, day } = (&self).into(); + PyDate::new(py, year, month, day) } } -impl IntoPy for NaiveDate { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &NaiveDate { + type Target = PyDate; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } -impl FromPyObject<'_> for NaiveDate { - fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - #[cfg(not(Py_LIMITED_API))] - { - let date = ob.downcast::()?; - py_date_to_naive_date(date) - } - #[cfg(Py_LIMITED_API)] - { - check_type(ob, &DatetimeTypes::get(ob.py()).date, "PyDate")?; - py_date_to_naive_date(ob) - } +impl FromPyObject<'_, '_> for NaiveDate { + type Error = PyErr; + + fn extract(ob: Borrowed<'_, '_, PyAny>) -> Result { + let date = &*ob.cast::()?; + py_date_to_naive_date(date) } } -impl ToPyObject for NaiveTime { - fn to_object(&self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for NaiveTime { + type Target = PyTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let TimeArgs { hour, min, sec, micro, truncated_leap_second, - } = self.into(); - #[cfg(not(Py_LIMITED_API))] - let time = - PyTime::new_bound(py, hour, min, sec, micro, None).expect("Failed to construct time"); - #[cfg(Py_LIMITED_API)] - let time = DatetimeTypes::get(py) - .time - .bind(py) - .call1((hour, min, sec, micro)) - .expect("failed to construct datetime.time"); + } = (&self).into(); + + let time = PyTime::new(py, hour, min, sec, micro, None)?; + if truncated_leap_second { warn_truncated_leap_second(&time); } - time.into() + + Ok(time) } } -impl IntoPy for NaiveTime { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &NaiveTime { + type Target = PyTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } -impl FromPyObject<'_> for NaiveTime { - fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - #[cfg(not(Py_LIMITED_API))] - { - let time = ob.downcast::()?; - py_time_to_naive_time(time) - } - #[cfg(Py_LIMITED_API)] - { - check_type(ob, &DatetimeTypes::get(ob.py()).time, "PyTime")?; - py_time_to_naive_time(ob) - } +impl FromPyObject<'_, '_> for NaiveTime { + type Error = PyErr; + + fn extract(ob: Borrowed<'_, '_, PyAny>) -> Result { + let time = &*ob.cast::()?; + py_time_to_naive_time(time) } } -impl ToPyObject for NaiveDateTime { - fn to_object(&self, py: Python<'_>) -> PyObject { - naive_datetime_to_py_datetime(py, self, None) +impl<'py> IntoPyObject<'py> for NaiveDateTime { + type Target = PyDateTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let DateArgs { year, month, day } = (&self.date()).into(); + let TimeArgs { + hour, + min, + sec, + micro, + truncated_leap_second, + } = (&self.time()).into(); + + let datetime = PyDateTime::new(py, year, month, day, hour, min, sec, micro, None)?; + + if truncated_leap_second { + warn_truncated_leap_second(&datetime); + } + + Ok(datetime) } } -impl IntoPy for NaiveDateTime { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &NaiveDateTime { + type Target = PyDateTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } -impl FromPyObject<'_> for NaiveDateTime { - fn extract_bound(dt: &Bound<'_, PyAny>) -> PyResult { - #[cfg(not(Py_LIMITED_API))] - let dt = dt.downcast::()?; - #[cfg(Py_LIMITED_API)] - check_type(dt, &DatetimeTypes::get(dt.py()).datetime, "PyDateTime")?; +impl FromPyObject<'_, '_> for NaiveDateTime { + type Error = PyErr; + + fn extract(dt: Borrowed<'_, '_, PyAny>) -> Result { + let dt = &*dt.cast::()?; // If the user tries to convert a timezone aware datetime into a naive one, // we return a hard error. We could silently remove tzinfo, or assume local timezone // and do a conversion, but better leave this decision to the user of the library. - #[cfg(not(Py_LIMITED_API))] - let has_tzinfo = dt.get_tzinfo_bound().is_some(); - #[cfg(Py_LIMITED_API)] - let has_tzinfo = !dt.getattr(intern!(dt.py(), "tzinfo"))?.is_none(); + let has_tzinfo = dt.get_tzinfo().is_some(); if has_tzinfo { return Err(PyTypeError::new_err("expected a datetime without tzinfo")); } @@ -260,101 +272,137 @@ impl FromPyObject<'_> for NaiveDateTime { } } -impl ToPyObject for DateTime { - fn to_object(&self, py: Python<'_>) -> PyObject { - // FIXME: convert to better timezone representation here than just convert to fixed offset - // See https://github.com/PyO3/pyo3/issues/3266 - let tz = self.offset().fix().to_object(py); - let tz = tz.bind(py).downcast().unwrap(); - naive_datetime_to_py_datetime(py, &self.naive_local(), Some(tz)) +impl<'py, Tz: TimeZone> IntoPyObject<'py> for DateTime +where + Tz: IntoPyObject<'py>, +{ + type Target = PyDateTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (&self).into_pyobject(py) } } -impl IntoPy for DateTime { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py, Tz: TimeZone> IntoPyObject<'py> for &DateTime +where + Tz: IntoPyObject<'py>, +{ + type Target = PyDateTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let tz = self.timezone().into_bound_py_any(py)?.cast_into()?; + + let DateArgs { year, month, day } = (&self.naive_local().date()).into(); + let TimeArgs { + hour, + min, + sec, + micro, + truncated_leap_second, + } = (&self.naive_local().time()).into(); + + let fold = matches!( + self.timezone().offset_from_local_datetime(&self.naive_local()), + LocalResult::Ambiguous(_, latest) if self.offset().fix() == latest.fix() + ); + + let datetime = PyDateTime::new_with_fold( + py, + year, + month, + day, + hour, + min, + sec, + micro, + Some(&tz), + fold, + )?; + + if truncated_leap_second { + warn_truncated_leap_second(&datetime); + } + + Ok(datetime) } } -impl FromPyObject<'py>> FromPyObject<'_> for DateTime { - fn extract_bound(dt: &Bound<'_, PyAny>) -> PyResult> { - #[cfg(not(Py_LIMITED_API))] - let dt = dt.downcast::()?; - #[cfg(Py_LIMITED_API)] - check_type(dt, &DatetimeTypes::get(dt.py()).datetime, "PyDateTime")?; +impl<'py, Tz> FromPyObject<'_, 'py> for DateTime +where + Tz: TimeZone + FromPyObjectOwned<'py>, +{ + type Error = PyErr; - #[cfg(not(Py_LIMITED_API))] - let tzinfo = dt.get_tzinfo_bound(); - #[cfg(Py_LIMITED_API)] - let tzinfo: Option<&PyAny> = dt.getattr(intern!(dt.py(), "tzinfo"))?.extract()?; + fn extract(dt: Borrowed<'_, 'py, PyAny>) -> Result { + let dt = &*dt.cast::()?; + let tzinfo = dt.get_tzinfo(); let tz = if let Some(tzinfo) = tzinfo { - tzinfo.extract()? + tzinfo.extract().map_err(Into::into)? } else { + // Special case: allow naive `datetime` objects for `DateTime`, interpreting them as local time. + #[cfg(feature = "chrono-local")] + if let Some(tz) = Tz::as_local_tz(crate::conversion::private::Token) { + return py_datetime_to_datetime_with_timezone(dt, tz); + } + return Err(PyTypeError::new_err( "expected a datetime with non-None tzinfo", )); }; - let naive_dt = NaiveDateTime::new(py_date_to_naive_date(dt)?, py_time_to_naive_time(dt)?); - naive_dt.and_local_timezone(tz).single().ok_or_else(|| { - PyValueError::new_err(format!( - "The datetime {:?} contains an incompatible or ambiguous timezone", - dt - )) - }) + + py_datetime_to_datetime_with_timezone(dt, tz) } } -impl ToPyObject for FixedOffset { - fn to_object(&self, py: Python<'_>) -> PyObject { - let seconds_offset = self.local_minus_utc(); +impl<'py> IntoPyObject<'py> for FixedOffset { + type Target = PyTzInfo; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; - #[cfg(not(Py_LIMITED_API))] - { - let td = PyDelta::new_bound(py, 0, seconds_offset, 0, true) - .expect("failed to construct timedelta"); - timezone_from_offset(&td) - .expect("Failed to construct PyTimezone") - .into() - } - #[cfg(Py_LIMITED_API)] - { - let td = Duration::seconds(seconds_offset.into()).into_py(py); - DatetimeTypes::get(py) - .timezone - .call1(py, (td,)) - .expect("failed to construct datetime.timezone") - } + fn into_pyobject(self, py: Python<'py>) -> Result { + let seconds_offset = self.local_minus_utc(); + let td = PyDelta::new(py, 0, seconds_offset, 0, true)?; + PyTzInfo::fixed_offset(py, td) } } -impl IntoPy for FixedOffset { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &FixedOffset { + type Target = PyTzInfo; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } -impl FromPyObject<'_> for FixedOffset { +impl FromPyObject<'_, '_> for FixedOffset { + type Error = PyErr; + /// Convert python tzinfo to rust [`FixedOffset`]. /// /// Note that the conversion will result in precision lost in microseconds as chrono offset /// does not supports microseconds. - fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - #[cfg(not(Py_LIMITED_API))] - let ob: &PyTzInfo = ob.extract()?; - #[cfg(Py_LIMITED_API)] - check_type(ob, &DatetimeTypes::get(ob.py()).tzinfo, "PyTzInfo")?; + fn extract(ob: Borrowed<'_, '_, PyAny>) -> Result { + let ob = ob.cast::()?; - // Passing `()` (so Python's None) to the `utcoffset` function will only + // Passing Python's None to the `utcoffset` function will only // work for timezones defined as fixed offsets in Python. // Any other timezone would require a datetime as the parameter, and return // None if the datetime is not provided. // Trying to convert None to a PyDelta in the next line will then fail. - let py_timedelta = ob.call_method1("utcoffset", ((),))?; + let py_timedelta = + ob.call_method1(intern!(ob.py(), "utcoffset"), (PyNone::get(ob.py()),))?; if py_timedelta.is_none() { return Err(PyTypeError::new_err(format!( - "{:?} is not a fixed offset timezone", - ob + "{ob:?} is not a fixed offset timezone" ))); } let total_seconds: Duration = py_timedelta.extract()?; @@ -365,21 +413,32 @@ impl FromPyObject<'_> for FixedOffset { } } -impl ToPyObject for Utc { - fn to_object(&self, py: Python<'_>) -> PyObject { - timezone_utc_bound(py).into() +impl<'py> IntoPyObject<'py> for Utc { + type Target = PyTzInfo; + type Output = Borrowed<'static, 'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + PyTzInfo::utc(py) } } -impl IntoPy for Utc { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &Utc { + type Target = PyTzInfo; + type Output = Borrowed<'static, 'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } -impl FromPyObject<'_> for Utc { - fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - let py_utc = timezone_utc_bound(ob.py()); +impl FromPyObject<'_, '_> for Utc { + type Error = PyErr; + + fn extract(ob: Borrowed<'_, '_, PyAny>) -> Result { + let py_utc = PyTzInfo::utc(ob.py())?; if ob.eq(py_utc)? { Ok(Utc) } else { @@ -388,6 +447,61 @@ impl FromPyObject<'_> for Utc { } } +#[cfg(feature = "chrono-local")] +impl<'py> IntoPyObject<'py> for Local { + type Target = PyTzInfo; + type Output = Borrowed<'static, 'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + static LOCAL_TZ: PyOnceLock> = PyOnceLock::new(); + let tz = LOCAL_TZ + .get_or_try_init(py, || { + let iana_name = iana_time_zone::get_timezone().map_err(|e| { + PyRuntimeError::new_err(format!("Could not get local timezone: {e}")) + })?; + PyTzInfo::timezone(py, iana_name).map(Bound::unbind) + })? + .bind_borrowed(py); + Ok(tz) + } +} + +#[cfg(feature = "chrono-local")] +impl<'py> IntoPyObject<'py> for &Local { + type Target = PyTzInfo; + type Output = Borrowed<'static, 'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) + } +} + +#[cfg(feature = "chrono-local")] +impl FromPyObject<'_, '_> for Local { + type Error = PyErr; + + fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult { + let local_tz = Local.into_pyobject(ob.py())?; + if ob.eq(local_tz)? { + Ok(Local) + } else { + let name = local_tz.getattr("key")?.cast_into::()?; + Err(PyValueError::new_err(format!( + "expected local timezone {}", + name.to_cow()? + ))) + } + } + + #[inline] + fn as_local_tz(_: crate::conversion::private::Token) -> Option { + Some(Local) + } +} + struct DateArgs { year: i32, month: u8, @@ -428,49 +542,22 @@ impl From<&NaiveTime> for TimeArgs { } } -fn naive_datetime_to_py_datetime( - py: Python<'_>, - naive_datetime: &NaiveDateTime, - #[cfg(not(Py_LIMITED_API))] tzinfo: Option<&Bound<'_, PyTzInfo>>, - #[cfg(Py_LIMITED_API)] tzinfo: Option<&Bound<'_, PyAny>>, -) -> PyObject { - let DateArgs { year, month, day } = (&naive_datetime.date()).into(); - let TimeArgs { - hour, - min, - sec, - micro, - truncated_leap_second, - } = (&naive_datetime.time()).into(); - #[cfg(not(Py_LIMITED_API))] - let datetime = PyDateTime::new_bound(py, year, month, day, hour, min, sec, micro, tzinfo) - .expect("failed to construct datetime"); - #[cfg(Py_LIMITED_API)] - let datetime = DatetimeTypes::get(py) - .datetime - .bind(py) - .call1((year, month, day, hour, min, sec, micro, tzinfo)) - .expect("failed to construct datetime.datetime"); - if truncated_leap_second { - warn_truncated_leap_second(&datetime); - } - datetime.into() -} - fn warn_truncated_leap_second(obj: &Bound<'_, PyAny>) { let py = obj.py(); - if let Err(e) = PyErr::warn_bound( + if let Err(e) = PyErr::warn( py, - &py.get_type::().as_borrowed(), - "ignored leap-second, `datetime` does not support leap-seconds", + &py.get_type::(), + c"ignored leap-second, `datetime` does not support leap-seconds", 0, ) { - e.write_unraisable_bound(py, Some(&obj.as_borrowed())) + e.write_unraisable(py, Some(obj)) }; } #[cfg(not(Py_LIMITED_API))] -fn py_date_to_naive_date(py_date: &impl PyDateAccess) -> PyResult { +fn py_date_to_naive_date( + py_date: impl std::ops::Deref, +) -> PyResult { NaiveDate::from_ymd_opt( py_date.get_year(), py_date.get_month().into(), @@ -490,7 +577,9 @@ fn py_date_to_naive_date(py_date: &Bound<'_, PyAny>) -> PyResult { } #[cfg(not(Py_LIMITED_API))] -fn py_time_to_naive_time(py_time: &impl PyTimeAccess) -> PyResult { +fn py_time_to_naive_time( + py_time: impl std::ops::Deref, +) -> PyResult { NaiveTime::from_hms_micro_opt( py_time.get_hour().into(), py_time.get_minute().into(), @@ -517,56 +606,36 @@ fn py_time_to_naive_time(py_time: &Bound<'_, PyAny>) -> PyResult { .ok_or_else(|| PyValueError::new_err("invalid or out-of-range time")) } -#[cfg(Py_LIMITED_API)] -fn check_type(value: &Bound<'_, PyAny>, t: &PyObject, type_name: &'static str) -> PyResult<()> { - if !value.is_instance(t.bind(value.py()))? { - return Err(DowncastError::new(value, type_name).into()); - } - Ok(()) -} - -#[cfg(Py_LIMITED_API)] -struct DatetimeTypes { - date: PyObject, - datetime: PyObject, - time: PyObject, - timedelta: PyObject, - timezone: PyObject, - timezone_utc: PyObject, - tzinfo: PyObject, -} - -#[cfg(Py_LIMITED_API)] -impl DatetimeTypes { - fn get(py: Python<'_>) -> &Self { - static TYPES: GILOnceCell = GILOnceCell::new(); - TYPES - .get_or_try_init(py, || { - let datetime = py.import_bound("datetime")?; - let timezone = datetime.getattr("timezone")?; - Ok::<_, PyErr>(Self { - date: datetime.getattr("date")?.into(), - datetime: datetime.getattr("datetime")?.into(), - time: datetime.getattr("time")?.into(), - timedelta: datetime.getattr("timedelta")?.into(), - timezone_utc: timezone.getattr("utc")?.into(), - timezone: timezone.into(), - tzinfo: datetime.getattr("tzinfo")?.into(), - }) - }) - .expect("failed to load datetime module") +fn py_datetime_to_datetime_with_timezone( + dt: &Bound<'_, PyDateTime>, + tz: Tz, +) -> PyResult> { + let naive_dt = NaiveDateTime::new(py_date_to_naive_date(dt)?, py_time_to_naive_time(dt)?); + match naive_dt.and_local_timezone(tz) { + LocalResult::Single(value) => Ok(value), + LocalResult::Ambiguous(earliest, latest) => { + #[cfg(not(Py_LIMITED_API))] + let fold = dt.get_fold(); + + #[cfg(Py_LIMITED_API)] + let fold = dt.getattr(intern!(dt.py(), "fold"))?.extract::()? > 0; + + if fold { + Ok(latest) + } else { + Ok(earliest) + } + } + LocalResult::None => Err(PyValueError::new_err(format!( + "The datetime {dt:?} contains an incompatible timezone" + ))), } } -#[cfg(Py_LIMITED_API)] -fn timezone_utc_bound(py: Python<'_>) -> Bound<'_, PyAny> { - DatetimeTypes::get(py).timezone_utc.bind(py).clone() -} - #[cfg(test)] mod tests { use super::*; - use crate::{types::PyTuple, Py}; + use crate::{test_utils::assert_warnings, types::PyTuple, BoundObject}; use std::{cmp::Ordering, panic}; #[test] @@ -578,10 +647,10 @@ mod tests { use crate::types::any::PyAnyMethods; use crate::types::dict::PyDictMethods; - Python::with_gil(|py| { - let locals = crate::types::PyDict::new_bound(py); - py.run_bound( - "import zoneinfo; zi = zoneinfo.ZoneInfo('Europe/London')", + Python::attach(|py| { + let locals = crate::types::PyDict::new(py); + py.run( + c"import zoneinfo; zi = zoneinfo.ZoneInfo('Europe/London')", None, Some(&locals), ) @@ -599,7 +668,7 @@ mod tests { fn test_timezone_aware_to_naive_fails() { // Test that if a user tries to convert a python's timezone aware datetime into a naive // one, the conversion fails. - Python::with_gil(|py| { + Python::attach(|py| { let py_datetime = new_py_datetime_ob(py, "datetime", (2022, 1, 1, 1, 0, 0, 0, python_utc(py))); // Now test that converting a PyDateTime with tzinfo to a NaiveDateTime fails @@ -615,7 +684,7 @@ mod tests { fn test_naive_to_timezone_aware_fails() { // Test that if a user tries to convert a python's timezone aware datetime into a naive // one, the conversion fails. - Python::with_gil(|py| { + Python::attach(|py| { let py_datetime = new_py_datetime_ob(py, "datetime", (2022, 1, 1, 1, 0, 0, 0)); // Now test that converting a PyDateTime with tzinfo to a NaiveDateTime fails let res: PyResult> = py_datetime.extract(); @@ -637,15 +706,15 @@ mod tests { fn test_invalid_types_fail() { // Test that if a user tries to convert a python's timezone aware datetime into a naive // one, the conversion fails. - Python::with_gil(|py| { - let none = py.None().into_ref(py); + Python::attach(|py| { + let none = py.None().into_bound(py); assert_eq!( none.extract::().unwrap_err().to_string(), - "TypeError: 'NoneType' object cannot be converted to 'PyDelta'" + "TypeError: 'NoneType' object cannot be cast as 'timedelta'" ); assert_eq!( none.extract::().unwrap_err().to_string(), - "TypeError: 'NoneType' object cannot be converted to 'PyTzInfo'" + "TypeError: 'NoneType' object cannot be cast as 'tzinfo'" ); assert_eq!( none.extract::().unwrap_err().to_string(), @@ -653,43 +722,40 @@ mod tests { ); assert_eq!( none.extract::().unwrap_err().to_string(), - "TypeError: 'NoneType' object cannot be converted to 'PyTime'" + "TypeError: 'NoneType' object cannot be cast as 'time'" ); assert_eq!( none.extract::().unwrap_err().to_string(), - "TypeError: 'NoneType' object cannot be converted to 'PyDate'" + "TypeError: 'NoneType' object cannot be cast as 'date'" ); assert_eq!( none.extract::().unwrap_err().to_string(), - "TypeError: 'NoneType' object cannot be converted to 'PyDateTime'" + "TypeError: 'NoneType' object cannot be cast as 'datetime'" ); assert_eq!( none.extract::>().unwrap_err().to_string(), - "TypeError: 'NoneType' object cannot be converted to 'PyDateTime'" + "TypeError: 'NoneType' object cannot be cast as 'datetime'" ); assert_eq!( none.extract::>() .unwrap_err() .to_string(), - "TypeError: 'NoneType' object cannot be converted to 'PyDateTime'" + "TypeError: 'NoneType' object cannot be cast as 'datetime'" ); }); } #[test] - fn test_pyo3_timedelta_topyobject() { + fn test_pyo3_timedelta_into_pyobject() { // Utility function used to check different durations. // The `name` parameter is used to identify the check in case of a failure. let check = |name: &'static str, delta: Duration, py_days, py_seconds, py_ms| { - Python::with_gil(|py| { - let delta = delta.to_object(py); + Python::attach(|py| { + let delta = delta.into_pyobject(py).unwrap(); let py_delta = new_py_datetime_ob(py, "timedelta", (py_days, py_seconds, py_ms)); assert!( - delta.bind(py).eq(&py_delta).unwrap(), - "{}: {} != {}", - name, - delta, - py_delta + delta.eq(&py_delta).unwrap(), + "{name}: {delta} != {py_delta}" ); }); }; @@ -706,10 +772,14 @@ mod tests { let delta = Duration::seconds(86399999999999) + Duration::nanoseconds(999999000); // max check("delta max value", delta, 999999999, 86399, 999999); - // Also check that trying to convert an out of bound value panics. - Python::with_gil(|py| { - assert!(panic::catch_unwind(|| Duration::min_value().to_object(py)).is_err()); - assert!(panic::catch_unwind(|| Duration::max_value().to_object(py)).is_err()); + // Also check that trying to convert an out of bound value errors. + Python::attach(|py| { + // min_value and max_value were deprecated in chrono 0.4.39 + #[allow(deprecated)] + { + assert!(Duration::min_value().into_pyobject(py).is_err()); + assert!(Duration::max_value().into_pyobject(py).is_err()); + } }); } @@ -718,10 +788,10 @@ mod tests { // Utility function used to check different durations. // The `name` parameter is used to identify the check in case of a failure. let check = |name: &'static str, delta: Duration, py_days, py_seconds, py_ms| { - Python::with_gil(|py| { + Python::attach(|py| { let py_delta = new_py_datetime_ob(py, "timedelta", (py_days, py_seconds, py_ms)); let py_delta: Duration = py_delta.extract().unwrap(); - assert_eq!(py_delta, delta, "{}: {} != {}", name, py_delta, delta); + assert_eq!(py_delta, delta, "{name}: {py_delta} != {delta}"); }) }; @@ -745,7 +815,7 @@ mod tests { // This check is to assert that we can't construct every possible Duration from a PyDelta // since they have different bounds. - Python::with_gil(|py| { + Python::attach(|py| { let low_days: i32 = -1000000000; // This is possible assert!(panic::catch_unwind(|| Duration::days(low_days as i64)).is_ok()); @@ -773,20 +843,18 @@ mod tests { } #[test] - fn test_pyo3_date_topyobject() { + fn test_pyo3_date_into_pyobject() { let eq_ymd = |name: &'static str, year, month, day| { - Python::with_gil(|py| { + Python::attach(|py| { let date = NaiveDate::from_ymd_opt(year, month, day) .unwrap() - .to_object(py); + .into_pyobject(py) + .unwrap(); let py_date = new_py_datetime_ob(py, "date", (year, month, day)); assert_eq!( - date.bind(py).compare(&py_date).unwrap(), + date.compare(&py_date).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - date, - py_date + "{name}: {date} != {py_date}" ); }) }; @@ -800,11 +868,11 @@ mod tests { #[test] fn test_pyo3_date_frompyobject() { let eq_ymd = |name: &'static str, year, month, day| { - Python::with_gil(|py| { + Python::attach(|py| { let py_date = new_py_datetime_ob(py, "date", (year, month, day)); let py_date: NaiveDate = py_date.extract().unwrap(); let date = NaiveDate::from_ymd_opt(year, month, day).unwrap(); - assert_eq!(py_date, date, "{}: {} != {}", name, date, py_date); + assert_eq!(py_date, date, "{name}: {date} != {py_date}"); }) }; @@ -815,8 +883,8 @@ mod tests { } #[test] - fn test_pyo3_datetime_topyobject_utc() { - Python::with_gil(|py| { + fn test_pyo3_datetime_into_pyobject_utc() { + Python::attach(|py| { let check_utc = |name: &'static str, year, month, day, hour, minute, second, ms, py_ms| { let datetime = NaiveDate::from_ymd_opt(year, month, day) @@ -824,7 +892,7 @@ mod tests { .and_hms_micro_opt(hour, minute, second, ms) .unwrap() .and_utc(); - let datetime = datetime.to_object(py); + let datetime = datetime.into_pyobject(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", @@ -840,12 +908,9 @@ mod tests { ), ); assert_eq!( - datetime.bind(py).compare(&py_datetime).unwrap(), + datetime.compare(&py_datetime).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - datetime, - py_datetime + "{name}: {datetime} != {py_datetime}" ); }; @@ -863,8 +928,8 @@ mod tests { } #[test] - fn test_pyo3_datetime_topyobject_fixed_offset() { - Python::with_gil(|py| { + fn test_pyo3_datetime_into_pyobject_fixed_offset() { + Python::attach(|py| { let check_fixed_offset = |name: &'static str, year, month, day, hour, minute, second, ms, py_ms| { let offset = FixedOffset::east_opt(3600).unwrap(); @@ -874,20 +939,17 @@ mod tests { .unwrap() .and_local_timezone(offset) .unwrap(); - let datetime = datetime.to_object(py); - let py_tz = offset.to_object(py); + let datetime = datetime.into_pyobject(py).unwrap(); + let py_tz = offset.into_pyobject(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", (year, month, day, hour, minute, second, py_ms, py_tz), ); assert_eq!( - datetime.bind(py).compare(&py_datetime).unwrap(), + datetime.compare(&py_datetime).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - datetime, - py_datetime + "{name}: {datetime} != {py_datetime}" ); }; @@ -904,9 +966,38 @@ mod tests { }) } + #[test] + #[cfg(all(Py_3_9, feature = "chrono-tz", not(windows)))] + fn test_pyo3_datetime_into_pyobject_tz() { + Python::attach(|py| { + let datetime = NaiveDate::from_ymd_opt(2024, 12, 11) + .unwrap() + .and_hms_opt(23, 3, 13) + .unwrap() + .and_local_timezone(chrono_tz::Tz::Europe__London) + .unwrap(); + let datetime = datetime.into_pyobject(py).unwrap(); + let py_datetime = new_py_datetime_ob( + py, + "datetime", + ( + 2024, + 12, + 11, + 23, + 3, + 13, + 0, + python_zoneinfo(py, "Europe/London"), + ), + ); + assert_eq!(datetime.compare(&py_datetime).unwrap(), Ordering::Equal); + }) + } + #[test] fn test_pyo3_datetime_frompyobject_utc() { - Python::with_gil(|py| { + Python::attach(|py| { let year = 2014; let month = 5; let day = 6; @@ -914,7 +1005,7 @@ mod tests { let minute = 8; let second = 9; let micro = 999_999; - let tz_utc = timezone_utc_bound(py); + let tz_utc = PyTzInfo::utc(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", @@ -930,9 +1021,36 @@ mod tests { }) } + #[test] + #[cfg(feature = "chrono-local")] + fn test_pyo3_naive_datetime_frompyobject_local() { + Python::attach(|py| { + let year = 2014; + let month = 5; + let day = 6; + let hour = 7; + let minute = 8; + let second = 9; + let micro = 999_999; + let py_datetime = new_py_datetime_ob( + py, + "datetime", + (year, month, day, hour, minute, second, micro), + ); + let py_datetime: DateTime = py_datetime.extract().unwrap(); + let expected_datetime = NaiveDate::from_ymd_opt(year, month, day) + .unwrap() + .and_hms_micro_opt(hour, minute, second, micro) + .unwrap() + .and_local_timezone(Local) + .unwrap(); + assert_eq!(py_datetime, expected_datetime); + }) + } + #[test] fn test_pyo3_datetime_frompyobject_fixed_offset() { - Python::with_gil(|py| { + Python::attach(|py| { let year = 2014; let month = 5; let day = 6; @@ -941,7 +1059,7 @@ mod tests { let second = 9; let micro = 999_999; let offset = FixedOffset::east_opt(3600).unwrap(); - let py_tz = offset.to_object(py); + let py_tz = offset.into_pyobject(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", @@ -974,27 +1092,33 @@ mod tests { } #[test] - fn test_pyo3_offset_fixed_topyobject() { - Python::with_gil(|py| { + fn test_pyo3_offset_fixed_into_pyobject() { + Python::attach(|py| { // Chrono offset - let offset = FixedOffset::east_opt(3600).unwrap().to_object(py); + let offset = FixedOffset::east_opt(3600) + .unwrap() + .into_pyobject(py) + .unwrap(); // Python timezone from timedelta let td = new_py_datetime_ob(py, "timedelta", (0, 3600, 0)); let py_timedelta = new_py_datetime_ob(py, "timezone", (td,)); // Should be equal - assert!(offset.as_ref(py).eq(py_timedelta).unwrap()); + assert!(offset.eq(py_timedelta).unwrap()); // Same but with negative values - let offset = FixedOffset::east_opt(-3600).unwrap().to_object(py); + let offset = FixedOffset::east_opt(-3600) + .unwrap() + .into_pyobject(py) + .unwrap(); let td = new_py_datetime_ob(py, "timedelta", (0, -3600, 0)); let py_timedelta = new_py_datetime_ob(py, "timezone", (td,)); - assert!(offset.as_ref(py).eq(py_timedelta).unwrap()); + assert!(offset.eq(py_timedelta).unwrap()); }) } #[test] fn test_pyo3_offset_fixed_frompyobject() { - Python::with_gil(|py| { + Python::attach(|py| { let py_timedelta = new_py_datetime_ob(py, "timedelta", (0, 3600, 0)); let py_tzinfo = new_py_datetime_ob(py, "timezone", (py_timedelta,)); let offset: FixedOffset = py_tzinfo.extract().unwrap(); @@ -1003,17 +1127,17 @@ mod tests { } #[test] - fn test_pyo3_offset_utc_topyobject() { - Python::with_gil(|py| { - let utc = Utc.to_object(py); + fn test_pyo3_offset_utc_into_pyobject() { + Python::attach(|py| { + let utc = Utc.into_pyobject(py).unwrap(); let py_utc = python_utc(py); - assert!(utc.bind(py).is(&py_utc)); + assert!(utc.is(&py_utc)); }) } #[test] fn test_pyo3_offset_utc_frompyobject() { - Python::with_gil(|py| { + Python::attach(|py| { let py_utc = python_utc(py); let py_utc: Utc = py_utc.extract().unwrap(); assert_eq!(Utc, py_utc); @@ -1030,20 +1154,15 @@ mod tests { } #[test] - fn test_pyo3_time_topyobject() { - Python::with_gil(|py| { + fn test_pyo3_time_into_pyobject() { + Python::attach(|py| { let check_time = |name: &'static str, hour, minute, second, ms, py_ms| { let time = NaiveTime::from_hms_micro_opt(hour, minute, second, ms) .unwrap() - .to_object(py); + .into_pyobject(py) + .unwrap(); let py_time = new_py_datetime_ob(py, "time", (hour, minute, second, py_ms)); - assert!( - time.bind(py).eq(&py_time).unwrap(), - "{}: {} != {}", - name, - time, - py_time - ); + assert!(time.eq(&py_time).unwrap(), "{name}: {time} != {py_time}"); }; check_time("regular", 3, 5, 7, 999_999, 999_999); @@ -1065,7 +1184,7 @@ mod tests { let minute = 5; let second = 7; let micro = 999_999; - Python::with_gil(|py| { + Python::attach(|py| { let py_time = new_py_datetime_ob(py, "time", (hour, minute, second, micro)); let py_time: NaiveTime = py_time.extract().unwrap(); let time = NaiveTime::from_hms_micro_opt(hour, minute, second, micro).unwrap(); @@ -1073,21 +1192,25 @@ mod tests { }) } - fn new_py_datetime_ob<'py>( - py: Python<'py>, - name: &str, - args: impl IntoPy>, - ) -> Bound<'py, PyAny> { - py.import_bound("datetime") + fn new_py_datetime_ob<'py, A>(py: Python<'py>, name: &str, args: A) -> Bound<'py, PyAny> + where + A: IntoPyObject<'py, Target = PyTuple>, + { + py.import("datetime") .unwrap() .getattr(name) .unwrap() - .call1(args) + .call1( + args.into_pyobject(py) + .map_err(Into::into) + .unwrap() + .into_bound(), + ) .unwrap() } fn python_utc(py: Python<'_>) -> Bound<'_, PyAny> { - py.import_bound("datetime") + py.import("datetime") .unwrap() .getattr("timezone") .unwrap() @@ -1095,24 +1218,34 @@ mod tests { .unwrap() } - #[cfg(not(target_arch = "wasm32"))] + #[cfg(all(Py_3_9, feature = "chrono-tz", not(windows)))] + fn python_zoneinfo<'py>(py: Python<'py>, timezone: &str) -> Bound<'py, PyAny> { + py.import("zoneinfo") + .unwrap() + .getattr("ZoneInfo") + .unwrap() + .call1((timezone,)) + .unwrap() + } + + #[cfg(not(any(target_arch = "wasm32")))] mod proptests { use super::*; - use crate::tests::common::CatchWarnings; - use crate::types::any::PyAnyMethods; + use crate::test_utils::CatchWarnings; use crate::types::IntoPyDict; use proptest::prelude::*; + use std::ffi::CString; proptest! { // Range is limited to 1970 to 2038 due to windows limitations #[test] fn test_pyo3_offset_fixed_frompyobject_created_in_python(timestamp in 0..(i32::MAX as i64), timedelta in -86399i32..=86399i32) { - Python::with_gil(|py| { + Python::attach(|py| { - let globals = [("datetime", py.import_bound("datetime").unwrap())].into_py_dict_bound(py); - let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); - let t = py.eval_bound(&code, Some(&globals), None).unwrap(); + let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py).unwrap(); + let code = format!("datetime.datetime.fromtimestamp({timestamp}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={timedelta})))"); + let t = py.eval(&CString::new(code).unwrap(), Some(&globals), None).unwrap(); // Get ISO 8601 string from python let py_iso_str = t.call_method0("isoformat").unwrap(); @@ -1135,20 +1268,20 @@ mod tests { fn test_duration_roundtrip(days in -999999999i64..=999999999i64) { // Test roundtrip conversion rust->python->rust for all allowed // python values of durations (from -999999999 to 999999999 days), - Python::with_gil(|py| { + Python::attach(|py| { let dur = Duration::days(days); - let py_delta = dur.into_py(py); - let roundtripped: Duration = py_delta.extract(py).expect("Round trip"); + let py_delta = dur.into_pyobject(py).unwrap(); + let roundtripped: Duration = py_delta.extract().expect("Round trip"); assert_eq!(dur, roundtripped); }) } #[test] fn test_fixed_offset_roundtrip(secs in -86399i32..=86399i32) { - Python::with_gil(|py| { + Python::attach(|py| { let offset = FixedOffset::east_opt(secs).unwrap(); - let py_offset = offset.into_py(py); - let roundtripped: FixedOffset = py_offset.extract(py).expect("Round trip"); + let py_offset = offset.into_pyobject(py).unwrap(); + let roundtripped: FixedOffset = py_offset.extract().expect("Round trip"); assert_eq!(offset, roundtripped); }) } @@ -1161,12 +1294,12 @@ mod tests { ) { // Test roundtrip conversion rust->python->rust for all allowed // python dates (from year 1 to year 9999) - Python::with_gil(|py| { + Python::attach(|py| { // We use to `from_ymd_opt` constructor so that we only test valid `NaiveDate`s. // This is to skip the test if we are creating an invalid date, like February 31. if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) { - let py_date = date.to_object(py); - let roundtripped: NaiveDate = py_date.extract(py).expect("Round trip"); + let py_date = date.into_pyobject(py).unwrap(); + let roundtripped: NaiveDate = py_date.extract().expect("Round trip"); assert_eq!(date, roundtripped); } }) @@ -1183,11 +1316,11 @@ mod tests { // Python time has a resolution of microseconds, so we only test // NaiveTimes with microseconds resolution, even if NaiveTime has nanosecond // resolution. - Python::with_gil(|py| { + Python::attach(|py| { if let Some(time) = NaiveTime::from_hms_micro_opt(hour, min, sec, micro) { // Wrap in CatchWarnings to avoid to_object firing warning for truncated leap second - let py_time = CatchWarnings::enter(py, |_| Ok(time.to_object(py))).unwrap(); - let roundtripped: NaiveTime = py_time.extract(py).expect("Round trip"); + let py_time = CatchWarnings::enter(py, |_| time.into_pyobject(py)).unwrap(); + let roundtripped: NaiveTime = py_time.extract().expect("Round trip"); // Leap seconds are not roundtripped let expected_roundtrip_time = micro.checked_sub(1_000_000).map(|micro| NaiveTime::from_hms_micro_opt(hour, min, sec, micro).unwrap()).unwrap_or(time); assert_eq!(expected_roundtrip_time, roundtripped); @@ -1205,13 +1338,13 @@ mod tests { sec in 0u32..=60u32, micro in 0u32..=999_999u32 ) { - Python::with_gil(|py| { + Python::attach(|py| { let date_opt = NaiveDate::from_ymd_opt(year, month, day); let time_opt = NaiveTime::from_hms_micro_opt(hour, min, sec, micro); if let (Some(date), Some(time)) = (date_opt, time_opt) { let dt = NaiveDateTime::new(date, time); - let pydt = dt.to_object(py); - let roundtripped: NaiveDateTime = pydt.extract(py).expect("Round trip"); + let pydt = dt.into_pyobject(py).unwrap(); + let roundtripped: NaiveDateTime = pydt.extract().expect("Round trip"); assert_eq!(dt, roundtripped); } }) @@ -1227,14 +1360,14 @@ mod tests { sec in 0u32..=59u32, micro in 0u32..=1_999_999u32 ) { - Python::with_gil(|py| { + Python::attach(|py| { let date_opt = NaiveDate::from_ymd_opt(year, month, day); let time_opt = NaiveTime::from_hms_micro_opt(hour, min, sec, micro); if let (Some(date), Some(time)) = (date_opt, time_opt) { let dt: DateTime = NaiveDateTime::new(date, time).and_utc(); // Wrap in CatchWarnings to avoid into_py firing warning for truncated leap second - let py_dt = CatchWarnings::enter(py, |_| Ok(dt.into_py(py))).unwrap(); - let roundtripped: DateTime = py_dt.extract(py).expect("Round trip"); + let py_dt = CatchWarnings::enter(py, |_| dt.into_pyobject(py)).unwrap(); + let roundtripped: DateTime = py_dt.extract().expect("Round trip"); // Leap seconds are not roundtripped let expected_roundtrip_time = micro.checked_sub(1_000_000).map(|micro| NaiveTime::from_hms_micro_opt(hour, min, sec, micro).unwrap()).unwrap_or(time); let expected_roundtrip_dt: DateTime = NaiveDateTime::new(date, expected_roundtrip_time).and_utc(); @@ -1254,15 +1387,15 @@ mod tests { micro in 0u32..=1_999_999u32, offset_secs in -86399i32..=86399i32 ) { - Python::with_gil(|py| { + Python::attach(|py| { let date_opt = NaiveDate::from_ymd_opt(year, month, day); let time_opt = NaiveTime::from_hms_micro_opt(hour, min, sec, micro); let offset = FixedOffset::east_opt(offset_secs).unwrap(); if let (Some(date), Some(time)) = (date_opt, time_opt) { let dt: DateTime = NaiveDateTime::new(date, time).and_local_timezone(offset).unwrap(); // Wrap in CatchWarnings to avoid into_py firing warning for truncated leap second - let py_dt = CatchWarnings::enter(py, |_| Ok(dt.into_py(py))).unwrap(); - let roundtripped: DateTime = py_dt.extract(py).expect("Round trip"); + let py_dt = CatchWarnings::enter(py, |_| dt.into_pyobject(py)).unwrap(); + let roundtripped: DateTime = py_dt.extract().expect("Round trip"); // Leap seconds are not roundtripped let expected_roundtrip_time = micro.checked_sub(1_000_000).map(|micro| NaiveTime::from_hms_micro_opt(hour, min, sec, micro).unwrap()).unwrap_or(time); let expected_roundtrip_dt: DateTime = NaiveDateTime::new(date, expected_roundtrip_time).and_local_timezone(offset).unwrap(); @@ -1270,6 +1403,43 @@ mod tests { } }) } + + #[test] + #[cfg(all(feature = "chrono-local", not(target_os = "windows")))] + fn test_local_datetime_roundtrip( + year in 1i32..=9999i32, + month in 1u32..=12u32, + day in 1u32..=31u32, + hour in 0u32..=23u32, + min in 0u32..=59u32, + sec in 0u32..=59u32, + micro in 0u32..=1_999_999u32, + ) { + Python::attach(|py| { + let date_opt = NaiveDate::from_ymd_opt(year, month, day); + let time_opt = NaiveTime::from_hms_micro_opt(hour, min, sec, micro); + if let (Some(date), Some(time)) = (date_opt, time_opt) { + let dts = match NaiveDateTime::new(date, time).and_local_timezone(Local) { + LocalResult::None => return, + LocalResult::Single(dt) => [Some((dt, false)), None], + LocalResult::Ambiguous(dt1, dt2) => [Some((dt1, false)), Some((dt2, true))], + }; + for (dt, fold) in dts.iter().filter_map(|input| *input) { + // Wrap in CatchWarnings to avoid into_py firing warning for truncated leap second + let py_dt = CatchWarnings::enter(py, |_| dt.into_pyobject(py)).unwrap(); + let roundtripped: DateTime = py_dt.extract().expect("Round trip"); + // Leap seconds are not roundtripped + let expected_roundtrip_time = micro.checked_sub(1_000_000).map(|micro| NaiveTime::from_hms_micro_opt(hour, min, sec, micro).unwrap()).unwrap_or(time); + let expected_roundtrip_dt: DateTime = if fold { + NaiveDateTime::new(date, expected_roundtrip_time).and_local_timezone(Local).latest() + } else { + NaiveDateTime::new(date, expected_roundtrip_time).and_local_timezone(Local).earliest() + }.unwrap(); + assert_eq!(expected_roundtrip_dt, roundtripped); + } + } + }) + } } } } diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index b4d0e7edf8c..433a56b4ebf 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -21,59 +21,76 @@ //! //! ```rust,no_run //! use chrono_tz::Tz; -//! use pyo3::{Python, ToPyObject}; +//! use pyo3::{Python, PyResult, IntoPyObject, types::PyAnyMethods}; //! -//! fn main() { -//! pyo3::prepare_freethreaded_python(); -//! Python::with_gil(|py| { +//! fn main() -> PyResult<()> { +//! Python::initialize(); +//! Python::attach(|py| { //! // Convert to Python -//! let py_tzinfo = Tz::Europe__Paris.to_object(py); +//! let py_tzinfo = Tz::Europe__Paris.into_pyobject(py)?; //! // Convert back to Rust -//! assert_eq!(py_tzinfo.extract::(py).unwrap(), Tz::Europe__Paris); -//! }); +//! assert_eq!(py_tzinfo.extract::()?, Tz::Europe__Paris); +//! Ok(()) +//! }) //! } //! ``` +use crate::conversion::IntoPyObject; use crate::exceptions::PyValueError; -use crate::sync::GILOnceCell; -use crate::types::{any::PyAnyMethods, PyType}; -use crate::{ - intern, Bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, -}; +use crate::pybacked::PyBackedStr; +use crate::types::{any::PyAnyMethods, PyTzInfo}; +use crate::{intern, Borrowed, Bound, FromPyObject, PyAny, PyErr, Python}; use chrono_tz::Tz; use std::str::FromStr; -impl ToPyObject for Tz { - fn to_object(&self, py: Python<'_>) -> PyObject { - static ZONE_INFO: GILOnceCell> = GILOnceCell::new(); - ZONE_INFO - .get_or_try_init_type_ref(py, "zoneinfo", "ZoneInfo") - .unwrap() - .call1((self.name(),)) - .unwrap() - .unbind() +impl<'py> IntoPyObject<'py> for Tz { + type Target = PyTzInfo; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + PyTzInfo::timezone(py, self.name()) } } -impl IntoPy for Tz { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) +impl<'py> IntoPyObject<'py> for &Tz { + type Target = PyTzInfo; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) } } -impl FromPyObject<'_> for Tz { - fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - Tz::from_str(ob.getattr(intern!(ob.py(), "key"))?.extract()?) - .map_err(|e| PyValueError::new_err(e.to_string())) +impl FromPyObject<'_, '_> for Tz { + type Error = PyErr; + + fn extract(ob: Borrowed<'_, '_, PyAny>) -> Result { + Tz::from_str( + &ob.getattr(intern!(ob.py(), "key"))? + .extract::()?, + ) + .map_err(|e| PyValueError::new_err(e.to_string())) } } #[cfg(all(test, not(windows)))] // Troubles loading timezones on Windows mod tests { use super::*; + use crate::prelude::PyAnyMethods; + use crate::types::IntoPyDict; + use crate::types::PyTzInfo; + use crate::Bound; + use crate::Python; + use chrono::offset::LocalResult; + use chrono::NaiveDate; + use chrono::{DateTime, Utc}; + use chrono_tz::Tz; #[test] fn test_frompyobject() { - Python::with_gil(|py| { + Python::attach(|py| { assert_eq!( new_zoneinfo(py, "Europe/Paris").extract::().unwrap(), Tz::Europe__Paris @@ -87,32 +104,105 @@ mod tests { } #[test] - fn test_topyobject() { - Python::with_gil(|py| { - let assert_eq = |l: PyObject, r: Bound<'_, PyAny>| { - assert!(l.bind(py).eq(r).unwrap()); + fn test_ambiguous_datetime_to_pyobject() { + let dates = [ + DateTime::::from_str("2020-10-24 23:00:00 UTC").unwrap(), + DateTime::::from_str("2020-10-25 00:00:00 UTC").unwrap(), + DateTime::::from_str("2020-10-25 01:00:00 UTC").unwrap(), + ]; + + let dates = dates.map(|dt| dt.with_timezone(&Tz::Europe__London)); + + assert_eq!( + dates.map(|dt| dt.to_string()), + [ + "2020-10-25 00:00:00 BST", + "2020-10-25 01:00:00 BST", + "2020-10-25 01:00:00 GMT" + ] + ); + + let dates = Python::attach(|py| { + let pydates = dates.map(|dt| dt.into_pyobject(py).unwrap()); + assert_eq!( + pydates + .clone() + .map(|dt| dt.getattr("hour").unwrap().extract::().unwrap()), + [0, 1, 1] + ); + + assert_eq!( + pydates + .clone() + .map(|dt| dt.getattr("fold").unwrap().extract::().unwrap() > 0), + [false, false, true] + ); + + pydates.map(|dt| dt.extract::>().unwrap()) + }); + + assert_eq!( + dates.map(|dt| dt.to_string()), + [ + "2020-10-25 00:00:00 BST", + "2020-10-25 01:00:00 BST", + "2020-10-25 01:00:00 GMT" + ] + ); + } + + #[test] + fn test_nonexistent_datetime_from_pyobject() { + // Pacific_Apia skipped the 30th of December 2011 entirely + + let naive_dt = NaiveDate::from_ymd_opt(2011, 12, 30) + .unwrap() + .and_hms_opt(2, 0, 0) + .unwrap(); + let tz = Tz::Pacific__Apia; + + // sanity check + assert_eq!(naive_dt.and_local_timezone(tz), LocalResult::None); + + Python::attach(|py| { + // create as a Python object manually + let py_tz = tz.into_pyobject(py).unwrap(); + let py_dt_naive = naive_dt.into_pyobject(py).unwrap(); + let py_dt = py_dt_naive + .call_method( + "replace", + (), + Some(&[("tzinfo", py_tz)].into_py_dict(py).unwrap()), + ) + .unwrap(); + + // now try to extract + let err = py_dt.extract::>().unwrap_err(); + assert_eq!(err.to_string(), "ValueError: The datetime datetime.datetime(2011, 12, 30, 2, 0, tzinfo=zoneinfo.ZoneInfo(key='Pacific/Apia')) contains an incompatible timezone"); + }); + } + + #[test] + #[cfg(not(Py_GIL_DISABLED))] // https://github.com/python/cpython/issues/116738#issuecomment-2404360445 + fn test_into_pyobject() { + Python::attach(|py| { + let assert_eq = |l: Bound<'_, PyTzInfo>, r: Bound<'_, PyTzInfo>| { + assert!(l.eq(&r).unwrap(), "{l:?} != {r:?}"); }; assert_eq( - Tz::Europe__Paris.to_object(py), + Tz::Europe__Paris.into_pyobject(py).unwrap(), new_zoneinfo(py, "Europe/Paris"), ); - assert_eq(Tz::UTC.to_object(py), new_zoneinfo(py, "UTC")); + assert_eq(Tz::UTC.into_pyobject(py).unwrap(), new_zoneinfo(py, "UTC")); assert_eq( - Tz::Etc__GMTMinus5.to_object(py), + Tz::Etc__GMTMinus5.into_pyobject(py).unwrap(), new_zoneinfo(py, "Etc/GMT-5"), ); }); } - fn new_zoneinfo<'py>(py: Python<'py>, name: &str) -> Bound<'py, PyAny> { - zoneinfo_class(py).call1((name,)).unwrap() - } - - fn zoneinfo_class(py: Python<'_>) -> Bound<'_, PyAny> { - py.import_bound("zoneinfo") - .unwrap() - .getattr("ZoneInfo") - .unwrap() + fn new_zoneinfo<'py>(py: Python<'py>, name: &str) -> Bound<'py, PyTzInfo> { + PyTzInfo::timezone(py, name).unwrap() } } diff --git a/src/conversions/either.rs b/src/conversions/either.rs index c763cdf95e5..6a5acebae75 100644 --- a/src/conversions/either.rs +++ b/src/conversions/either.rs @@ -26,18 +26,19 @@ //! //! ```rust //! use either::Either; -//! use pyo3::{Python, ToPyObject}; +//! use pyo3::{Python, PyResult, IntoPyObject, types::PyAnyMethods}; //! -//! fn main() { -//! pyo3::prepare_freethreaded_python(); -//! Python::with_gil(|py| { +//! fn main() -> PyResult<()> { +//! Python::initialize(); +//! Python::attach(|py| { //! // Create a string and an int in Python. -//! let py_str = "crab".to_object(py); -//! let py_int = 42.to_object(py); +//! let py_str = "crab".into_pyobject(py)?; +//! let py_int = 42i32.into_pyobject(py)?; //! // Now convert it to an Either. -//! let either_str: Either = py_str.extract(py).unwrap(); -//! let either_int: Either = py_int.extract(py).unwrap(); -//! }); +//! let either_str: Either = py_str.extract()?; +//! let either_int: Either = py_int.extract()?; +//! Ok(()) +//! }) //! } //! ``` //! @@ -46,55 +47,69 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - exceptions::PyTypeError, types::any::PyAnyMethods, Bound, FromPyObject, IntoPy, PyAny, - PyObject, PyResult, Python, ToPyObject, + exceptions::PyTypeError, Borrowed, Bound, FromPyObject, IntoPyObject, IntoPyObjectExt, PyAny, + PyErr, Python, }; use either::Either; #[cfg_attr(docsrs, doc(cfg(feature = "either")))] -impl IntoPy for Either +impl<'py, L, R> IntoPyObject<'py> for Either where - L: IntoPy, - R: IntoPy, + L: IntoPyObject<'py>, + R: IntoPyObject<'py>, { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { match self { - Either::Left(l) => l.into_py(py), - Either::Right(r) => r.into_py(py), + Either::Left(l) => l.into_bound_py_any(py), + Either::Right(r) => r.into_bound_py_any(py), } } } #[cfg_attr(docsrs, doc(cfg(feature = "either")))] -impl ToPyObject for Either +impl<'a, 'py, L, R> IntoPyObject<'py> for &'a Either where - L: ToPyObject, - R: ToPyObject, + &'a L: IntoPyObject<'py>, + &'a R: IntoPyObject<'py>, { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { match self { - Either::Left(l) => l.to_object(py), - Either::Right(r) => r.to_object(py), + Either::Left(l) => l.into_bound_py_any(py), + Either::Right(r) => r.into_bound_py_any(py), } } } #[cfg_attr(docsrs, doc(cfg(feature = "either")))] -impl<'py, L, R> FromPyObject<'py> for Either +impl<'a, 'py, L, R> FromPyObject<'a, 'py> for Either where - L: FromPyObject<'py>, - R: FromPyObject<'py>, + L: FromPyObject<'a, 'py>, + R: FromPyObject<'a, 'py>, { + type Error = PyErr; + #[inline] - fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { if let Ok(l) = obj.extract::() { Ok(Either::Left(l)) } else if let Ok(r) = obj.extract::() { Ok(Either::Right(r)) } else { - let err_msg = format!("failed to convert the value to '{}'", Self::type_input()); + // TODO: it might be nice to use the `type_input()` name here once `type_input` + // is not experimental, rather than the Rust type names. + let err_msg = format!( + "failed to convert the value to 'Union[{}, {}]'", + std::any::type_name::(), + std::any::type_name::() + ); Err(PyTypeError::new_err(err_msg)) } } @@ -107,9 +122,12 @@ where #[cfg(test)] mod tests { + use std::borrow::Cow; + use crate::exceptions::PyTypeError; - use crate::{Python, ToPyObject}; + use crate::{IntoPyObject, Python}; + use crate::types::PyAnyMethods; use either::Either; #[test] @@ -118,32 +136,32 @@ mod tests { type E1 = Either; type E2 = Either; - Python::with_gil(|py| { + Python::attach(|py| { let l = E::Left(42); - let obj_l = l.to_object(py); - assert_eq!(obj_l.extract::(py).unwrap(), 42); - assert_eq!(obj_l.extract::(py).unwrap(), l); + let obj_l = (&l).into_pyobject(py).unwrap(); + assert_eq!(obj_l.extract::().unwrap(), 42); + assert_eq!(obj_l.extract::().unwrap(), l); let r = E::Right("foo".to_owned()); - let obj_r = r.to_object(py); - assert_eq!(obj_r.extract::<&str>(py).unwrap(), "foo"); - assert_eq!(obj_r.extract::(py).unwrap(), r); + let obj_r = (&r).into_pyobject(py).unwrap(); + assert_eq!(obj_r.extract::>().unwrap(), "foo"); + assert_eq!(obj_r.extract::().unwrap(), r); - let obj_s = "foo".to_object(py); - let err = obj_s.extract::(py).unwrap_err(); + let obj_s = "foo".into_pyobject(py).unwrap(); + let err = obj_s.extract::().unwrap_err(); assert!(err.is_instance_of::(py)); assert_eq!( err.to_string(), - "TypeError: failed to convert the value to 'Union[int, float]'" + "TypeError: failed to convert the value to 'Union[i32, f32]'" ); - let obj_i = 42.to_object(py); - assert_eq!(obj_i.extract::(py).unwrap(), E1::Left(42)); - assert_eq!(obj_i.extract::(py).unwrap(), E2::Left(42.0)); + let obj_i = 42i32.into_pyobject(py).unwrap(); + assert_eq!(obj_i.extract::().unwrap(), E1::Left(42)); + assert_eq!(obj_i.extract::().unwrap(), E2::Left(42.0)); - let obj_f = 42.0.to_object(py); - assert_eq!(obj_f.extract::(py).unwrap(), E1::Right(42.0)); - assert_eq!(obj_f.extract::(py).unwrap(), E2::Left(42.0)); + let obj_f = 42.0f64.into_pyobject(py).unwrap(); + assert_eq!(obj_f.extract::().unwrap(), E1::Right(42.0)); + assert_eq!(obj_f.extract::().unwrap(), E2::Left(42.0)); }); } } diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index d25a10af9ee..5923854a188 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -34,7 +34,6 @@ //! //! ```rust //! use pyo3::prelude::*; -//! use pyo3::wrap_pyfunction; //! use std::path::PathBuf; //! //! // A wrapper around a Rust function. @@ -46,7 +45,7 @@ //! } //! //! fn main() { -//! let error = Python::with_gil(|py| -> PyResult> { +//! let error = Python::attach(|py| -> PyResult> { //! let fun = wrap_pyfunction!(py_open, py)?; //! let text = fun.call1(("foo.txt",))?.extract::>()?; //! Ok(text) @@ -73,10 +72,10 @@ //! // An arbitrary example of a Python api you //! // could call inside an application... //! // This might return a `PyErr`. -//! let res = Python::with_gil(|py| { +//! let res = Python::attach(|py| { //! let zlib = PyModule::import(py, "zlib")?; //! let decompress = zlib.getattr("decompress")?; -//! let bytes = PyBytes::new_bound(py, bytes); +//! let bytes = PyBytes::new(py, bytes); //! let value = decompress.call1((bytes,))?; //! value.extract::>() //! })?; @@ -120,7 +119,7 @@ impl From for PyErr { Err(error) => error, }; } - PyRuntimeError::new_err(format!("{:?}", error)) + PyRuntimeError::new_err(format!("{error:?}")) } } @@ -148,12 +147,12 @@ mod tests { #[test] fn test_pyo3_exception_contents() { let err = h().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); - Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict_bound(py); - let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); + Python::attach(|py| { + let locals = [("err", pyerr)].into_py_dict(py).unwrap(); + let pyerr = py.run(c"raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } @@ -165,12 +164,12 @@ mod tests { #[test] fn test_pyo3_exception_contents2() { let err = k().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); - Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict_bound(py); - let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); + Python::attach(|py| { + let locals = [("err", pyerr)].into_py_dict(py).unwrap(); + let pyerr = py.run(c"raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } @@ -180,7 +179,7 @@ mod tests { let origin_exc = PyValueError::new_err("Value Error"); let report: Report = origin_exc.into(); let converted: PyErr = report.into(); - assert!(Python::with_gil( + assert!(Python::attach( |py| converted.is_instance_of::(py) )) } @@ -190,7 +189,7 @@ mod tests { let mut report: Report = origin_exc.into(); report = report.wrap_err("Wrapped"); let converted: PyErr = report.into(); - assert!(Python::with_gil( + assert!(Python::attach( |py| converted.is_instance_of::(py) )) } diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index 7e57d332d91..48735a9734c 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -17,90 +17,124 @@ //! Note that you must use compatible versions of hashbrown and PyO3. //! The required hashbrown version may vary based on the version of PyO3. use crate::{ - types::any::PyAnyMethods, - types::dict::PyDictMethods, - types::frozenset::PyFrozenSetMethods, - types::set::{new_from_iter, PySetMethods}, - types::{IntoPyDict, PyDict, PyFrozenSet, PySet}, - Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, + conversion::{FromPyObjectOwned, IntoPyObject}, + types::{ + any::PyAnyMethods, + dict::PyDictMethods, + frozenset::PyFrozenSetMethods, + set::{try_new_from_iter, PySetMethods}, + PyDict, PyFrozenSet, PySet, + }, + Borrowed, Bound, FromPyObject, PyAny, PyErr, PyResult, Python, }; use std::{cmp, hash}; -impl ToPyObject for hashbrown::HashMap +impl<'py, K, V, H> IntoPyObject<'py> for hashbrown::HashMap where - K: hash::Hash + cmp::Eq + ToPyObject, - V: ToPyObject, + K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + V: IntoPyObject<'py>, H: hash::BuildHasher, { - fn to_object(&self, py: Python<'_>) -> PyObject { - IntoPyDict::into_py_dict_bound(self, py).into() + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k, v)?; + } + Ok(dict) } } -impl IntoPy for hashbrown::HashMap +impl<'a, 'py, K, V, H> IntoPyObject<'py> for &'a hashbrown::HashMap where - K: hash::Hash + cmp::Eq + IntoPy, - V: IntoPy, + &'a K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + &'a V: IntoPyObject<'py>, H: hash::BuildHasher, { - fn into_py(self, py: Python<'_>) -> PyObject { - let iter = self - .into_iter() - .map(|(k, v)| (k.into_py(py), v.into_py(py))); - IntoPyDict::into_py_dict_bound(iter, py).into() + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k, v)?; + } + Ok(dict) } } -impl<'py, K, V, S> FromPyObject<'py> for hashbrown::HashMap +impl<'py, K, V, S> FromPyObject<'_, 'py> for hashbrown::HashMap where - K: FromPyObject<'py> + cmp::Eq + hash::Hash, - V: FromPyObject<'py>, + K: FromPyObjectOwned<'py> + cmp::Eq + hash::Hash, + V: FromPyObjectOwned<'py>, S: hash::BuildHasher + Default, { - fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { - let dict = ob.downcast::()?; + type Error = PyErr; + + fn extract(ob: Borrowed<'_, 'py, PyAny>) -> Result { + let dict = ob.cast::()?; let mut ret = hashbrown::HashMap::with_capacity_and_hasher(dict.len(), S::default()); for (k, v) in dict.iter() { - ret.insert(k.extract()?, v.extract()?); + ret.insert( + k.extract().map_err(Into::into)?, + v.extract().map_err(Into::into)?, + ); } Ok(ret) } } -impl ToPyObject for hashbrown::HashSet +impl<'py, K, H> IntoPyObject<'py> for hashbrown::HashSet where - T: hash::Hash + Eq + ToPyObject, + K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + H: hash::BuildHasher, { - fn to_object(&self, py: Python<'_>) -> PyObject { - new_from_iter(py, self) - .expect("Failed to create Python set from hashbrown::HashSet") - .into() + type Target = PySet; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + try_new_from_iter(py, self) } } -impl IntoPy for hashbrown::HashSet +impl<'a, 'py, K, H> IntoPyObject<'py> for &'a hashbrown::HashSet where - K: IntoPy + Eq + hash::Hash, - S: hash::BuildHasher + Default, + &'a K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + H: hash::BuildHasher, { - fn into_py(self, py: Python<'_>) -> PyObject { - new_from_iter(py, self.into_iter().map(|item| item.into_py(py))) - .expect("Failed to create Python set from hashbrown::HashSet") - .into() + type Target = PySet; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + try_new_from_iter(py, self) } } -impl<'py, K, S> FromPyObject<'py> for hashbrown::HashSet +impl<'py, K, S> FromPyObject<'_, 'py> for hashbrown::HashSet where - K: FromPyObject<'py> + cmp::Eq + hash::Hash, + K: FromPyObjectOwned<'py> + cmp::Eq + hash::Hash, S: hash::BuildHasher + Default, { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - match ob.downcast::() { - Ok(set) => set.iter().map(|any| any.extract()).collect(), + type Error = PyErr; + + fn extract(ob: Borrowed<'_, 'py, PyAny>) -> PyResult { + match ob.cast::() { + Ok(set) => set + .iter() + .map(|any| any.extract().map_err(Into::into)) + .collect(), Err(err) => { - if let Ok(frozen_set) = ob.downcast::() { - frozen_set.iter().map(|any| any.extract()).collect() + if let Ok(frozen_set) = ob.cast::() { + frozen_set + .iter() + .map(|any| any.extract().map_err(Into::into)) + .collect() } else { Err(PyErr::from(err)) } @@ -112,18 +146,19 @@ where #[cfg(test)] mod tests { use super::*; - use crate::types::any::PyAnyMethods; + use crate::types::IntoPyDict; + use std::collections::hash_map::RandomState; #[test] - fn test_hashbrown_hashmap_to_python() { - Python::with_gil(|py| { - let mut map = hashbrown::HashMap::::new(); + fn test_hashbrown_hashmap_into_pyobject() { + Python::attach(|py| { + let mut map = + hashbrown::HashMap::::with_hasher(RandomState::new()); map.insert(1, 1); - let m = map.to_object(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = (&map).into_pyobject(py).unwrap(); - assert!(py_map.len() == 1); + assert_eq!(py_map.len(), 1); assert!( py_map .get_item(1) @@ -136,35 +171,15 @@ mod tests { assert_eq!(map, py_map.extract().unwrap()); }); } - #[test] - fn test_hashbrown_hashmap_into_python() { - Python::with_gil(|py| { - let mut map = hashbrown::HashMap::::new(); - map.insert(1, 1); - - let m: PyObject = map.into_py(py); - let py_map: &PyDict = m.downcast(py).unwrap(); - - assert!(py_map.len() == 1); - assert!( - py_map - .get_item(1) - .unwrap() - .unwrap() - .extract::() - .unwrap() - == 1 - ); - }); - } #[test] fn test_hashbrown_hashmap_into_dict() { - Python::with_gil(|py| { - let mut map = hashbrown::HashMap::::new(); + Python::attach(|py| { + let mut map = + hashbrown::HashMap::::with_hasher(RandomState::new()); map.insert(1, 1); - let py_map = map.into_py_dict_bound(py); + let py_map = map.into_py_dict(py).unwrap(); assert_eq!(py_map.len(), 1); assert_eq!( @@ -181,25 +196,26 @@ mod tests { #[test] fn test_extract_hashbrown_hashset() { - Python::with_gil(|py| { - let set = PySet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); - let hash_set: hashbrown::HashSet = set.extract().unwrap(); + Python::attach(|py| { + let set = PySet::new(py, [1, 2, 3, 4, 5]).unwrap(); + let hash_set: hashbrown::HashSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); - let set = PyFrozenSet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); - let hash_set: hashbrown::HashSet = set.extract().unwrap(); + let set = PyFrozenSet::new(py, [1, 2, 3, 4, 5]).unwrap(); + let hash_set: hashbrown::HashSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); }); } #[test] - fn test_hashbrown_hashset_into_py() { - Python::with_gil(|py| { - let hs: hashbrown::HashSet = [1, 2, 3, 4, 5].iter().cloned().collect(); + fn test_hashbrown_hashset_into_pyobject() { + Python::attach(|py| { + let hs: hashbrown::HashSet = + [1, 2, 3, 4, 5].iter().cloned().collect(); - let hso: PyObject = hs.clone().into_py(py); + let hso = hs.clone().into_pyobject(py).unwrap(); - assert_eq!(hs, hso.extract(py).unwrap()); + assert_eq!(hs, hso.extract().unwrap()); }); } } diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index ec7ed5de557..cccd3c7a49a 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -71,7 +71,7 @@ //! } //! //! #[pymodule] -//! fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { //! m.add_function(wrap_pyfunction!(calculate_statistics, m)?)?; //! Ok(()) //! } @@ -87,48 +87,65 @@ //! # if another hash table was used, the order could be random //! ``` -use crate::types::any::PyAnyMethods; -use crate::types::dict::PyDictMethods; +use crate::conversion::{FromPyObjectOwned, IntoPyObject}; use crate::types::*; -use crate::{Bound, FromPyObject, IntoPy, PyErr, PyObject, Python, ToPyObject}; +use crate::{Borrowed, Bound, FromPyObject, PyErr, Python}; use std::{cmp, hash}; -impl ToPyObject for indexmap::IndexMap +impl<'py, K, V, H> IntoPyObject<'py> for indexmap::IndexMap where - K: hash::Hash + cmp::Eq + ToPyObject, - V: ToPyObject, + K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + V: IntoPyObject<'py>, H: hash::BuildHasher, { - fn to_object(&self, py: Python<'_>) -> PyObject { - IntoPyDict::into_py_dict_bound(self, py).into() + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k, v)?; + } + Ok(dict) } } -impl IntoPy for indexmap::IndexMap +impl<'a, 'py, K, V, H> IntoPyObject<'py> for &'a indexmap::IndexMap where - K: hash::Hash + cmp::Eq + IntoPy, - V: IntoPy, + &'a K: IntoPyObject<'py> + cmp::Eq + hash::Hash, + &'a V: IntoPyObject<'py>, H: hash::BuildHasher, { - fn into_py(self, py: Python<'_>) -> PyObject { - let iter = self - .into_iter() - .map(|(k, v)| (k.into_py(py), v.into_py(py))); - IntoPyDict::into_py_dict_bound(iter, py).into() + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); + for (k, v) in self { + dict.set_item(k, v)?; + } + Ok(dict) } } -impl<'py, K, V, S> FromPyObject<'py> for indexmap::IndexMap +impl<'py, K, V, S> FromPyObject<'_, 'py> for indexmap::IndexMap where - K: FromPyObject<'py> + cmp::Eq + hash::Hash, - V: FromPyObject<'py>, + K: FromPyObjectOwned<'py> + cmp::Eq + hash::Hash, + V: FromPyObjectOwned<'py>, S: hash::BuildHasher + Default, { - fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { - let dict = ob.downcast::()?; + type Error = PyErr; + + fn extract(ob: Borrowed<'_, 'py, PyAny>) -> Result { + let dict = ob.cast::()?; let mut ret = indexmap::IndexMap::with_capacity_and_hasher(dict.len(), S::default()); for (k, v) in dict.iter() { - ret.insert(k.extract()?, v.extract()?); + ret.insert( + k.extract().map_err(Into::into)?, + v.extract().map_err(Into::into)?, + ); } Ok(ret) } @@ -137,21 +154,18 @@ where #[cfg(test)] mod test_indexmap { - use crate::types::any::PyAnyMethods; - use crate::types::dict::PyDictMethods; use crate::types::*; - use crate::{IntoPy, PyObject, Python, ToPyObject}; + use crate::{IntoPyObject, Python}; #[test] - fn test_indexmap_indexmap_to_python() { - Python::with_gil(|py| { + fn test_indexmap_indexmap_into_pyobject() { + Python::attach(|py| { let mut map = indexmap::IndexMap::::new(); map.insert(1, 1); - let m = map.to_object(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = (&map).into_pyobject(py).unwrap(); - assert!(py_map.len() == 1); + assert_eq!(py_map.len(), 1); assert!( py_map .get_item(1) @@ -168,35 +182,13 @@ mod test_indexmap { }); } - #[test] - fn test_indexmap_indexmap_into_python() { - Python::with_gil(|py| { - let mut map = indexmap::IndexMap::::new(); - map.insert(1, 1); - - let m: PyObject = map.into_py(py); - let py_map: &PyDict = m.downcast(py).unwrap(); - - assert!(py_map.len() == 1); - assert!( - py_map - .get_item(1) - .unwrap() - .unwrap() - .extract::() - .unwrap() - == 1 - ); - }); - } - #[test] fn test_indexmap_indexmap_into_dict() { - Python::with_gil(|py| { + Python::attach(|py| { let mut map = indexmap::IndexMap::::new(); map.insert(1, 1); - let py_map = map.into_py_dict_bound(py); + let py_map = map.into_py_dict(py).unwrap(); assert_eq!(py_map.len(), 1); assert_eq!( @@ -213,7 +205,7 @@ mod test_indexmap { #[test] fn test_indexmap_indexmap_insertion_order_round_trip() { - Python::with_gil(|py| { + Python::attach(|py| { let n = 20; let mut map = indexmap::IndexMap::::new(); @@ -225,7 +217,7 @@ mod test_indexmap { } } - let py_map = map.clone().into_py_dict_bound(py); + let py_map = (&map).into_py_dict(py).unwrap(); let trip_map = py_map.extract::>().unwrap(); diff --git a/src/conversions/jiff.rs b/src/conversions/jiff.rs new file mode 100644 index 00000000000..db1a591b968 --- /dev/null +++ b/src/conversions/jiff.rs @@ -0,0 +1,1229 @@ +#![cfg(feature = "jiff-02")] + +//! Conversions to and from [jiff](https://docs.rs/jiff/)’s `Span`, `SignedDuration`, `TimeZone`, +//! `Offset`, `Date`, `Time`, `DateTime`, `Zoned`, and `Timestamp`. +//! +//! # Setup +//! +//! To use this feature, add this to your **`Cargo.toml`**: +//! +//! ```toml +//! [dependencies] +//! jiff = "0.2" +#![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"jiff-02\"] }")] +//! ``` +//! +//! Note that you must use compatible versions of jiff and PyO3. +//! The required jiff version may vary based on the version of PyO3. +//! +//! # Example: Convert a `datetime.datetime` to jiff `Zoned` +//! +//! ```rust +//! # #![cfg_attr(windows, allow(unused_imports))] +//! # use jiff_02 as jiff; +//! use jiff::{Zoned, SignedDuration, ToSpan}; +//! use pyo3::{Python, PyResult, IntoPyObject, types::PyAnyMethods}; +//! +//! # #[cfg(windows)] +//! # fn main() -> () {} +//! # #[cfg(not(windows))] +//! fn main() -> PyResult<()> { +//! Python::initialize(); +//! Python::attach(|py| { +//! // Build some jiff values +//! let jiff_zoned = Zoned::now(); +//! let jiff_span = 1.second(); +//! // Convert them to Python +//! let py_datetime = jiff_zoned.into_pyobject(py)?; +//! let py_timedelta = SignedDuration::try_from(jiff_span)?.into_pyobject(py)?; +//! // Do an operation in Python +//! let py_sum = py_datetime.call_method1("__add__", (py_timedelta,))?; +//! // Convert back to Rust +//! let jiff_sum: Zoned = py_sum.extract()?; +//! println!("Zoned: {}", jiff_sum); +//! Ok(()) +//! }) +//! } +//! ``` +use crate::exceptions::{PyTypeError, PyValueError}; +use crate::pybacked::PyBackedStr; +use crate::types::{PyAnyMethods, PyNone}; +use crate::types::{PyDate, PyDateTime, PyDelta, PyTime, PyTzInfo, PyTzInfoAccess}; +#[cfg(not(Py_LIMITED_API))] +use crate::types::{PyDateAccess, PyDeltaAccess, PyTimeAccess}; +use crate::{intern, Borrowed, Bound, FromPyObject, IntoPyObject, PyAny, PyErr, PyResult, Python}; +use jiff::civil::{Date, DateTime, ISOWeekDate, Time}; +use jiff::tz::{Offset, TimeZone}; +use jiff::{SignedDuration, Span, Timestamp, Zoned}; +#[cfg(feature = "jiff-02")] +use jiff_02 as jiff; + +fn datetime_to_pydatetime<'py>( + py: Python<'py>, + datetime: &DateTime, + fold: bool, + timezone: Option<&TimeZone>, +) -> PyResult> { + PyDateTime::new_with_fold( + py, + datetime.year().into(), + datetime.month().try_into()?, + datetime.day().try_into()?, + datetime.hour().try_into()?, + datetime.minute().try_into()?, + datetime.second().try_into()?, + (datetime.subsec_nanosecond() / 1000).try_into()?, + timezone + .map(|tz| tz.into_pyobject(py)) + .transpose()? + .as_ref(), + fold, + ) +} + +#[cfg(not(Py_LIMITED_API))] +fn pytime_to_time(time: &impl PyTimeAccess) -> PyResult