diff --git a/.circleci/config.yml b/.circleci/config.yml index 62cefcabe..13bcbe069 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,25 +1,43 @@ -version: 2 +version: 2.1 -jobs: - osx-python3.6: - macos: - xcode: 12.5.1 - environment: - PYTHON: python3 +commands: + cibw_prepare_environment: + description: "Prepare the environment for testing." steps: - - checkout - - run: name: Prepare the environment. command: bash .circleci/prepare.sh + cibw_run_tests: + description: "Runs tests, with CIBW_ENABLE=all on the main branch" + steps: - run: - name: Test. - command: venv/bin/python ./bin/run_tests.py + name: Test + command: | + if [ "${CIRCLE_BRANCH}" == "main" ]; then + echo "INFO: Exporting CIBW_ENABLE=all for main branch test run." + export CIBW_ENABLE=all + else + echo "INFO: CIBW_ENABLE not set for this branch test run." + fi + + venv/bin/python ./bin/run_tests.py no_output_timeout: 30m - linux-python3.6: +jobs: + osx-python312: + macos: + xcode: 15.4.0 + resource_class: macos.m1.medium.gen1 + environment: + PYTHON: python3 + steps: + - checkout + - cibw_prepare_environment + - cibw_run_tests + + linux-python312: docker: - - image: circleci/python:3.6 + - image: cimg/python:3.12 environment: PYTHON: python3 # Temporarily restrict the tests that are run on CircleCI to prevent @@ -28,18 +46,27 @@ jobs: steps: - checkout - setup_remote_docker + - cibw_prepare_environment + - cibw_run_tests - - run: - name: Prepare the environment. - command: bash .circleci/prepare.sh - - run: - name: Test. - command: venv/bin/python ./bin/run_tests.py - no_output_timeout: 30m + linux-aarch64: + machine: + image: default + resource_class: arm.medium + environment: + PYTHON: python3 + # Temporarily restrict the tests that are run on CircleCI to prevent + # test timeouts. + PYTEST_ADDOPTS: -k "unit_test or main_tests or test_0_basic or test_docker_images" + steps: + - checkout + - cibw_prepare_environment + - cibw_run_tests workflows: version: 2 all-tests: jobs: - - osx-python3.6 - - linux-python3.6 + - osx-python312 + - linux-python312 + - linux-aarch64 diff --git a/.circleci/prepare.sh b/.circleci/prepare.sh index 991f0202a..6023be5ce 100644 --- a/.circleci/prepare.sh +++ b/.circleci/prepare.sh @@ -2,9 +2,15 @@ set -o errexit set -o xtrace +if [ "$(uname -s)" == "Darwin" ]; then + sudo softwareupdate --install-rosetta --agree-to-license +else + docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all +fi + $PYTHON --version -$PYTHON -m pip --version -$PYTHON -m virtualenv -p "$PYTHON" venv -venv/bin/python -m pip install -e ".[dev]" +$PYTHON -m venv venv +venv/bin/python -m pip install -U pip +venv/bin/python -m pip install -e. --group test venv/bin/python -m pip freeze venv/bin/python --version diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 000000000..02cc023bb --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,91 @@ +only_if: changesInclude('.cirrus.yml') || ($CIRRUS_BRANCH == "main" && !changesIncludeOnly('.github/*', 'bin/*', 'docs/*', '.circleci/*', '.travis.yml', '.pre-commit-config.yaml', '.readthedocs.yml', 'azure-pipelines.yml', 'README.md', 'mkdocs.yml', 'noxfile.py')) || $CIRRUS_BRANCH =~ 'cirrus.*' + +run_tests: &RUN_TESTS + install_cibuildwheel_script: + - python -m pip install -U pip + - python -m pip install -e. --group test + run_cibuildwheel_tests_script: + - python ./bin/run_tests.py + +linux_x86_task: + timeout_in: 120m + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 8G + env: + VENV_ROOT: ${HOME}/venv-cibuildwheel + PATH: ${VENV_ROOT}/bin:${PATH} + install_pre_requirements_script: + - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all + - add-apt-repository -y ppa:deadsnakes/ppa + - apt-get update + - apt-get install -y python3.12-venv + - python3.12 -m venv ${VENV_ROOT} + <<: *RUN_TESTS + +linux_aarch64_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 4 + memory: 4G + env: + VENV_ROOT: ${HOME}/venv-cibuildwheel + PATH: ${VENV_ROOT}/bin:${PATH} + install_pre_requirements_script: + - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all + - add-apt-repository -y ppa:deadsnakes/ppa + - apt-get update + - apt-get install -y python3.12-venv + - python3.12 -m venv ${VENV_ROOT} + <<: *RUN_TESTS + +windows_x86_task: + # The task takes ~55 minutes while the timeout happens + # after 60 minutes by default, let's allow some wiggle room. + timeout_in: 120m + windows_container: + image: cirrusci/windowsservercore:visualstudio2022 + cpu: 8 + memory: 8G + + install_pre_requirements_script: + - certutil -generateSSTFromWU roots.sst + - certutil -addstore -f root roots.sst + - del roots.sst + - choco install -y --no-progress python3 --version 3.12.4 + - refreshenv + - powershell -Command "$cleanPath = $env:PATH -replace ';+$', ''; Add-Content -Path $env:CIRRUS_ENV -Value ('PATH=' + $cleanPath)" + <<: *RUN_TESTS + +macos_arm64_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-runner:sequoia + env: + VENV_ROOT: ${HOME}/venv-cibuildwheel + PATH: ${VENV_ROOT}/bin:${PATH} + install_pre_requirements_script: + - brew install python@3.12 + - python3.12 -m venv ${VENV_ROOT} + <<: *RUN_TESTS + +macos_arm64_cp38_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-runner:sequoia + env: + VENV_ROOT: ${HOME}/venv-cibuildwheel + PATH: ${VENV_ROOT}/bin:${PATH} + PYTEST_ADDOPTS: --run-cp38-universal2 -k 'test_cp38_arm64_testing_universal2_installer or test_arch_auto or test_dummy_serial' + install_pre_requirements_script: + - brew install python@3.12 + - python3.12 -m venv ${VENV_ROOT} + - curl -fsSLO https://www.python.org/ftp/python/3.8.10/python-3.8.10-macos11.pkg + - sudo installer -pkg python-3.8.10-macos11.pkg -target / + - rm python-3.8.10-macos11.pkg + - sh "/Applications/Python 3.8/Install Certificates.command" + <<: *RUN_TESTS diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..53363ffcd --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +* text=auto + +*.py diff=python +*.md diff=markdown + +*.svg -diff diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ce05ac5c9..6c4b36953 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,7 @@ updates: directory: "/" schedule: interval: "weekly" - ignore: - - dependency-name: "actions/*" - update-types: ["version-update:semver-minor", "version-update:semver-patch"] + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5d79d9d4b..34e3aacea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,30 +11,32 @@ jobs: dist: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - name: Build SDist and wheel - run: pipx run build - - - uses: actions/upload-artifact@v2 - with: - path: dist/* - - - name: Check metadata - run: pipx run twine check dist/* + - uses: hynek/build-and-inspect-python-package@v2 publish: needs: [dist] runs-on: ubuntu-latest if: github.event_name == 'release' && github.event.action == 'published' + environment: + name: pypi + url: https://pypi.org/p/cibuildwheel + permissions: + id-token: write + attestations: write steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: - name: artifact + name: Packages path: dist - - uses: pypa/gh-action-pypi-publish@v1.5.0 + - name: Generate artifact attestation for sdist and wheel + uses: actions/attest-build-provenance@db473fddc028af60658334401dc6fa3ffd8669fd # v2.3.0 + with: + subject-path: "dist/cibuildwheel-*" + + - uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: ${{ secrets.pypi_password }} + attestations: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0dcd29cf0..edc55f027 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,9 +4,27 @@ on: push: branches: - main + - 2.x pull_request: + types: + - opened + - synchronize + - reopened + - labeled paths-ignore: - - 'docs/**' + - .ci* + - bin/* + - docs/** + - examples/azure-pipelines-* + - examples/ci* + - examples/travis-ci-* + - .pre-commit-config.yaml + - .readthedocs.yml + - .travis.yml + - README.md + - azure-pipelines.yml + - mkdocs.yml + - noxfile.py workflow_dispatch: # allow manual runs on branches without a PR @@ -15,74 +33,221 @@ concurrency: cancel-in-progress: true jobs: - pre-commit: - name: Pre-commit checks (mypy, flake8, etc.) + lint: + name: Linters (mypy, flake8, etc.) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.3 - - run: pipx run nox -s check_manifest + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + id: python + with: + python-version: "3.x" + - uses: pre-commit/action@v3.0.1 + - name: PyLint checks + run: pipx run --python "${{ steps.python.outputs.python-path }}" nox -s pylint -- --output-format=github test: - name: Test cibuildwheel on ${{ matrix.os }} - needs: pre-commit + name: Test on ${{ matrix.os }} (${{ matrix.python_version }}) + needs: lint runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-11] - python_version: ['3.10'] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-15] + python_version: ['3.13'] + include: + - os: ubuntu-latest + python_version: '3.11' + - os: ubuntu-latest + python_version: '3.14' timeout-minutes: 180 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Install Python ${{ matrix.python_version }} with: python-version: ${{ matrix.python_version }} + allow-prereleases: true + + - uses: astral-sh/setup-uv@v6 + + # free some space to prevent reaching GHA disk space limits + - name: Clean docker images + if: runner.os == 'Linux' + run: | + docker system prune -a -f + df -h + + # for oci_container unit tests + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 - name: Install dependencies run: | - python -m pip install ".[test]" + uv sync --no-dev --group test - - name: Sample build - if: "contains(github.event.pull_request.labels.*.name, 'CI: Sample build')" + - uses: joerick/pr-labels-action@v1.0.9 + - name: Set CIBW_ENABLE + shell: bash run: | - python bin/sample_build.py + if [[ "${{ github.ref_name }}" == "main" ]]; then + CIBW_ENABLE=all + else + # get the default CIBW_ENABLE value from the test module + CIBW_ENABLE=$(uv run --no-sync python -c 'import sys, test.utils as c; sys.stdout.write(c.DEFAULT_CIBW_ENABLE)') - - name: Get some sample wheels + # if this is a PR, check for labels + if [[ -n "$GITHUB_PR_LABEL_CI_PYPY" ]]; then + CIBW_ENABLE+=" pypy" + fi + if [[ -n "$GITHUB_PR_LABEL_CI_GRAALPY" ]]; then + CIBW_ENABLE+=" graalpy" + fi + fi + echo "CIBW_ENABLE=${CIBW_ENABLE}" >> $GITHUB_ENV + + - name: Generate a sample project run: | - python -m test.test_projects test.test_0_basic.basic_project sample_proj - cibuildwheel --output-dir wheelhouse sample_proj + uv run --no-sync -m test.test_projects test.test_0_basic.basic_project sample_proj + + - name: Run a sample build (GitHub Action) + uses: ./ + with: + package-dir: sample_proj + output-dir: wheelhouse env: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + CIBW_BUILD_FRONTEND: 'build[uv]' + + - name: Run a sample build (GitHub Action, only) + uses: ./ + with: + package-dir: sample_proj + output-dir: wheelhouse_only + only: cp312-${{ runner.os == 'Linux' && (runner.arch == 'ARM64' && 'manylinux_aarch64' || 'manylinux_x86_64') || (runner.os == 'Windows' && 'win_amd64' || 'macosx_x86_64') }} - - uses: actions/upload-artifact@v2 + - name: Create custom configuration file + shell: bash + run: | + cat > sample_proj/cibw.toml <> "$GITHUB_OUTPUT" test-emulated: - name: Test emulated cibuildwheel using qemu - needs: pre-commit + name: Test Linux ${{ matrix.arch }} using qemu + needs: emulated-archs runs-on: ubuntu-latest timeout-minutes: 180 + strategy: + matrix: + arch: ${{ fromJSON(needs.emulated-archs.outputs.archs) }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + - uses: astral-sh/setup-uv@v6 - name: Install dependencies - run: | - python -m pip install ".[test]" + run: uv sync --no-dev --group test - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v1 - with: - platforms: all + uses: docker/setup-qemu-action@v3 - name: Run the emulation tests + run: uv run --no-sync pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py + + test-pyodide: + name: Test pyodide + needs: lint + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + name: Install Python 3.12 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@v6 + + - name: Install dependencies + run: uv sync --no-dev --group test + + - name: Generate a sample project run: | - pytest --run-emulation test/test_emulation.py + uv run --no-sync -m test.test_projects test.test_0_basic.basic_project sample_proj + + - name: Run a sample build (GitHub Action) + uses: ./ + with: + package-dir: sample_proj + output-dir: wheelhouse + env: + CIBW_PLATFORM: pyodide + + - name: Run a sample build (GitHub Action) for an overridden Pyodide version + uses: ./ + with: + package-dir: sample_proj + output-dir: wheelhouse + # In case this breaks at any point in time, switch to using the latest version + # available or any other version that is not the same as the default one set + # in cibuildwheel/resources/build-platforms.toml. + env: + CIBW_PLATFORM: pyodide + CIBW_BUILD: "cp312*" + CIBW_PYODIDE_VERSION: "0.27.6" + + - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' + run: | + uv run --no-sync ./bin/run_tests.py + env: + CIBW_PLATFORM: pyodide diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index bbce092e9..2f856d47e 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -1,6 +1,15 @@ name: Update dependencies on: + pull_request: + paths: + - '.github/workflows/update-dependencies.yml' + - 'bin/update_pythons.py' + - 'bin/update_docker.py' + - 'bin/update_virtualenv.py' + - 'bin/projects.py' + - 'docs/data/projects.yml' + - 'noxfile.py' workflow_dispatch: schedule: - cron: '0 6 * * 1' # "At 06:00 on Monday." @@ -11,8 +20,19 @@ jobs: if: github.repository_owner == 'pypa' || github.event_name != 'schedule' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: excitedleigh/setup-nox@v2.0.0 + + # we use this step to grab a Github App auth token, so that PRs generated by this workflow + # run the GHA tests. + - uses: actions/create-github-app-token@v2 + id: generate-token + if: github.ref == 'refs/heads/main' && github.repository == 'pypa/cibuildwheel' + with: + app-id: ${{ secrets.CIBUILDWHEEL_BOT_APP_ID }} + private-key: ${{ secrets.CIBUILDWHEEL_BOT_APP_PRIVATE_KEY }} + + - uses: actions/checkout@v4 + + - uses: wntrblm/nox@2025.05.01 - name: "Run update: dependencies" run: nox --force-color -s update_constraints @@ -21,18 +41,9 @@ jobs: - name: "Run update: docs user projects" run: nox --force-color -s update_proj -- --auth=${{ secrets.GITHUB_TOKEN }} - # we use this step to grab a Github App auth token, so that PRs generated by this workflow - # run the GHA tests. - - uses: tibdex/github-app-token@v1 - id: generate-token - if: github.ref == 'refs/heads/main' && github.repository == 'pypa/cibuildwheel' - with: - app_id: ${{ secrets.CIBUILDWHEEL_BOT_APP_ID }} - private_key: ${{ secrets.CIBUILDWHEEL_BOT_APP_PRIVATE_KEY }} - - name: Create Pull Request if: github.ref == 'refs/heads/main' && github.repository == 'pypa/cibuildwheel' - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v7 with: commit-message: Update dependencies title: '[Bot] Update dependencies' @@ -41,5 +52,10 @@ jobs: PR generated by "Update dependencies" [workflow](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}). branch: update-dependencies-pr + sign-commits: true token: ${{ steps.generate-token.outputs.token }} delete-branch: true + labels: | + CI: GraalPy + CI: PyPy + dependencies diff --git a/.github/workflows/update-major-minor-tag.yml b/.github/workflows/update-major-minor-tag.yml new file mode 100644 index 000000000..9958811bc --- /dev/null +++ b/.github/workflows/update-major-minor-tag.yml @@ -0,0 +1,30 @@ +name: Update the vX.Y tag + +on: + release: + types: [released] + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Tag name that the major.minor tag will point to' + required: true + +env: + TAG_NAME: ${{ github.event.inputs.TAG_NAME || github.event.release.tag_name }} + +jobs: + update_tag: + name: Update the major.minor tag to include the ${{ github.event.inputs.TAG_NAME || github.event.release.tag_name }} changes + environment: + name: releaseNewActionVersion + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Update the ${{ env.TAG_NAME }} tag + id: update-major-minor-tag + uses: joerick/update-vX.Y-tag-action@v1.0 + with: + source-tag: ${{ env.TAG_NAME }} diff --git a/.gitignore b/.gitignore index 98559d4f0..d61e97835 100644 --- a/.gitignore +++ b/.gitignore @@ -84,9 +84,7 @@ celerybeat-schedule # virtualenv .venv -venv/ -venv3/ -venv2/ +venv*/ ENV/ env/ env2/ @@ -112,8 +110,5 @@ all_known_setup.yaml # mkdocs site/ -# Virtual environments -venv* - # PyCharm .idea/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d6cd1763..a5a7cf483 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ linux: - image: python:3.8 + image: python:3.12 services: - name: docker:dind entrypoint: ["env", "-u", "DOCKER_HOST"] @@ -9,7 +9,49 @@ linux: DOCKER_DRIVER: overlay2 # See https://github.com/docker-library/docker/pull/166 DOCKER_TLS_CERTDIR: "" + + # skip all but the basic tests + # (comment the below line in a PR to debug a Gitlab-specific issue) + PYTEST_ADDOPTS: -k "unit_test or test_0_basic" --suppress-no-test-exit-code + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + variables: + CIBW_ENABLE: "all" script: - curl -sSL https://get.docker.com/ | sh - - python -m pip install -e ".[dev]" + - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all + - python -m pip install -U pip + - python -m pip install -e. pytest-custom-exit-code --group test - python ./bin/run_tests.py + +windows: + image: mcr.microsoft.com/windows/servercore:1809 + variables: + PYTEST_ADDOPTS: -k "unit_test or test_0_basic" --suppress-no-test-exit-code + before_script: + - choco install python -y --allow-downgrade --version 3.12.4 + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + variables: + CIBW_ENABLE: "all" + script: + - py -m pip install -U pip + - py -m pip install -e. pytest-custom-exit-code --group test + - py bin\run_tests.py + tags: + - saas-windows-medium-amd64 + +macos: + image: macos-14-xcode-15 + variables: + PYTEST_ADDOPTS: -k "unit_test or test_0_basic" --suppress-no-test-exit-code + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + variables: + CIBW_ENABLE: "all" + script: + - python3 -m pip install -U pip + - python3 -m pip install -e. pytest-custom-exit-code --group test + - python3 ./bin/run_tests.py + tags: + - saas-macos-medium-m1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c8279f28c..94bf4a707 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v5.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -13,92 +13,45 @@ repos: - id: mixed-line-ending - id: trailing-whitespace -- repo: https://github.com/asottile/pyupgrade - rev: v2.31.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.13 hooks: - - id: pyupgrade - name: PyUpgrade 3.6+ - args: ["--py36-plus"] - exclude: ^bin/ - - id: pyupgrade - name: PyUpgrade 3.7+ on bin - exclude: ^(cibuildwheel|unit_test|test)/ - args: ["--py37-plus"] - -# Autoremoves unused imports -- repo: https://github.com/hadialqattan/pycln - rev: v1.1.0 - hooks: - - id: pycln - args: [--config=pyproject.toml] - -- repo: https://github.com/PyCQA/isort - rev: 5.10.1 - hooks: - - id: isort - -- repo: https://github.com/psf/black - rev: 21.12b0 - hooks: - - id: black - -- repo: https://github.com/asottile/setup-cfg-fmt - rev: v1.20.0 - hooks: - - id: setup-cfg-fmt + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.931 + rev: v1.16.0 hooks: - id: mypy - exclude: ^(bin|cibuildwheel/resources|docs)/.*py$ - args: ["--python-version=3.6", "--scripts-are-modules", "--show-error-codes"] + name: mypy 3.11 on cibuildwheel/ + args: ["--python-version=3.11"] additional_dependencies: &mypy-dependencies - - nox - - packaging>=21.0 + - bracex + - dependency-groups>=1.2 + - nox>=2025.2.9 + - orjson + - packaging - pygithub + - pytest - rich - - tomli + - tomli_w - types-certifi - types-click - - types-dataclasses - types-jinja2 - types-pyyaml - types-requests - - bracex + - types-setuptools + - uv + - validate-pyproject - id: mypy - name: mypy 3.7+ on bin/ - files: ^((bin|docs)/.*py)$ - args: ["--python-version=3.7", "--scripts-are-modules", "--show-error-codes"] + name: mypy 3.13 + exclude: ^cibuildwheel/resources/.*py$ + args: ["--python-version=3.13"] additional_dependencies: *mypy-dependencies -- repo: https://github.com/asottile/yesqa - rev: v1.3.0 - hooks: - - id: yesqa - additional_dependencies: &flake8-dependencies - - flake8-bugbear - -- repo: https://github.com/PyCQA/flake8 - rev: 4.0.1 - hooks: - - id: flake8 - exclude: cibuildwheel/resources/ - additional_dependencies: *flake8-dependencies - -- repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 - hooks: - - id: python-check-blanket-noqa - stages: [manual] - - id: python-check-blanket-type-ignore - stages: [manual] - - id: python-no-log-warn - - id: python-no-eval - - id: python-use-type-annotations - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.8.0.3 + rev: v0.10.0.1 hooks: - id: shellcheck @@ -108,16 +61,37 @@ repos: name: Disallow improper capitalization language: pygrep entry: PyBind|Numpy|Cmake|Github|PyTest - types: - - markdown - - id: update-readme-changelog - name: Update README changelog + types: [markdown] + exclude: ^docs/working-examples\.md$ # Autogenerated + - id: cog + name: Cog the README language: python - entry: bin/update_readme_changelog.py - files: ^docs/changelog.md$ + pass_filenames: false + entry: cog -c -P -r -I ./bin README.md + files: '^(README\.md|docs/changelog\.md|docs/options\.md|bin/readme.*)$' + additional_dependencies: [cogapp>=3.5] - repo: https://github.com/codespell-project/codespell - rev: v2.1.0 + rev: v2.4.1 hooks: - id: codespell - args: ["-L", "sur"] + args: ["-w"] + exclude: ^docs/working-examples\.md$ # Autogenerated + + +- repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.33.0 + hooks: + - id: check-dependabot + - id: check-github-actions + - id: check-github-workflows + - id: check-gitlab-ci + # Disabled due to schema issue for now: + # - id: check-readthedocs + - id: check-travis + - id: check-jsonschema + name: Check projects + args: [--schemafile, docs/data/projects.schema.json] + files: '^docs/data/projects.yml$' + - id: check-metaschema + files: '^docs/data/projects.schema.json$' diff --git a/.readthedocs.yml b/.readthedocs.yml index b59f4199f..70d3153a3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,13 +2,10 @@ # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details version: 2 -mkdocs: - configuration: mkdocs.yml - -python: - version: 3.8 - install: - - method: pip - path: . - extra_requirements: - - docs +build: + os: ubuntu-24.04 + commands: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + - NO_COLOR=1 uv run --python 3.14 --managed-python --no-dev --group docs mkdocs build --strict --site-dir $READTHEDOCS_OUTPUT/html diff --git a/.travis.yml b/.travis.yml index 7115e5d19..7b3160a18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,28 +1,34 @@ os: linux -dist: focal +dist: jammy language: python branches: only: - main +# only build the main branch or PR that explicitely want to test with Travis CI +if: (type = "push") OR (commit_message =~ /travis-ci/) + jobs: include: - - name: Linux | x86_64 + i686 | Python 3.6 - python: 3.6 + - name: Linux | x86_64 + i686 | Python 3.12 + python: 3.12 services: docker - env: PYTHON=python + env: + - PYTHON=python + # a build using CIBW_ENABLE=all does not fit into Travis' time limit, + # so only the defaults are tested - - name: Linux | arm64 | Python 3.6 - python: 3.6 + - name: Linux | arm64 | Python 3.12 + python: 3.12 services: docker - arch: arm64-graviton2 - group: edge - virt: vm - env: PYTHON=python + arch: arm64 + env: + - PYTHON=python + - CIBW_ENABLE=all - - name: Linux | ppc64le | Python 3.6 - python: 3.6 + - name: Linux | ppc64le | Python 3.12 + python: 3.12 services: docker arch: ppc64le allow_failure: True @@ -31,25 +37,43 @@ jobs: # skip test_manylinuxXXXX_only, it uses too much disk space # c.f. https://travis-ci.community/t/running-out-of-disk-space-quota-when-using-docker-on-ppc64le/11634 - PYTEST_ADDOPTS='-k "not test_manylinuxXXXX_only"' + - CIBW_ENABLE=all - - name: Windows | x86_64 | Python 3.6 + - name: Windows | x86_64 | Python 3.12 os: windows language: shell before_install: - - choco upgrade python3 -y --version 3.6.8 --limit-output - # Update root certificates to fix SSL error; see http://www.chawn.com/RootCerts.htm - - powershell "md C:\temp\certs; CertUtil -generateSSTFromWU C:\temp\certs\RootStore.sst; Get-ChildItem -Path C:\\temp\certs\Rootstore.sst | Import-Certificate -CertStoreLocation Cert:\\LocalMachine\\Root\\ | out-null" + # http://woshub.com/updating-trusted-root-certificates-in-windows-10 + - certutil -generateSSTFromWU roots.sst + - powershell -Command 'Get-ChildItem -Path roots.sst | Import-Certificate -CertStoreLocation Cert:\LocalMachine\Root' + - rm -f roots.sst + - choco upgrade python3 -y --version 3.12.8 --limit-output --params "/InstallDir:C:\\Python312" env: - - PYTHON=C:\\Python36\\python + - PYTHON=C:\\Python312\\python + # a build using CIBW_ENABLE=all does not fit into Travis' time limit, + # so only the defaults are tested - - name: Linux | s390x | Python 3.6 - python: 3.6 + - name: Linux | s390x | Python 3.12 + python: 3.12 services: docker arch: s390x - env: PYTHON=python + allow_failure: True + env: + - PYTHON=python + - CIBW_ENABLE=all install: +- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all; fi - $PYTHON -m pip install -U pip -- $PYTHON -m pip install -e ".[dev]" pytest-custom-exit-code +- $PYTHON -m pip install -e. --group test -script: $PYTHON ./bin/run_tests.py +script: | + # travis_wait disable the output while waiting + # use the same kind of tricks as in multibuild + (while true; do echo "travis_keep_alive"; sleep 300; done) & + SPINNER_PID=$! + disown + result=0 + $PYTHON ./bin/run_tests.py --num-processes 2 || result=1 + kill -9 ${SPINNER_PID} + test ${result} -eq 0 diff --git a/CI.md b/CI.md index 8ac4ada58..6b45fffc3 100644 --- a/CI.md +++ b/CI.md @@ -1,11 +1,11 @@ -This is a summary of the Python versions and platforms covered by the different CI platforms: +This is a summary of the host Python versions and platforms covered by the different CI platforms: -| | 3.6 | 3.7 | 3.8 | -|----------|------------------------------|-----------------------------|------------------| -| Linux | Travis CI / CircleCI | AppVeyor¹ / GitHub Actions | Azure Pipelines | -| macOS | CircleCI | AppVeyor¹ / GitHub Actions | Azure Pipelines | -| Windows | Travis CI / Azure Pipelines | AppVeyor¹ / GitHub Actions | Azure Pipelines | +| | 3.11 | 3.12 | 3.13 | +|---------|----------------------------------|---------------------------------------------|----------------| +| Linux | Azure Pipelines / GitHub Actions | CircleCI¹ / Cirrus CI / GitLab¹ / Travis CI | GitHub Actions | +| macOS | Azure Pipelines | CircleCI¹ / Cirrus CI / GitLab¹ | GitHub Actions | +| Windows | Azure Pipelines | Cirrus CI / GitLab¹ / Travis CI | GitHub Actions | -> ¹ AppVeyor only runs the "basic" test to reduce load. +> ¹ Runs a reduced set of tests to reduce CI load -Non-x86 architectures are covered on Travis CI using Python 3.6. +Non-x86 architectures are covered on Travis CI using Python 3.12. diff --git a/LICENSE b/LICENSE index e9b2ceeee..7aefbf192 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ This project is licensed under the 'BSD 2-clause license'. -Copyright (c) 2017-2019, Joe Rickerby and contributors. All rights reserved. +Copyright (c) 2017-2025, Joe Rickerby and contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 6fc0b3942..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,6 +0,0 @@ -recursive-include cibuildwheel *.cfg -recursive-include cibuildwheel *.in -recursive-include cibuildwheel *.py -recursive-include cibuildwheel *.toml -recursive-include cibuildwheel *.txt -recursive-include cibuildwheel *.typed diff --git a/README.md b/README.md index 41cb62037..0abcf8f9e 100644 --- a/README.md +++ b/README.md @@ -2,62 +2,72 @@ cibuildwheel ============ [![PyPI](https://img.shields.io/pypi/v/cibuildwheel.svg)](https://pypi.python.org/pypi/cibuildwheel) -[![Documentation Status](https://readthedocs.org/projects/cibuildwheel/badge/?version=stable)](https://cibuildwheel.readthedocs.io/en/stable/?badge=stable) +[![Documentation Status](https://readthedocs.org/projects/cibuildwheel/badge/?version=stable)](https://cibuildwheel.pypa.io/en/stable/?badge=stable) [![Actions Status](https://github.com/pypa/cibuildwheel/workflows/Test/badge.svg)](https://github.com/pypa/cibuildwheel/actions) -[![Travis Status](https://img.shields.io/travis/com/pypa/cibuildwheel/main?logo=travis)](https://travis-ci.com/pypa/cibuildwheel) -[![Appveyor status](https://ci.appveyor.com/api/projects/status/gt3vwl88yt0y3hur/branch/main?svg=true)](https://ci.appveyor.com/project/joerick/cibuildwheel/branch/main) +[![Travis Status](https://img.shields.io/travis/com/pypa/cibuildwheel/main?logo=travis)](https://travis-ci.com/github/pypa/cibuildwheel) [![CircleCI Status](https://img.shields.io/circleci/build/gh/pypa/cibuildwheel/main?logo=circleci)](https://circleci.com/gh/pypa/cibuildwheel) [![Azure Status](https://dev.azure.com/joerick0429/cibuildwheel/_apis/build/status/pypa.cibuildwheel?branchName=main)](https://dev.azure.com/joerick0429/cibuildwheel/_build/latest?definitionId=4&branchName=main) -[Documentation](https://cibuildwheel.readthedocs.org) +[Documentation](https://cibuildwheel.pypa.io) Python wheels are great. Building them across **Mac, Linux, Windows**, on **multiple versions of Python**, is not. -`cibuildwheel` is here to help. `cibuildwheel` runs on your CI server - currently it supports GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, and GitLab CI - and it builds and tests your wheels across all of your platforms. +`cibuildwheel` is here to help. `cibuildwheel` runs on your CI server - currently it supports GitHub Actions, Azure Pipelines, Travis CI, CircleCI, and GitLab CI - and it builds and tests your wheels across all of your platforms. What does it do? ---------------- -| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | Windows Arm64 | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | -|---------------|----|-----|-----|-----|-----|----|-----|----|-----|-----| -| CPython 3.6 | ✅ | N/A | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.7 | ✅ | N/A | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅³ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | -| PyPy 3.7 v7.3 | ✅ | N/A | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | -| PyPy 3.8 v7.3 | ✅ | N/A | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | - -¹ PyPy is only supported for manylinux wheels.
+While cibuildwheel itself requires a recent Python version to run (we support the last three releases), it can target the following versions to build wheels: + +| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | Windows Arm64 | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | manylinux
musllinux armv7l | iOS | Pyodide | +|--------------------|----|-----|-----|-----|-----|----|-----|----|-----|-----|---|-----|-----| +| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | +| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | +| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | +| CPython 3.11 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | +| CPython 3.12 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | ✅⁴ | +| CPython 3.13³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | ✅ | N/A | +| CPython 3.14³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | ✅ | N/A | +| PyPy 3.8 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | +| PyPy 3.9 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | +| PyPy 3.10 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | +| PyPy 3.11 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | +| GraalPy 3.11 v24.2 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | N/A | ✅¹ | N/A | N/A | N/A | N/A | N/A | + +¹ PyPy & GraalPy are only supported for manylinux wheels.
² Windows arm64 support is experimental.
-³ Alpine 3.14 and very briefly 3.15's default python3 [was not able to load](https://github.com/pypa/cibuildwheel/issues/934) musllinux wheels. This has been fixed; please upgrade the python package if using Alpine from before the fix.
+³ Free-threaded mode requires opt-in using [`enable`](https://cibuildwheel.pypa.io/en/stable/options/#enable).
+⁴ Experimental, not yet supported on PyPI, but can be used directly in web deployment. Use `--platform pyodide` to build.
+⁵ manylinux armv7l support is experimental. As there are no RHEL based image for this architecture, it's using an Ubuntu based image instead.
-- Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy -- Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, and GitLab CI +- Builds manylinux, musllinux, macOS 10.9+ (10.13+ for Python 3.12+), and Windows wheels for CPython, PyPy, and GraalPy +- Works on GitHub Actions, Azure Pipelines, Travis CI, CircleCI, GitLab CI, and Cirrus CI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) - Runs your library's tests against the wheel-installed version of your library -See the [cibuildwheel 1 documentation](https://cibuildwheel.readthedocs.io/en/1.x/) if you need to build unsupported versions of Python, such as Python 2. +See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) if you need to build unsupported versions of Python, such as Python 2. Usage ----- `cibuildwheel` runs inside a CI service. Supported platforms depend on which service you're using: -| | Linux | macOS | Windows | Linux ARM | -|-----------------|-------|-------|---------|--------------| -| GitHub Actions | ✅ | ✅ | ✅ | ✅¹ | -| Azure Pipelines | ✅ | ✅ | ✅ | | -| Travis CI | ✅ | | ✅ | ✅ | -| AppVeyor | ✅ | ✅ | ✅ | | -| CircleCI | ✅ | ✅ | | | -| Gitlab CI | ✅ | | | | +| | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | iOS | +|-----------------|-------|-------|---------|-----------|-----------|-------------|-----| +| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅³ | +| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅³ | +| Travis CI | ✅ | | ✅ | ✅ | | | | +| CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅³ | +| Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅³ | +| Cirrus CI | ✅ | ✅ | ✅ | ✅ | ✅ | | | -¹ [Requires emulation](https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
+¹ [Requires emulation](https://cibuildwheel.pypa.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
+² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
+³ Requires a macOS runner; runs tests on the simulator for the runner's architecture. @@ -66,6 +76,7 @@ Example setup To build manylinux, musllinux, macOS, and Windows wheels on GitHub Actions, you could use this `.github/workflows/wheels.yml`: + ```yaml name: Build @@ -77,56 +88,83 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macOS-10.15] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Used to host cibuildwheel - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.3.1 + run: python -m pip install cibuildwheel==3.0.0 - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse # to supply options, put them in 'env', like: # env: # CIBW_SOME_OPTION: value + # ... - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl ``` + -For more information, including PyPI deployment, and the use of other CI services or the dedicated GitHub Action, check out the [documentation](https://cibuildwheel.readthedocs.org) and the [examples](https://github.com/pypa/cibuildwheel/tree/main/examples). +For more information, including PyPI deployment, and the use of other CI services or the dedicated GitHub Action, check out the [documentation](https://cibuildwheel.pypa.io) and the [examples](https://github.com/pypa/cibuildwheel/tree/main/examples). -Options -------- +How it works +------------ + +The following diagram summarises the steps that cibuildwheel takes on each platform. + +![](docs/data/how-it-works.png) + +Explore an interactive version of this diagram [in the docs](https://cibuildwheel.pypa.io/en/stable/#how-it-works). + + + + + | | Option | Description | -|---|--------|-------------| -| **Build selection** | [`CIBW_PLATFORM`](https://cibuildwheel.readthedocs.io/en/stable/options/#platform) | Override the auto-detected target platform | -| | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)
[`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) | Choose the Python versions to build | -| | [`CIBW_ARCHS`](https://cibuildwheel.readthedocs.io/en/stable/options/#archs) | Change the architectures built on your machine by default. | -| | [`CIBW_PROJECT_REQUIRES_PYTHON`](https://cibuildwheel.readthedocs.io/en/stable/options/#requires-python) | Manually set the Python compatibility of your project | -| | [`CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons) | Enable building with pre-release versions of Python if available | -| **Build customization** | [`CIBW_BUILD_FRONTEND`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-frontend) | Set the tool to use to build, either "pip" (default for now) or "build" | -| | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build | -| | [`CIBW_ENVIRONMENT_PASS_LINUX`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment-pass) | Set environment variables on the host to pass-through to the container during the build. | -| | [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. | -| | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | -| | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel | -| | [`CIBW_MANYLINUX_*_IMAGE`
`CIBW_MUSLLINUX_*_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#linux-image) | Specify alternative manylinux / musllinux Docker images | -| | [`CIBW_DEPENDENCY_VERSIONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#dependency-versions) | Specify how cibuildwheel controls the versions of the tools it uses | -| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel | -| | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute a shell command before testing each wheel | -| | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | -| | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras) | Install your wheel for testing using extras_require | -| | [`CIBW_TEST_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-skip) | Skip running tests on some builds | -| **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-verbosity) | Increase/decrease the output of pip wheel | - -These options can be specified in a pyproject.toml file, as well; see [configuration](https://cibuildwheel.readthedocs.io/en/stable/options/#configuration). +|---|---|---| +| **Build selection** | [`platform`](https://cibuildwheel.pypa.io/en/stable/options/#platform) | Override the auto-detected target platform | +| | [`build`
`skip`](https://cibuildwheel.pypa.io/en/stable/options/#build-skip) | Choose the Python versions to build | +| | [`archs`](https://cibuildwheel.pypa.io/en/stable/options/#archs) | Change the architectures built on your machine by default. | +| | [`project-requires-python`](https://cibuildwheel.pypa.io/en/stable/options/#requires-python) | Manually set the Python compatibility of your project | +| | [`enable`](https://cibuildwheel.pypa.io/en/stable/options/#enable) | Enable building with extra categories of selectors present. | +| | [`allow-empty`](https://cibuildwheel.pypa.io/en/stable/options/#allow-empty) | Suppress the error code if no wheels match the specified build identifiers | +| **Build customization** | [`build-frontend`](https://cibuildwheel.pypa.io/en/stable/options/#build-frontend) | Set the tool to use to build, either "build" (default), "build\[uv\]", or "pip" | +| | [`config-settings`](https://cibuildwheel.pypa.io/en/stable/options/#config-settings) | Specify config-settings for the build backend. | +| | [`environment`](https://cibuildwheel.pypa.io/en/stable/options/#environment) | Set environment variables | +| | [`environment-pass`](https://cibuildwheel.pypa.io/en/stable/options/#environment-pass) | Set environment variables on the host to pass-through to the container. | +| | [`before-all`](https://cibuildwheel.pypa.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. | +| | [`before-build`](https://cibuildwheel.pypa.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | +| | [`xbuild-tools`](https://cibuildwheel.pypa.io/en/stable/options/#xbuild-tools) | Binaries on the path that should be included in an isolated cross-build environment. | +| | [`repair-wheel-command`](https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each built wheel | +| | [`manylinux-*-image`
`musllinux-*-image`](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) | Specify manylinux / musllinux container images | +| | [`container-engine`](https://cibuildwheel.pypa.io/en/stable/options/#container-engine) | Specify the container engine to use when building Linux wheels | +| | [`dependency-versions`](https://cibuildwheel.pypa.io/en/stable/options/#dependency-versions) | Control the versions of the tools cibuildwheel uses | +| | [`pyodide-version`](https://cibuildwheel.pypa.io/en/stable/options/#pyodide-version) | Specify the Pyodide version to use for `pyodide` platform builds | +| **Testing** | [`test-command`](https://cibuildwheel.pypa.io/en/stable/options/#test-command) | The command to test each built wheel | +| | [`before-test`](https://cibuildwheel.pypa.io/en/stable/options/#before-test) | Execute a shell command before testing each wheel | +| | [`test-sources`](https://cibuildwheel.pypa.io/en/stable/options/#test-sources) | Files and folders from the source tree that are copied into an isolated tree before running the tests | +| | [`test-requires`](https://cibuildwheel.pypa.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | +| | [`test-extras`](https://cibuildwheel.pypa.io/en/stable/options/#test-extras) | Install your wheel for testing using `extras_require` | +| | [`test-groups`](https://cibuildwheel.pypa.io/en/stable/options/#test-groups) | Specify test dependencies from your project's `dependency-groups` | +| | [`test-skip`](https://cibuildwheel.pypa.io/en/stable/options/#test-skip) | Skip running tests on some builds | +| | [`test-environment`](https://cibuildwheel.pypa.io/en/stable/options/#test-environment) | Set environment variables for the test environment | +| **Debugging** | [`debug-keep-container`](https://cibuildwheel.pypa.io/en/stable/options/#debug-keep-container) | Keep the container after running for debugging. | +| | [`debug-traceback`](https://cibuildwheel.pypa.io/en/stable/options/#debug-traceback) | Print full traceback when errors occur. | +| | [`build-verbosity`](https://cibuildwheel.pypa.io/en/stable/options/#build-verbosity) | Increase/decrease the output of the build | + + + + +These options can be specified in a pyproject.toml file, or as environment variables, see [configuration docs](https://cibuildwheel.pypa.io/en/latest/configuration/). Working examples ---------------- @@ -140,40 +178,40 @@ Here are some repos that use cibuildwheel. | Name | CI | OS | Notes | |-----------------------------------|----|----|:------| | [scikit-learn][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The machine learning library. A complex but clean config using many of cibuildwheel's features to build a large project with Cython and C++ extensions. | -| [Tornado][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed. | | [pytorch-fairseq][] | ![github icon][] | ![apple icon][] ![linux icon][] | Facebook AI Research Sequence-to-Sequence Toolkit written in Python. | +| [duckdb][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | DuckDB is an analytical in-process SQL database management system | +| [NumPy][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The fundamental package for scientific computing with Python. | +| [Tornado][] | ![github icon][] | ![linux icon][] ![apple icon][] ![windows icon][] | Tornado is a Python web framework and asynchronous networking library. Uses stable ABI for a small C extension. | +| [NCNN][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | ncnn is a high-performance neural network inference framework optimized for the mobile platform | | [Matplotlib][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The venerable Matplotlib, a Python library with C++ portions | -| [MyPy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | MyPyC, the compiled component of MyPy. | -| [pydantic][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Data parsing and validation using Python type hints | -| [uvloop][] | ![github icon][] | ![apple icon][] ![linux icon][] | Ultra fast asyncio event loop. | -| [psutil][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Cross-platform lib for process and system monitoring in Python | -| [vaex][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Out-of-Core hybrid Apache Arrow/NumPy DataFrame for Python, ML, visualization and exploration of big tabular data at a billion rows per second 🚀 | -| [Google Benchmark][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A microbenchmark support library | +| [MyPy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | The compiled version of MyPy using MyPyC. | +| [Prophet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Tool for producing high quality forecasts for time series data that has multiple seasonality with linear or non-linear growth. | +| [Kivy][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Open source UI framework written in Python, running on Windows, Linux, macOS, Android and iOS | [scikit-learn]: https://github.com/scikit-learn/scikit-learn +[pytorch-fairseq]: https://github.com/facebookresearch/fairseq +[duckdb]: https://github.com/duckdb/duckdb +[NumPy]: https://github.com/numpy/numpy [Tornado]: https://github.com/tornadoweb/tornado -[pytorch-fairseq]: https://github.com/pytorch/fairseq +[NCNN]: https://github.com/Tencent/ncnn [Matplotlib]: https://github.com/matplotlib/matplotlib [MyPy]: https://github.com/mypyc/mypy_mypyc-wheels -[pydantic]: https://github.com/samuelcolvin/pydantic -[uvloop]: https://github.com/MagicStack/uvloop -[psutil]: https://github.com/giampaolo/psutil -[vaex]: https://github.com/vaexio/vaex -[Google Benchmark]: https://github.com/google/benchmark +[Prophet]: https://github.com/facebook/prophet +[Kivy]: https://github.com/kivy/kivy -[appveyor icon]: docs/data/readme_icons/appveyor.svg [github icon]: docs/data/readme_icons/github.svg [azurepipelines icon]: docs/data/readme_icons/azurepipelines.svg [circleci icon]: docs/data/readme_icons/circleci.svg [gitlab icon]: docs/data/readme_icons/gitlab.svg [travisci icon]: docs/data/readme_icons/travisci.svg +[cirrusci icon]: docs/data/readme_icons/cirrusci.svg [windows icon]: docs/data/readme_icons/windows.svg [apple icon]: docs/data/readme_icons/apple.svg [linux icon]: docs/data/readme_icons/linux.svg -> ℹ️ That's just a handful, there are many more! Check out the [Working Examples](https://cibuildwheel.readthedocs.io/en/stable/working-examples) page in the docs. +> ℹ️ That's just a handful, there are many more! Check out the [Working Examples](https://cibuildwheel.pypa.io/en/stable/working-examples) page in the docs. Legal note ---------- @@ -187,77 +225,89 @@ This is similar to static linking, so it might have some license implications. C Changelog ========= - + - +### v3.0.0 -### v2.3.1 +_11 June 2025_ -_14 December 2021_ +See @henryiii's [release post](https://iscinumpy.dev/post/cibuildwheel-3-0-0/) for more info on new features! -- 🐛 Setting pip options like `PIP_USE_DEPRECATED` in `CIBW_ENVIRONMENT` no longer adversely affects cibuildwheel's ability to set up a Python environment (#956) -- 📚 Docs fixes and improvements +- 🌟 Adds the ability to [build wheels for iOS](https://cibuildwheel.pypa.io/en/stable/platforms/#ios)! Set the [`platform` option](https://cibuildwheel.pypa.io/en/stable/options/#platform) to `ios` on a Mac with the iOS toolchain to try it out! (#2286, #2363, #2432) +- 🌟 Adds support for the GraalPy interpreter! Enable for your project using the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable). (#1538, #2411, #2414) +- ✨ Adds CPython 3.14 support, under the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable) `cpython-prerelease`. This version of cibuildwheel uses 3.14.0b2. (#2390) -### v2.3.0 + _While CPython is in beta, the ABI can change, so your wheels might not be compatible with the final release. For this reason, we don't recommend distributing wheels until RC1, at which point 3.14 will be available in cibuildwheel without the flag._ (#2390) +- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), and changes the working directory for tests. (#2062, #2284, #2437) -_26 November 2021_ + - If this option is set, cibuildwheel will copy the files and folders specified in `test-sources` into the temporary directory we run from. This is required for iOS builds, but also useful for other platforms, as it allows you to avoid placeholders. + - If this option is not set, behaviour matches v2.x - cibuildwheel will run the tests from a temporary directory, and you can use the `{project}` placeholder in the `test-command` to refer to the project directory. (#2420) -- 📈 cibuildwheel now defaults to manylinux2014 image for linux builds, rather than manylinux2010. If you want to stick with manylinux2010, it's simple to set this using [the image options](https://cibuildwheel.readthedocs.io/en/stable/options/#linux-image). (#926) -- ✨ You can now pass environment variables from the host machine into the Docker container during a Linux build. Check out [the docs for `CIBW_ENVIRONMENT_PASS_LINUX `](https://cibuildwheel.readthedocs.io/en/latest/options/#environment-pass) for the details. (#914) -- ✨ Added support for building PyPy 3.8 wheels. (#881) -- ✨ Added support for building Windows arm64 CPython wheels on a Windows arm64 runner. We can't test this in CI yet, so for now, this is experimental. (#920) -- 📚 Improved the deployment documentation (#911) -- 🛠 Changed the escaping behaviour inside cibuildwheel's option placeholders e.g. `{project}` in `before_build` or `{dest_dir}` in `repair_wheel_command`. This allows bash syntax like `${SOME_VAR}` to passthrough without being interpreted as a placeholder by cibuildwheel. See [this section](https://cibuildwheel.readthedocs.io/en/stable/options/#placeholders) in the docs for more info. (#889) -- 🛠 Pip updated to 21.3, meaning it now defaults to in-tree builds again. If this causes an issue with your project, setting environment variable `PIP_USE_DEPRECATED=out-of-tree-build` is available as a temporary flag to restore the old behaviour. However, be aware that this flag will probably be removed soon. (#881) -- 🐛 You can now access the current Python interpreter using `python3` within a build on Windows (#917) +- ✨ Adds [`dependency-versions`](https://cibuildwheel.pypa.io/en/stable/options/#dependency-versions) inline syntax (#2122) +- ✨ Improves support for Pyodide builds and adds the experimental [`pyodide-version`](https://cibuildwheel.pypa.io/en/stable/options/#pyodide-version) option, which allows you to specify the version of Pyodide to use for builds. (#2002) +- ✨ Add `pyodide-prerelease` [enable](https://cibuildwheel.pypa.io/en/stable/options/#enable) option, with an early build of 0.28 (Python 3.13). (#2431) +- ✨ Adds the [`test-environment`](https://cibuildwheel.pypa.io/en/stable/options/#test-environment) option, which allows you to set environment variables for the test command. (#2388) +- ✨ Adds the [`xbuild-tools`](https://cibuildwheel.pypa.io/en/stable/options/#xbuild-tools) option, which allows you to specify tools safe for cross-compilation. Currently only used on iOS; will be useful for Android in the future. (#2317) +- 🛠 The default [manylinux image](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) has changed from `manylinux2014` to `manylinux_2_28`. (#2330) +- 🛠 EOL images `manylinux1`, `manylinux2010`, `manylinux_2_24` and `musllinux_1_1` can no longer be specified by their shortname. The full OCI name can still be used for these images, if you wish. (#2316) +- 🛠 Invokes `build` rather than `pip wheel` to build wheels by default. You can control this via the [`build-frontend`](https://cibuildwheel.pypa.io/en/stable/options/#build-frontend) option. You might notice that you can see your build log output now! (#2321) +- 🛠 Build verbosity settings have been reworked to have consistent meanings between build backends when non-zero. (#2339) +- 🛠 Removed the `CIBW_PRERELEASE_PYTHONS` and `CIBW_FREE_THREADED_SUPPORT` options - these have been folded into the [`enable`](https://cibuildwheel.pypa.io/en/stable/options/#enable) option instead. (#2095) +- 🛠 Build environments no longer have setuptools and wheel preinstalled. (#2329) +- 🛠 Use the standard Schema line for the integrated JSONSchema. (#2433) +- ⚠️ Dropped support for building Python 3.6 and 3.7 wheels. If you need to build wheels for these versions, use cibuildwheel v2.23.3 or earlier. (#2282) +- ⚠️ The minimum Python version required to run cibuildwheel is now Python 3.11. You can still build wheels for Python 3.8 and newer. (#1912) +- ⚠️ 32-bit Linux wheels no longer built by default - the [arch](https://cibuildwheel.pypa.io/en/stable/options/#archs) was removed from `"auto"`. It now requires explicit `"auto32"`. Note that modern manylinux images (like the new default, `manylinux_2_28`) do not have 32-bit versions. (#2458) +- ⚠️ PyPy wheels no longer built by default, due to a change to our options system. To continue building PyPy wheels, you'll now need to set the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable) to `pypy` or `pypy-eol`. (#2095) +- ⚠️ Dropped official support for Appveyor. If it was working for you before, it will probably continue to do so, but we can't be sure, because our CI doesn't run there anymore. (#2386) +- 📚 A reorganisation of the docs, and numerous updates. (#2280) +- 📚 Use Python 3.14 color output in docs CLI output. (#2407) +- 📚 Docs now primarily use the pyproject.toml name of options, rather than the environment variable name. (#2389) +- 📚 README table now matches docs and auto-updates. (#2427, #2428) -### v2.2.2 +### v2.23.3 -_26 October 2021_ +_26 April 2025_ -- 🐛 Fix bug in the GitHub Action step causing a syntax error (#895) +- 🛠 Dependency updates, including Python 3.13.3 (#2371) -### v2.2.1 +### v2.23.2 -_26 October 2021_ +_24 March 2025_ -- 🛠 Added a `config-file` option on the GitHub Action to specify something other than pyproject.toml in your GitHub Workflow file. (#883) -- 🐛 Fix missing resources in sdist and released wheel on PyPI. We've also made some internal changes to our release processes to make them more reliable. (#893, #894) +- 🐛 Workaround an issue with pyodide builds when running cibuildwheel with a Python that was installed via UV (#2328 via #2331) +- 🛠 Dependency updates, including a manylinux update that fixes an ['undefined symbol' error](https://github.com/pypa/manylinux/issues/1760) in gcc-toolset (#2334) -### v2.2.0 +### v2.23.1 -_22 October 2021_ +_15 March 2025_ -- 🌟 Added support for [musllinux](https://www.python.org/dev/peps/pep-0656/). Support for this new wheel format lets projects build wheels for Linux distributions that use [musl libc](https://musl.libc.org/), notably, [Alpine](https://alpinelinux.org/) Docker containers. (#768) +- ⚠️ Added warnings when the shorthand values `manylinux1`, `manylinux2010`, `manylinux_2_24`, and `musllinux_1_1` are used to specify the images in linux builds. The shorthand to these (unmaintainted) images will be removed in v3.0. If you want to keep using these images, explicitly opt-in using the full image URL, which can be found in [this file](https://github.com/pypa/cibuildwheel/blob/v2.23.1/cibuildwheel/resources/pinned_docker_images.cfg). (#2312) +- 🛠 Dependency updates, including a manylinux update which fixes an [issue with rustup](https://github.com/pypa/cibuildwheel/issues/2303). (#2315) - Musllinux builds are enabled by default. If you're not ready to build musllinux, add `*-musllinux_*` to your [`CIBW_SKIP`/`skip`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) option. Or, you might have to make some changes to your options - to simplify that process, you can use... +### v2.23.0 -- 🌟 TOML option overrides! This provides much greater flexibility in configuration via pyproject.toml. (#854) +_1 March 2025_ - You can now set build options for any subset of your builds using a match pattern. So, for example, you can customise CPython 3.8 builds with an override on `cp38-*` or musllinux builds by selecting `*musllinux*`. Check out [the docs](https://cibuildwheel.readthedocs.io/en/latest/options/#overrides) for more info on the specifics. +- ✨ Adds official support for the new GitHub Actions Arm runners. In fact these worked out-of-the-box, now we include them in our tests and example configs. (#2135 via #2281) +- ✨ Adds support for building PyPy 3.11 wheels (#2268 via #2281) +- 🛠 Adopts the beta pypa/manylinux image for armv7l builds (#2269 via #2281) +- 🛠 Dependency updates, including Pyodide 0.27 (#2117 and #2281) -- 🛠 Added support for building PyPy wheels on macOS 11 CI runners. (#875) - -- 🛠 Setting an empty string for the [`CIBW_*_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) option will now fallthrough to the config file or cibuildwheel's default, rather than causing an error. This makes the option easier to use in CI build matricies. (#829) - -- 🛠 Support for TOML 1.0 when reading config files, via the `tomli` package. (#876) - -Note: This version is not available on PyPI due to some missing resources in the release files. Please use a later version instead. - - + --- That's the last few versions. -ℹ️ **Want more changelog? Head over to [the changelog page in the docs](https://cibuildwheel.readthedocs.io/en/stable/changelog/).** +ℹ️ **Want more changelog? Head over to [the changelog page in the docs](https://cibuildwheel.pypa.io/en/stable/changelog/).** --- Contributing ============ -For more info on how to contribute to cibuildwheel, see the [docs](https://cibuildwheel.readthedocs.io/en/latest/contributing/). +For more info on how to contribute to cibuildwheel, see the [docs](https://cibuildwheel.pypa.io/en/latest/contributing/). Everyone interacting with the cibuildwheel project via codebase, issue tracker, chat rooms, or otherwise is expected to follow the [PSF Code of Conduct](https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md). @@ -268,6 +318,7 @@ Maintainers - Yannick Jadoul [@YannickJadoul](https://github.com/YannickJadoul) - Matthieu Darbois [@mayeut](https://github.com/mayeut) - Henry Schreiner [@henryiii](https://github.com/henryiii) +- Grzegorz Bokota [@Czaki](https://github.com/Czaki) Credits ------- @@ -289,6 +340,6 @@ Massive props also to- See also ======== -If you'd like to keep wheel building separate from the package itself, check out [astrofrog/autowheel](https://github.com/astrofrog/autowheel). It builds packages using cibuildwheel from source distributions on PyPI. - Another very similar tool to consider is [matthew-brett/multibuild](http://github.com/matthew-brett/multibuild). `multibuild` is a shell script toolbox for building a wheel on various platforms. It is used as a basis to build some of the big data science tools, like SciPy. + +If you are building Rust wheels, you can get by without some of the tricks required to make GLIBC work via manylinux; this is especially relevant for cross-compiling, which is easy with Rust. See [maturin-action](https://github.com/PyO3/maturin-action) for a tool that is optimized for building Rust wheels and cross-compiling. diff --git a/action.yml b/action.yml index 703cb33a5..3e19ab982 100644 --- a/action.yml +++ b/action.yml @@ -13,6 +13,10 @@ inputs: description: 'File containing the config, defaults to {package}/pyproject.toml' required: false default: '' + only: + description: 'Build a specific wheel only. No need for arch/platform if this is set' + required: false + default: '' branding: icon: package color: yellow @@ -20,14 +24,68 @@ branding: runs: using: composite steps: + - uses: actions/setup-python@v5 + id: python + with: + python-version: "3.11 - 3.13" + update-environment: false + + - id: cibw + run: | + # Install cibuildwheel + "${{ steps.python.outputs.python-path }}" -u << "EOF" + import os + import shutil + import sys + import venv + + from pathlib import Path + from subprocess import run + + + class EnvBuilder(venv.EnvBuilder): + def __init__(self): + super().__init__() + + def setup_scripts(self, context): + pass + + def post_setup(self, context): + super().post_setup(context) + self.bin_path = Path(context.env_exe).parent + run([sys.executable, "-m", "pip", "--python", context.env_exe, "install", r"${{ github.action_path }}"], check=True) + + + print("::group::Install cibuildwheel") + venv_path = Path(r"${{ runner.temp }}") / "cibw" + if venv_path.exists(): + shutil.rmtree(venv_path) + builder = EnvBuilder() + builder.create(venv_path) + cibw_path = [path for path in builder.bin_path.glob("cibuildwheel*") if path.stem == "cibuildwheel"][0] + with open(os.environ["GITHUB_OUTPUT"], "at") as f: + f.write(f"cibw-path={cibw_path}\n") + print("::endgroup::") + EOF + shell: bash # Redirecting stderr to stdout to fix interleaving issue in Actions. - run: > - pipx run - --spec '${{ github.action_path }}' - cibuildwheel - ${{ inputs.package-dir }} - --output-dir ${{ inputs.output-dir }} - --config-file "${{ inputs.config-file }}" + "${{ steps.cibw.outputs.cibw-path }}" + "${{ inputs.package-dir }}" + ${{ inputs.output-dir != '' && format('--output-dir "{0}"', inputs.output-dir) || ''}} + ${{ inputs.config-file != '' && format('--config-file "{0}"', inputs.config-file) || ''}} + ${{ inputs.only != '' && format('--only "{0}"', inputs.only) || ''}} 2>&1 shell: bash + if: runner.os != 'Windows' + + # Windows needs powershell to interact nicely with Meson + - run: > + & "${{ steps.cibw.outputs.cibw-path }}" + "${{ inputs.package-dir }}" + ${{ inputs.output-dir != '' && format('--output-dir "{0}"', inputs.output-dir) || ''}} + ${{ inputs.config-file != '' && format('--config-file "{0}"', inputs.config-file) || ''}} + ${{ inputs.only != '' && format('--only "{0}"', inputs.only) || ''}} + shell: pwsh + if: runner.os == 'Windows' diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index fbea6b736..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,35 +0,0 @@ -environment: - matrix: - - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu - APPVEYOR_JOB_NAME: "python37-x64-ubuntu" - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - APPVEYOR_JOB_NAME: "python37-x64-vs2015" - - APPVEYOR_BUILD_WORKER_IMAGE: macos-mojave - APPVEYOR_JOB_NAME: "python37-x64-macos-mojave" - -stack: python 3.7 - -build: off - -init: -- cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -- ps: | - $BRANCH = if ($env:APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH) { $env:APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH } else { $env:APPVEYOR_REPO_BRANCH } - if (-not ($BRANCH -eq 'main' -or $BRANCH.ToLower().StartsWith('appveyor-'))) { - $env:PYTEST_ADDOPTS='-k "unit_test or test_0_basic" --suppress-no-test-exit-code' - } - python -m pip install -U pip - -install: python -m pip install -e ".[dev]" pytest-custom-exit-code - -# the '-u' flag is required so the output is in the correct order. -# See https://github.com/pypa/cibuildwheel/pull/24 for more info. -test_script: python -u ./bin/run_tests.py - -branches: - only: - - main - -skip_commits: - files: - - docs/* diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5a47c585f..23d6928e0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,43 +1,72 @@ -jobs: -- job: linux_38 - timeoutInMinutes: 120 - pool: {vmImage: 'Ubuntu-18.04'} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - - bash: | - python -m pip install -e ".[dev]" pytest-azurepipelines - python ./bin/run_tests.py +pr: + paths: + exclude: + - .github/** + - bin/* + - docs/** + - examples/github-* + - examples/ci* + - examples/travis-ci-* + - .ci* + - .pre-commit-config.yaml + - .readthedocs.yml + - .travis.yml + - README.md + - mkdocs.yml + - noxfile.py -- job: macos_38 - pool: {vmImage: 'macOS-10.15'} +jobs: +- job: linux_311 + timeoutInMinutes: 180 + pool: {vmImage: 'ubuntu-latest'} steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.8' + versionSpec: '3.11' - bash: | - python -m pip install -e ".[dev]" pytest-azurepipelines + docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all + python -m pip install -U pip + python -m pip install -e. --group test + if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then + echo "INFO: Exporting CIBW_ENABLE=all for main branch test run." + export CIBW_ENABLE=all + else + echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))." + fi python ./bin/run_tests.py -- job: windows_36 - pool: {vmImage: 'vs2017-win2016'} - timeoutInMinutes: 180 +- job: macos_311 + pool: {vmImage: 'macOS-latest'} + timeoutInMinutes: 120 steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.6' + versionSpec: '3.11' - bash: | - python -m pip install -e ".[dev]" pytest-azurepipelines + python -m pip install -U pip + python -m pip install -e. --group test + if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then + echo "INFO: Exporting CIBW_ENABLE=all for main branch test run." + export CIBW_ENABLE=all + else + echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))." + fi python ./bin/run_tests.py -- job: windows_38 - pool: {vmImage: 'vs2017-win2016'} +- job: windows_311 + pool: {vmImage: 'windows-latest'} timeoutInMinutes: 180 steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.8' + versionSpec: '3.11' - bash: | - python -m pip install -e ".[dev]" pytest-azurepipelines + python -m pip install -U pip + python -m pip install -e. --group test + if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then + echo "INFO: Exporting CIBW_ENABLE=all for main branch test run." + export CIBW_ENABLE=all + else + echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))." + fi python ./bin/run_tests.py diff --git a/bin/bump_version.py b/bin/bump_version.py index 7e2b1b308..af2d7ae32 100755 --- a/bin/bump_version.py +++ b/bin/bump_version.py @@ -1,30 +1,29 @@ #!/usr/bin/env python3 +# /// script +# dependencies = ["click", "packaging"] +# /// -from __future__ import annotations -import glob import os import subprocess import sys +import tomllib import urllib.parse from pathlib import Path import click from packaging.version import InvalidVersion, Version -import cibuildwheel - config = [ # file path, version find/replace format + ("pyproject.toml", 'version = "{}"'), ("README.md", "cibuildwheel=={}"), ("cibuildwheel/__init__.py", '__version__ = "{}"'), ("docs/faq.md", "cibuildwheel=={}"), ("docs/faq.md", "cibuildwheel@v{}"), - ("docs/setup.md", "cibuildwheel=={}"), ("examples/*", "cibuildwheel=={}"), ("examples/*", "cibuildwheel@v{}"), - ("setup.cfg", "version = {}"), ] RED = "\u001b[31m" @@ -34,7 +33,8 @@ @click.command() def bump_version() -> None: - current_version = cibuildwheel.__version__ + with open("pyproject.toml", "rb") as f: + current_version = tomllib.load(f)["project"]["version"] try: commit_date_str = subprocess.run( @@ -60,7 +60,7 @@ def bump_version() -> None: print(e) print("Failed to get previous version tag information.") - git_changes_result = subprocess.run(["git diff-index --quiet HEAD --"], shell=True) + git_changes_result = subprocess.run(["git diff-index --quiet HEAD --"], shell=True, check=False) repo_has_uncommitted_changes = git_changes_result.returncode != 0 if repo_has_uncommitted_changes: @@ -68,7 +68,7 @@ def bump_version() -> None: sys.exit(1) # fmt: off - print( 'Current version:', current_version) # noqa: E201 whitespace + print( 'Current version:', current_version) new_version = input(' New version: ').strip() # fmt: on @@ -82,7 +82,7 @@ def bump_version() -> None: actions = [] for path_pattern, version_pattern in config: - paths = [Path(p) for p in glob.glob(path_pattern)] + paths = list(Path().glob(path_pattern)) if not paths: print(f"error: Pattern {path_pattern} didn't match any files") @@ -129,8 +129,8 @@ def bump_version() -> None: contents = contents.replace(find, replace) path.write_text(contents, encoding="utf8") - print("Files updated. If you want to update the changelog as part of this") - print("commit, do that now.") + print("Files updated. If you want to update docs/changelog.md as part of") + print("this commit, do that now.") print() while input('Type "done" to continue: ').strip().lower() != "done": @@ -142,7 +142,8 @@ def bump_version() -> None: "pre-commit", "run", "--files=docs/changelog.md", - ] + ], + check=False, ) # run pre-commit to check that no errors occurred on the second run @@ -180,11 +181,11 @@ def bump_version() -> None: print() print("Push the new version to GitHub with:") - print(" git push && git push --tags") + print(f" git push && git push origin v{new_version}") print() release_url = "https://github.com/pypa/cibuildwheel/releases/new?" + urllib.parse.urlencode( - {"tag": new_version} + {"tag": f"v{new_version}"} ) print("Then create a release at the URL:") print(f" {release_url}") diff --git a/bin/dev_run_test b/bin/dev_run_test deleted file mode 100755 index 85a0d81ce..000000000 --- a/bin/dev_run_test +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -CIBW_PLATFORM=linux pytest "$@" diff --git a/bin/generate_pyodide_constraints.py b/bin/generate_pyodide_constraints.py new file mode 100755 index 000000000..e378dd8ee --- /dev/null +++ b/bin/generate_pyodide_constraints.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import sys +import textwrap +from pathlib import Path + +import click + +from cibuildwheel.extra import get_pyodide_xbuildenv_info + + +@click.command() +@click.argument( + "pyodide-version", + type=str, +) +@click.option( + "--output-file", + type=click.Path(), + default=None, + help="Output file to write the constraints to. If not provided, the constraints will be printed to stdout.", +) +def generate_pyodide_constraints(pyodide_version: str, output_file: str | None = None) -> None: + """ + Generate constraints for a specific Pyodide version. The constraints are + generated based on the Pyodide version's xbuildenv info, which is retrieved + from the Pyodide repository. + + These constraints should then be 'pinned' using `uv pip compile`. + + Example usage: + + bin/generate_pyodide_constraints.py 0.27.0 + """ + xbuildenv_info = get_pyodide_xbuildenv_info() + try: + pyodide_version_xbuildenv_info = xbuildenv_info["releases"][pyodide_version] + except KeyError as e: + msg = f"Pyodide version {pyodide_version} not found in xbuildenv info. Versions available: {', '.join(xbuildenv_info['releases'].keys())}" + raise click.BadParameter(msg) from e + + pyodide_build_min_version = pyodide_version_xbuildenv_info.get("min_pyodide_build_version") + pyodide_build_max_version = pyodide_version_xbuildenv_info.get("max_pyodide_build_version") + + pyodide_build_specifier_parts: list[str] = [] + + if pyodide_build_min_version: + pyodide_build_specifier_parts.append(f">={pyodide_build_min_version}") + if pyodide_build_max_version: + pyodide_build_specifier_parts.append(f"<={pyodide_build_max_version}") + + pyodide_build_specifier = ",".join(pyodide_build_specifier_parts) + + constraints_txt = textwrap.dedent(f""" + pip + build[virtualenv] + pyodide-build{pyodide_build_specifier} + click<8.2 + """) + + if output_file is None: + print(constraints_txt) + else: + Path(output_file).write_text(constraints_txt) + print(f"Constraints written to {output_file}", file=sys.stderr) + + +if __name__ == "__main__": + generate_pyodide_constraints() diff --git a/bin/generate_schema.py b/bin/generate_schema.py new file mode 100755 index 000000000..cc4e8730f --- /dev/null +++ b/bin/generate_schema.py @@ -0,0 +1,374 @@ +#!/usr/bin/env -S uv run -q + +# /// script +# dependencies = ["pyyaml"] +# /// + +import argparse +import copy +import functools +import json +import sys +from typing import Any + +import yaml + +make_parser = functools.partial(argparse.ArgumentParser, allow_abbrev=False) +if sys.version_info >= (3, 14): + make_parser = functools.partial(make_parser, color=True, suggest_on_error=True) +parser = make_parser() +parser.add_argument("--schemastore", action="http://github.com/store_true", help="Generate schema_store version") +args = parser.parse_args() + +starter = """ +$schema: http://json-schema.org/draft-07/schema# +$id: https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/resources/cibuildwheel.schema.json +$defs: + inherit: + enum: + - none + - prepend + - append + default: none + description: How to inherit the parent's value. + enable: + enum: + - cpython-experimental-riscv64 + - cpython-freethreading + - cpython-prerelease + - graalpy + - pyodide-prerelease + - pypy + - pypy-eol + description: A Python version or flavor to enable. +additionalProperties: false +description: cibuildwheel's settings. +type: object +properties: + archs: + description: Change the architectures built on your machine by default. + type: string_array + before-all: + description: Execute a shell command on the build system before any wheels are built. + type: string_array + before-build: + description: Execute a shell command preparing each wheel's build. + type: string_array + before-test: + description: Execute a shell command before testing each wheel. + type: string_array + build: + default: ['*'] + description: Choose the Python versions to build. + type: string_array + build-frontend: + default: default + description: Set the tool to use to build, either "pip" (default for now), "build", or "build[uv]" + oneOf: + - enum: [pip, build, "build[uv]", default] + - type: string + pattern: '^pip; ?args:' + - type: string + pattern: '^build; ?args:' + - type: string + pattern: '^build\\[uv\\]; ?args:' + - type: object + additionalProperties: false + required: [name] + properties: + name: + enum: [pip, build, "build[uv]"] + args: + type: array + items: + type: string + build-verbosity: + type: integer + minimum: -3 + maximum: 3 + default: 0 + description: Increase/decrease the output of pip wheel. + config-settings: + description: Specify config-settings for the build backend. + type: string_table_array + container-engine: + oneOf: + - enum: [docker, podman] + - type: string + pattern: '^docker; ?(create_args|disable_host_mount):' + - type: string + pattern: '^podman; ?(create_args|disable_host_mount):' + - type: object + additionalProperties: false + required: [name] + properties: + name: + enum: [docker, podman] + create-args: + type: array + items: + type: string + disable-host-mount: + type: boolean + dependency-versions: + default: pinned + description: Specify how cibuildwheel controls the versions of the tools it uses + oneOf: + - enum: [pinned, latest] + - type: string + description: Path to a file containing dependency versions, or inline package specifications, starting with "packages:" + not: + enum: [pinned, latest] + - type: object + additionalProperties: false + properties: + file: + type: string + - type: object + additionalProperties: false + properties: + packages: + type: array + items: + type: string + enable: + description: Enable or disable certain builds. + oneOf: + - $ref: "#/$defs/enable" + - type: array + items: + $ref: "#/$defs/enable" + environment: + description: Set environment variables needed during the build. + type: string_table + environment-pass: + description: Set environment variables on the host to pass-through to the container + during the build. + type: string_array + manylinux-aarch64-image: + type: string + description: Specify alternative manylinux / musllinux container images + manylinux-armv7l-image: + type: string + description: Specify alternative manylinux / musllinux container images + manylinux-i686-image: + type: string + description: Specify alternative manylinux / musllinux container images + manylinux-ppc64le-image: + type: string + description: Specify alternative manylinux / musllinux container images + manylinux-pypy_aarch64-image: + type: string + description: Specify alternative manylinux / musllinux container images + manylinux-pypy_i686-image: + type: string + description: Specify alternative manylinux / musllinux container images + manylinux-pypy_x86_64-image: + type: string + description: Specify alternative manylinux / musllinux container images + manylinux-riscv64-image: + type: string + description: Specify alternative manylinux / musllinux container images + manylinux-s390x-image: + type: string + description: Specify alternative manylinux / musllinux container images + manylinux-x86_64-image: + type: string + description: Specify alternative manylinux / musllinux container images + musllinux-aarch64-image: + type: string + description: Specify alternative manylinux / musllinux container images + musllinux-armv7l-image: + type: string + description: Specify alternative manylinux / musllinux container images + musllinux-i686-image: + type: string + description: Specify alternative manylinux / musllinux container images + musllinux-ppc64le-image: + type: string + description: Specify alternative manylinux / musllinux container images + musllinux-riscv64-image: + type: string + description: Specify alternative manylinux / musllinux container images + musllinux-s390x-image: + type: string + description: Specify alternative manylinux / musllinux container images + musllinux-x86_64-image: + type: string + description: Specify alternative manylinux / musllinux container images + xbuild-tools: + description: Binaries on the path that should be included in an isolated cross-build environment + type: string_array + pyodide-version: + type: string + description: Specify the version of Pyodide to use + repair-wheel-command: + description: Execute a shell command to repair each built wheel. + type: string_array + skip: + description: Choose the Python versions to skip. + type: string_array + test-command: + description: Execute a shell command to test each built wheel. + type: string_array + test-extras: + description: Install your wheel for testing using `extras_require` + type: string_array + test-sources: + description: Test files that are required by the test environment + type: string_array + test-groups: + description: Install extra groups when testing + type: string_array + test-requires: + description: Install Python dependencies before running the tests + type: string_array + test-skip: + description: Skip running tests on some builds. + type: string_array + test-environment: + description: Set environment variables for the test environment + type: string_table +""" + +schema = yaml.safe_load(starter) + +string_array = yaml.safe_load( + """ +- type: string +- type: array + items: + type: string +""" +) + +string_table_array = yaml.safe_load( + """ +- type: string +- type: object + additionalProperties: false + patternProperties: + .+: + oneOf: + - type: string + - type: array + items: + type: string +""" +) + +string_table = yaml.safe_load( + """ +- type: string +- type: object + additionalProperties: false + patternProperties: + .+: + type: string +""" +) + +for value in schema["properties"].values(): + match value: + case {"type": "string_array"}: + del value["type"] + value["oneOf"] = string_array + case {"type": "string_table"}: + del value["type"] + value["oneOf"] = string_table + case {"type": "string_table_array"}: + del value["type"] + value["oneOf"] = string_table_array + +overrides = yaml.safe_load( + """ +type: array +description: An overrides array +items: + type: object + required: ["select"] + minProperties: 2 + additionalProperties: false + properties: + select: {} + inherit: + type: object + additionalProperties: false + properties: + before-all: {"$ref": "#/$defs/inherit"} + before-build: {"$ref": "#/$defs/inherit"} + xbuild-tools: {"$ref": "#/$defs/inherit"} + before-test: {"$ref": "#/$defs/inherit"} + config-settings: {"$ref": "#/$defs/inherit"} + container-engine: {"$ref": "#/$defs/inherit"} + environment: {"$ref": "#/$defs/inherit"} + environment-pass: {"$ref": "#/$defs/inherit"} + repair-wheel-command: {"$ref": "#/$defs/inherit"} + test-command: {"$ref": "#/$defs/inherit"} + test-extras: {"$ref": "#/$defs/inherit"} + test-sources: {"$ref": "#/$defs/inherit"} + test-requires: {"$ref": "#/$defs/inherit"} + test-environment: {"$ref": "#/$defs/inherit"} +""" +) + +for key, value in schema["properties"].items(): + value["title"] = f"CIBW_{key.replace('-', '_').upper()}" + +non_global_options = {k: {"$ref": f"#/properties/{k}"} for k in schema["properties"]} +del non_global_options["build"] +del non_global_options["skip"] +del non_global_options["test-skip"] +del non_global_options["enable"] + +overrides["items"]["properties"]["select"]["oneOf"] = string_array +overrides["items"]["properties"] |= non_global_options.copy() + +del overrides["items"]["properties"]["archs"] + +not_linux = non_global_options.copy() + +del not_linux["environment-pass"] +del not_linux["container-engine"] +for key in list(not_linux): + if "linux-" in key: + del not_linux[key] + + +def as_object(d: dict[str, Any]) -> dict[str, Any]: + return { + "type": "object", + "additionalProperties": False, + "properties": copy.deepcopy(d), + } + + +oses = { + "linux": as_object(non_global_options), + "windows": as_object(not_linux), + "macos": as_object(not_linux), + "pyodide": as_object(not_linux), + "ios": as_object(not_linux), +} + +oses["linux"]["properties"]["repair-wheel-command"] = { + **schema["properties"]["repair-wheel-command"], + "default": "auditwheel repair -w {dest_dir} {wheel}", +} +oses["macos"]["properties"]["repair-wheel-command"] = { + **schema["properties"]["repair-wheel-command"], + "default": "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}", +} + +del oses["linux"]["properties"]["dependency-versions"] + +schema["properties"]["overrides"] = overrides +schema["properties"] |= oses + +if args.schemastore: + schema["$id"] = "https://json.schemastore.org/partial-cibuildwheel.json" + schema["description"] = ( + "cibuildwheel's settings. Generated with ./bin/generate_schema.py --schemastore from cibuildwheel." + ) + +print(json.dumps(schema, indent=2)) diff --git a/bin/inspect_all_known_projects.py b/bin/inspect_all_known_projects.py index 542e42b80..60688c4bb 100755 --- a/bin/inspect_all_known_projects.py +++ b/bin/inspect_all_known_projects.py @@ -1,13 +1,23 @@ #!/usr/bin/env python3 -from __future__ import annotations + +""" +Check known projects for usage of requires-python. + +Usage: + + ./bin/inspect_all_known_projects.py --online=$GITHUB_TOKEN + +This will cache the results to all_known_setup.yaml; you can reprint +the results without the `--online` setting. +""" import ast +from collections.abc import Iterable, Iterator from pathlib import Path -from typing import Iterator import click import yaml -from ghapi.core import GhApi, HTTP404NotFoundError +from github import Github, GithubException from rich import print from cibuildwheel.projectfiles import Analyzer @@ -47,39 +57,44 @@ def check_repo(name: str, contents: str) -> str: class MaybeRemote: - def __init__(self, cached_file: Path | str, *, online: bool) -> None: - self.online = online - if self.online: - self.contents: dict[str, dict[str, str | None]] = { + github: Github | None + contents: dict[str, dict[str, str | None]] + + def __init__(self, cached_file: Path | str, *, online: str | None) -> None: + if online is not None: + self.github = Github(online) + self.contents = { "setup.py": {}, "setup.cfg": {}, "pyproject.toml": {}, } else: + self.github = None with open(cached_file) as f: self.contents = yaml.safe_load(f) def get(self, repo: str, filename: str) -> str | None: - if self.online: + if self.github: try: - self.contents[filename][repo] = ( - GhApi(*repo.split("/")).get_content(filename).decode() - ) - except HTTP404NotFoundError: + gh_file = self.github.get_repo(repo).get_contents(filename) + except GithubException: self.contents[filename][repo] = None + else: + assert not isinstance(gh_file, list) + self.contents[filename][repo] = gh_file.decoded_content.decode(encoding="utf-8") + return self.contents[filename][repo] elif repo in self.contents[filename]: return self.contents[filename][repo] else: - raise RuntimeError( - f"Trying to access {repo}:{filename} and not in cache, rebuild cache" - ) + msg = f"Trying to access {repo}:{filename} and not in cache, rebuild cache" + raise RuntimeError(msg) def save(self, filename: Path | str) -> None: with open(filename, "w") as f: yaml.safe_dump(self.contents, f, default_flow_style=False) - def on_each(self, repos: list[str]) -> Iterator[tuple[str, str, str | None]]: + def on_each(self, repos: Iterable[str]) -> Iterator[tuple[str, str, str | None]]: for repo in repos: print(f"[bold]{repo}:") for filename in sorted(self.contents, reverse=True): @@ -87,8 +102,8 @@ def on_each(self, repos: list[str]) -> Iterator[tuple[str, str, str | None]]: @click.command() -@click.option("--online", is_flag=True, help="Remember to set GITHUB_TOKEN") -def main(online: bool) -> None: +@click.option("--online", help="Set to $GITHUB_TOKEN") +def main(online: str | None) -> None: with open(DIR / "../docs/data/projects.yml") as f: known = yaml.safe_load(f) diff --git a/bin/make_dependency_update_pr.py b/bin/make_dependency_update_pr.py index be8a57ad5..0d46aada4 100755 --- a/bin/make_dependency_update_pr.py +++ b/bin/make_dependency_update_pr.py @@ -1,29 +1,30 @@ #!/usr/bin/env python3 -from __future__ import annotations import os +import subprocess import sys import textwrap import time from pathlib import Path -from subprocess import run import click -def shell(cmd, **kwargs): - return run([cmd], shell=True, **kwargs) +def shell(cmd: str, *, check: bool, **kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.run([cmd], shell=True, check=check, **kwargs) # type: ignore[call-overload, no-any-return] -def git_repo_has_changes(): - unstaged_changes = shell("git diff-index --quiet HEAD --").returncode != 0 - staged_changes = shell("git diff-index --quiet --cached HEAD --").returncode != 0 +def git_repo_has_changes() -> bool: + unstaged_changes: bool = shell("git diff-index --quiet HEAD --", check=False).returncode != 0 + staged_changes: bool = ( + shell("git diff-index --quiet --cached HEAD --", check=False).returncode != 0 + ) return unstaged_changes or staged_changes @click.command() -def main(): +def main() -> None: project_root = Path(__file__).parent / ".." os.chdir(project_root) @@ -54,10 +55,10 @@ def main(): f""" Update the versions of our dependencies. - PR generated by `{os.path.basename(__file__)}`. + PR generated by `{Path(__file__).name}`. """ ) - run( + subprocess.run( [ "gh", "pr", @@ -73,7 +74,7 @@ def main(): print("Done.") finally: # remove any local changes - shell("git checkout -- .") + shell("git checkout -- .", check=True) shell(f"git checkout {previous_branch}", check=True) shell(f"git branch -D --force {branch_name}", check=True) diff --git a/bin/projects.py b/bin/projects.py old mode 100644 new mode 100755 index 4b343391a..47a1dcb4f --- a/bin/projects.py +++ b/bin/projects.py @@ -1,5 +1,13 @@ #!/usr/bin/env python3 +# /// script +# dependencies = [ +# "click", +# "pyyaml", +# "pygithub", +# ] +# /// + """ Convert a yaml project list into a nice table. @@ -9,29 +17,28 @@ git diff """ -from __future__ import annotations - import builtins import functools import textwrap import urllib.request import xml.dom.minidom +from collections.abc import Iterable, Mapping, Sequence from datetime import datetime from io import StringIO from pathlib import Path -from typing import Any, TextIO +from typing import Any, Self, TextIO import click import yaml from github import Github, GithubException ICONS = ( - "appveyor", "github", "azurepipelines", "circleci", "gitlab", "travisci", + "cirrusci", "windows", "apple", "linux", @@ -41,7 +48,7 @@ class Project: NAME: int = 0 - def __init__(self, config: dict[str, Any], github: Github | None = None): + def __init__(self, config: Mapping[str, Any], github: Github | None = None): try: self.name: str = config["name"] self.gh: str = config["gh"] @@ -75,7 +82,7 @@ def __init__(self, config: dict[str, Any], github: Github | None = None): name_len = len(self.name) + 4 self.__class__.NAME = max(self.__class__.NAME, name_len) - def __lt__(self, other: Project) -> bool: + def __lt__(self, other: Self) -> bool: if self.online: return self.num_stars < other.num_stars else: @@ -85,8 +92,8 @@ def __lt__(self, other: Project) -> bool: def header(cls) -> str: return textwrap.dedent( f"""\ - | {'Name':{cls.NAME}} | CI | OS | Notes | - |{'':-^{cls.NAME+2 }}|----|----|:------|""" + | {"Name":{cls.NAME}} | CI | OS | Notes | + |{"":-^{cls.NAME + 2}}|----|----|:------|""" ) @property @@ -128,6 +135,7 @@ def fetch_icon(icon_name: str) -> None: document = xml.dom.minidom.parseString(original_svg_data) svgElement = document.documentElement + assert svgElement is not None assert svgElement.nodeName == "svg" svgElement.setAttribute("width", "16px") svgElement.setAttribute("fill", "#606060") @@ -148,7 +156,7 @@ def path_for_icon(icon_name: str, relative_to: Path | None = None) -> Path: def get_projects( - config: list[dict[str, Any]], + config: Iterable[Mapping[str, Any]], *, online: bool = True, auth: str | None = None, @@ -162,7 +170,9 @@ def get_projects( return sorted((Project(item, github) for item in config), reverse=online) -def render_projects(projects: list[Project], *, dest_path: Path, include_info: bool = True): +def render_projects( + projects: Sequence[Project], *, dest_path: Path, include_info: bool = True +) -> str: io = StringIO() print = functools.partial(builtins.print, file=io) @@ -190,10 +200,10 @@ def render_projects(projects: list[Project], *, dest_path: Path, include_info: b def insert_projects_table( file: Path, *, - projects: list[Project], + projects: Sequence[Project], input_filename: str, include_info: bool = True, -): +) -> None: text = file.read_text() projects_table = render_projects(projects, include_info=include_info, dest_path=file) @@ -207,7 +217,7 @@ def insert_projects_table( generated_note = f"" new_text = ( - f"{text[:start + len(start_str)]}\n{generated_note}\n\n{projects_table}\n{text[end:]}" + f"{text[: start + len(start_str)]}\n{generated_note}\n\n{projects_table}\n{text[end:]}" ) file.write_text(new_text) @@ -242,7 +252,7 @@ def projects( README_FILE, projects=projects[:10], input_filename=input.name, include_info=False ) insert_projects_table( - DOCS_PAGE, projects=projects, input_filename=input.name, include_info=True + DOCS_PAGE, projects=projects, input_filename=input.name, include_info=False ) diff --git a/bin/readme_changelog.py b/bin/readme_changelog.py new file mode 100755 index 000000000..29c30293e --- /dev/null +++ b/bin/readme_changelog.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import re +from pathlib import Path + +PROJECT_ROOT = Path(__file__).parent / ".." +CHANGELOG_FILE = PROJECT_ROOT / "docs" / "changelog.md" + +# https://regexr.com/622ds +FIRST_5_CHANGELOG_ENTRIES_REGEX = re.compile(r"""(^###.*?(?=###)){5}""", re.DOTALL | re.MULTILINE) + + +def mini_changelog() -> str: + changelog_text = CHANGELOG_FILE.read_text() + + mini_changelog_match = FIRST_5_CHANGELOG_ENTRIES_REGEX.search(changelog_text) + assert mini_changelog_match, "Failed to find the first few changelog entries" + + return f"\n{mini_changelog_match.group(0).strip()}\n" + + +if __name__ == "__main__": + print(mini_changelog()) diff --git a/bin/readme_options_table.py b/bin/readme_options_table.py new file mode 100755 index 000000000..203d6bf5d --- /dev/null +++ b/bin/readme_options_table.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import dataclasses +import re +from pathlib import Path +from typing import Final + +DIR: Final[Path] = Path(__file__).parent.parent.resolve() +OPTIONS_MD: Final[Path] = DIR / "docs" / "options.md" + +SECTION_HEADER_REGEX = re.compile(r"^## (?P.*?)$", re.MULTILINE) + +# https://regexr.com/8f1ff +OPTION_HEADER_REGEX = re.compile( + r"^### (?P.*?){.*#(?P\S+).*}\n+> ?(?P.*)$", re.MULTILINE +) + + +@dataclasses.dataclass(kw_only=True) +class Option: + name: str + id: str + desc: str + section: str + + +def get_table() -> str: + options_md = OPTIONS_MD.read_text(encoding="utf-8") + + sections = SECTION_HEADER_REGEX.split(options_md)[1:] + + options = [] + + for section_name, section_content in zip(sections[0::2], sections[1::2], strict=True): + for match in OPTION_HEADER_REGEX.finditer(section_content): + option = Option( + name=match.group("name").strip(), + id=match.group("id").strip(), + desc=match.group("desc").strip(), + section=section_name.strip(), + ) + options.append(option) + + table_md = "\n\n\n" + table_md += "| | Option | Description |\n" + table_md += "|---|---|---|\n" + last_section: str | None = None + + for option in options: + cells: list[str] = [] + + cells.append(f"**{option.section}**" if option.section != last_section else "") + last_section = option.section + + url = f"https://cibuildwheel.pypa.io/en/stable/options/#{option.id}" + name = option.name.replace(", ", "
") # Replace commas with line breaks + cells.append(f"[{name}]({url})") + + cells.append(option.desc) + + table_md += "| " + " | ".join(cells) + " |\n" + table_md += "\n" + + return table_md + + +if __name__ == "__main__": + print(get_table()) diff --git a/bin/run_example_ci_configs.py b/bin/run_example_ci_configs.py index 8c4248012..a6bd1bd06 100755 --- a/bin/run_example_ci_configs.py +++ b/bin/run_example_ci_configs.py @@ -1,32 +1,32 @@ #!/usr/bin/env python3 -from __future__ import annotations import os import shutil +import subprocess import sys import textwrap import time -from collections import namedtuple -from glob import glob +import typing from pathlib import Path -from subprocess import run from urllib.parse import quote import click +DIR = Path(__file__).parent.resolve() -def shell(cmd, **kwargs): - return run([cmd], shell=True, **kwargs) +def shell(cmd: str, *, check: bool, **kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.run([cmd], shell=True, check=check, **kwargs) # type: ignore[call-overload, no-any-return] -def git_repo_has_changes(): - unstaged_changes = shell("git diff-index --quiet HEAD --").returncode != 0 - staged_changes = shell("git diff-index --quiet --cached HEAD --").returncode != 0 + +def git_repo_has_changes() -> bool: + unstaged_changes = shell("git diff-index --quiet HEAD --", check=False).returncode != 0 + staged_changes = shell("git diff-index --quiet --cached HEAD --", check=False).returncode != 0 return unstaged_changes or staged_changes -def generate_basic_project(path): +def generate_basic_project(path: Path) -> None: sys.path.insert(0, "") from test.test_projects.c import new_c_project @@ -34,17 +34,17 @@ def generate_basic_project(path): project.generate(path) -CIService = namedtuple("CIService", "name dst_config_path badge_md") +class CIService(typing.NamedTuple): + name: str + dst_config_path: str + badge_md: str + + services = [ - CIService( - name="appveyor", - dst_config_path="appveyor.yml", - badge_md="[![Build status](https://ci.appveyor.com/api/projects/status/wbsgxshp05tt1tif/branch/{branch}?svg=true)](https://ci.appveyor.com/project/pypa/cibuildwheel/branch/{branch})", - ), CIService( name="azure-pipelines", dst_config_path="azure-pipelines.yml", - badge_md="[![Build Status](https://dev.azure.com/joerick0429/cibuildwheel/_apis/build/status/joerick.cibuildwheel?branchName={branch})](https://dev.azure.com/joerick0429/cibuildwheel/_build/latest?definitionId=2&branchName={branch})", + badge_md="[![Build Status](https://dev.azure.com/joerick0429/cibuildwheel/_apis/build/status/pypa.cibuildwheel?branchName={branch})](https://dev.azure.com/joerick0429/cibuildwheel/_build/latest?definitionId=2&branchName={branch})", ), CIService( name="circleci", @@ -59,24 +59,28 @@ def generate_basic_project(path): CIService( name="travis-ci", dst_config_path=".travis.yml", - badge_md="[![Build Status](https://travis-ci.org/pypa/cibuildwheel.svg?branch={branch})](https://travis-ci.org/pypa/cibuildwheel)", + badge_md="[![Build Status](https://app.travis-ci.com/pypa/cibuildwheel.svg?branch={branch})](https://app.travis-ci.com/pypa/cibuildwheel)", ), CIService( name="gitlab", dst_config_path=".gitlab-ci.yml", - badge_md="[![Gitlab](https://gitlab.com/pypa/cibuildwheel/badges/{branch}/pipeline.svg)](https://gitlab.com/pypa/cibuildwheel/-/commits/{branch})", + badge_md="[![Gitlab](https://gitlab.com/joerick/cibuildwheel/badges/{branch}/pipeline.svg)](https://gitlab.com/joerick/cibuildwheel/-/commits/{branch})", + ), + CIService( + name="cirrus-ci", + dst_config_path=".cirrus.yml", + badge_md="[![Cirrus CI](https://api.cirrus-ci.com/github/pypa/cibuildwheel.svg?branch={branch})](https://cirrus-ci.com/github/pypa/cibuildwheel/{branch})", ), ] -def ci_service_for_config_file(config_file): - service_name = Path(config_file).name.rsplit("-", 1)[0] - - for service in services: - if service.name == service_name: - return service - - raise ValueError(f"unknown ci service for config file {config_file}") +def ci_service_for_config_file(config_file: Path) -> CIService: + filename = config_file.name + try: + return next(s for s in services if filename.startswith(s.name)) + except StopIteration: + msg = f"unknown ci service for config file {config_file}" + raise ValueError(msg) from None @click.command() @@ -88,15 +92,16 @@ def run_example_ci_configs(config_files=None): """ if len(config_files) == 0: - config_files = glob("examples/*-minimal.yml") + config_files = Path("examples").glob("*-minimal.yml") # check each CI service has at most 1 config file - configs_by_service = {} + configs_by_service = set() for config_file in config_files: service = ci_service_for_config_file(config_file) if service.name in configs_by_service: - raise Exception("You cannot specify more than one config per CI service") - configs_by_service[service.name] = config_file + msg = "You cannot specify more than one config per CI service" + raise Exception(msg) + configs_by_service.add(service.name) if git_repo_has_changes(): print("Your git repo has uncommitted changes. Commit or stash before continuing.") @@ -117,23 +122,22 @@ def run_example_ci_configs(config_files=None): for config_file in config_files: service = ci_service_for_config_file(config_file) - src_config_file = Path(config_file) dst_config_file = example_project / service.dst_config_path dst_config_file.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(src_config_file, dst_config_file) + shutil.copyfile(config_file, dst_config_file) - run(["git", "add", example_project], check=True) + subprocess.run(["git", "add", example_project], check=True) message = textwrap.dedent( f"""\ Test example minimal configs - Testing files: {config_files} + Testing files: {[str(f) for f in config_files]} Generated from branch: {previous_branch} Time: {timestamp} """ ) - run(["git", "commit", "--no-verify", "--message", message], check=True) + subprocess.run(["git", "commit", "--no-verify", "--message", message], check=True) shell(f"git subtree --prefix={example_project} push origin {branch_name}", check=True) print("---") @@ -157,12 +161,11 @@ def run_example_ci_configs(config_files=None): finally: # remove any local changes shutil.rmtree(example_project, ignore_errors=True) - shell("git checkout -- .") + shell("git checkout -- .", check=True) shell(f"git checkout {previous_branch}", check=True) shell(f"git branch -D --force {branch_name}", check=True) if __name__ == "__main__": - os.chdir(os.path.dirname(__file__)) - os.chdir("..") + os.chdir(DIR) run_example_ci_configs(standalone_mode=True) diff --git a/bin/run_tests.py b/bin/run_tests.py index 44c6d3027..1c0923948 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -1,33 +1,94 @@ #!/usr/bin/env python3 + +import argparse +import functools import os import subprocess import sys from pathlib import Path if __name__ == "__main__": + if sys.version_info < (3, 13): + default_cpu_count = os.cpu_count() or 2 + else: + default_cpu_count = os.process_cpu_count() or 2 + + make_parser = functools.partial(argparse.ArgumentParser, allow_abbrev=False) + if sys.version_info >= (3, 14): + make_parser = functools.partial(make_parser, color=True, suggest_on_error=True) + parser = make_parser() + parser.add_argument( + "--run-podman", action="http://github.com/store_true", default=False, help="run podman tests (linux only)" + ) + parser.add_argument( + "--num-processes", + action="http://github.com/store", + default=default_cpu_count, + help="number of processes to use for testing", + ) + args = parser.parse_args() + # move cwd to the project root os.chdir(Path(__file__).resolve().parents[1]) - # run the unit tests + # unit tests unit_test_args = [sys.executable, "-m", "pytest", "unit_test"] - # run the docker unit tests only on Linux - if sys.platform.startswith("linux"): + + if sys.platform.startswith("linux") and os.environ.get("CIBW_PLATFORM", "linux") == "linux": + # run the docker unit tests only on Linux unit_test_args += ["--run-docker"] + + if args.run_podman: + unit_test_args += ["--run-podman"] + + print( + "\n\n================================== UNIT TESTS ==================================", + flush=True, + ) subprocess.run(unit_test_args, check=True) - # run the integration tests - subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "--numprocesses=2", - "-x", - "--durations", - "0", - "--timeout=2400", - "test", - ], - check=True, + # Run the serial integration tests without multiple processes + serial_integration_test_args = [ + sys.executable, + "-m", + "pytest", + "-m", + "serial", + "-x", + "--durations", + "0", + "--timeout=2400", + "test", + "-vv", + ] + print( + "\n\n=========================== SERIAL INTEGRATION TESTS ===========================", + flush=True, + ) + subprocess.run(serial_integration_test_args, check=True) + + # Non-serial integration tests + integration_test_args = [ + sys.executable, + "-m", + "pytest", + "-m", + "not serial", + f"--numprocesses={args.num_processes}", + "-x", + "--durations", + "0", + "--timeout=2400", + "test", + "-vv", + ] + + if sys.platform.startswith("linux") and args.run_podman: + integration_test_args += ["--run-podman"] + + print( + "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================", + flush=True, ) + subprocess.run(integration_test_args, check=True) diff --git a/bin/sample_build.py b/bin/sample_build.py index 90c636ed6..d98ce28ed 100755 --- a/bin/sample_build.py +++ b/bin/sample_build.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -from __future__ import annotations import argparse +import functools import os import subprocess import sys @@ -13,7 +13,10 @@ # move cwd to the project root os.chdir(Path(__file__).resolve().parents[1]) - parser = argparse.ArgumentParser(description="Runs a sample build") + make_parser = functools.partial(argparse.ArgumentParser, allow_abbrev=False) + if sys.version_info >= (3, 14): + make_parser = functools.partial(make_parser, color=True, suggest_on_error=True) + parser = make_parser(description="Runs a sample build") parser.add_argument("project_python_path", nargs="?", default="test.test_0_basic.basic_project") options = parser.parse_args() @@ -24,4 +27,8 @@ check=True, ) - sys.exit(subprocess.run([sys.executable, "-m", "cibuildwheel"], cwd=project_dir).returncode) + sys.exit( + subprocess.run( + [sys.executable, "-m", "cibuildwheel"], cwd=project_dir, check=False + ).returncode + ) diff --git a/bin/update_dependencies.py b/bin/update_dependencies.py deleted file mode 100755 index 69e62cfaa..000000000 --- a/bin/update_dependencies.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -# This file supports 3.6+ - -import os -import shutil -import subprocess -import sys -from pathlib import Path - -DIR = Path(__file__).parent.resolve() -RESOURCES = DIR.parent / "cibuildwheel/resources" - -python_version = "".join(str(v) for v in sys.version_info[:2]) - -env = os.environ.copy() - -# CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to -# regenerate the constraints files -env["CUSTOM_COMPILE_COMMAND"] = "bin/update_dependencies.py" - -os.chdir(DIR.parent) - -subprocess.run( - [ - "pip-compile", - "--allow-unsafe", - "--upgrade", - "cibuildwheel/resources/constraints.in", - f"--output-file=cibuildwheel/resources/constraints-python{python_version}.txt", - ], - check=True, - env=env, -) - -# default constraints.txt -if python_version == "39": - shutil.copyfile( - RESOURCES / f"constraints-python{python_version}.txt", - RESOURCES / "constraints.txt", - ) diff --git a/bin/update_docker.py b/bin/update_docker.py index 73da5a1ba..3c6ef79ed 100755 --- a/bin/update_docker.py +++ b/bin/update_docker.py @@ -1,56 +1,58 @@ #!/usr/bin/env python3 -from __future__ import annotations import configparser +import dataclasses from pathlib import Path -from typing import NamedTuple import requests +from packaging.version import Version DIR = Path(__file__).parent.resolve() RESOURCES = DIR.parent / "cibuildwheel/resources" -class Image(NamedTuple): +@dataclasses.dataclass(frozen=True) +class Image: manylinux_version: str - platform: str + platforms: list[str] image_name: str - tag: str | None # Set this to pin the image + tag: str | None = None # Set this to pin the image + use_platform_suffix: bool = False + + +class PyPAImage(Image): + def __init__(self, manylinux_version: str, platforms: list[str], tag: str | None = None): + image_name = f"quay.io/pypa/{manylinux_version}" + super().__init__(manylinux_version, platforms, image_name, tag, True) images = [ - # manylinux1 images - Image("manylinux1", "x86_64", "quay.io/pypa/manylinux1_x86_64", None), - Image("manylinux1", "i686", "quay.io/pypa/manylinux1_i686", None), - # manylinux2010 images - Image("manylinux2010", "x86_64", "quay.io/pypa/manylinux2010_x86_64", None), - Image("manylinux2010", "i686", "quay.io/pypa/manylinux2010_i686", None), - Image("manylinux2010", "pypy_x86_64", "quay.io/pypa/manylinux2010_x86_64", None), - Image("manylinux2010", "pypy_i686", "quay.io/pypa/manylinux2010_i686", None), # manylinux2014 images - Image("manylinux2014", "x86_64", "quay.io/pypa/manylinux2014_x86_64", None), - Image("manylinux2014", "i686", "quay.io/pypa/manylinux2014_i686", None), - Image("manylinux2014", "aarch64", "quay.io/pypa/manylinux2014_aarch64", None), - Image("manylinux2014", "ppc64le", "quay.io/pypa/manylinux2014_ppc64le", None), - Image("manylinux2014", "s390x", "quay.io/pypa/manylinux2014_s390x", None), - Image("manylinux2014", "pypy_x86_64", "quay.io/pypa/manylinux2014_x86_64", None), - Image("manylinux2014", "pypy_i686", "quay.io/pypa/manylinux2014_i686", None), - Image("manylinux2014", "pypy_aarch64", "quay.io/pypa/manylinux2014_aarch64", None), - # manylinux_2_24 images - Image("manylinux_2_24", "x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None), - Image("manylinux_2_24", "i686", "quay.io/pypa/manylinux_2_24_i686", None), - Image("manylinux_2_24", "aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None), - Image("manylinux_2_24", "ppc64le", "quay.io/pypa/manylinux_2_24_ppc64le", None), - Image("manylinux_2_24", "s390x", "quay.io/pypa/manylinux_2_24_s390x", None), - Image("manylinux_2_24", "pypy_x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None), - Image("manylinux_2_24", "pypy_i686", "quay.io/pypa/manylinux_2_24_i686", None), - Image("manylinux_2_24", "pypy_aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None), - # musllinux_1_1 images - Image("musllinux_1_1", "x86_64", "quay.io/pypa/musllinux_1_1_x86_64", None), - Image("musllinux_1_1", "i686", "quay.io/pypa/musllinux_1_1_i686", None), - Image("musllinux_1_1", "aarch64", "quay.io/pypa/musllinux_1_1_aarch64", None), - Image("musllinux_1_1", "ppc64le", "quay.io/pypa/musllinux_1_1_ppc64le", None), - Image("musllinux_1_1", "s390x", "quay.io/pypa/musllinux_1_1_s390x", None), + PyPAImage( + "manylinux2014", + [ + "x86_64", + "i686", + "aarch64", + "ppc64le", + "s390x", + "pypy_x86_64", + "pypy_i686", + "pypy_aarch64", + ], + ), + # manylinux_2_28 images + PyPAImage( + "manylinux_2_28", ["x86_64", "aarch64", "ppc64le", "s390x", "pypy_x86_64", "pypy_aarch64"] + ), + # manylinux_2_31 images + PyPAImage("manylinux_2_31", ["armv7l"]), + # manylinux_2_34 images + PyPAImage( + "manylinux_2_34", ["x86_64", "aarch64", "ppc64le", "s390x", "pypy_x86_64", "pypy_aarch64"] + ), + # musllinux_1_2 images + PyPAImage("musllinux_1_2", ["x86_64", "i686", "aarch64", "ppc64le", "s390x", "armv7l"]), ] config = configparser.ConfigParser() @@ -76,6 +78,21 @@ class Image(NamedTuple): for (name, info) in tags_dict.items() if info["manifest_digest"] == latest_tag["manifest_digest"] ) + elif image.image_name.startswith("ghcr.io/"): + repository = image.image_name[8:] + response = requests.get( + "https://ghcr.io/token", params={"scope": f"repository:{repository}:pull"} + ) + response.raise_for_status() + token = response.json()["token"] + response = requests.get( + f"https://ghcr.io/v2/%7Brepository%7D/tags/list", + headers={"Authorization": f"Bearer {token}"}, + ) + response.raise_for_status() + ghcr_tags = [(Version(tag), tag) for tag in response.json()["tags"] if tag != "latest"] + ghcr_tags.sort(reverse=True) + tag_name = ghcr_tags[0][1] else: response = requests.get(f"https://hub.docker.com/v2/repositories/%7Bimage.image_name%7D/tags") response.raise_for_status() @@ -91,10 +108,16 @@ class Image(NamedTuple): ) tag_name = pinned_tag["name"] - if not config.has_section(image.platform): - config[image.platform] = {} + for platform in image.platforms: + if not config.has_section(platform): + config[platform] = {} + suffix = "" + if image.use_platform_suffix: + suffix = f"_{platform.removeprefix('pypy_')}" + config[platform][image.manylinux_version] = f"{image.image_name}{suffix}:{tag_name}" - config[image.platform][image.manylinux_version] = f"{image.image_name}:{tag_name}" +if not config.has_section("riscv64"): + config["riscv64"] = {} with open(RESOURCES / "pinned_docker_images.cfg", "w") as f: config.write(f) diff --git a/bin/update_how_it_works_image.py b/bin/update_how_it_works_image.py new file mode 100755 index 000000000..504c9bc40 --- /dev/null +++ b/bin/update_how_it_works_image.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + + +import subprocess +import sys +from pathlib import Path + +try: + from html2image import Html2Image # type: ignore[import-not-found] +except ImportError: + sys.exit( + """ + html2image not found. Ensure you have Chrome (on Mac/Windows) or + Chromium (on Linux) installed, and then do: + pip install html2image + """ + ) + + +def main() -> None: + subprocess.run(["mkdocs", "build"], check=True) + + hti = Html2Image(custom_flags=["--force-device-scale-factor=2"]) + + html_str = Path("docs/diagram.md").read_text() + css_tags = f""" + + + + + """ + html_str = css_tags + html_str + + [screenshot, *_] = hti.screenshot( + html_str=html_str, + size=(830, 405), + ) + + dest_path = Path("docs/data/how-it-works.png") + dest_path.unlink(missing_ok=True) + + Path(screenshot).rename(dest_path) + + +if __name__ == "__main__": + main() diff --git a/bin/update_nodejs.py b/bin/update_nodejs.py new file mode 100755 index 000000000..a564fad00 --- /dev/null +++ b/bin/update_nodejs.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + + +import dataclasses +import difflib +import logging +import tomllib +from pathlib import Path +from typing import Final + +import click +import packaging.specifiers +import requests +import rich +from packaging.version import InvalidVersion, Version +from rich.logging import RichHandler +from rich.syntax import Syntax + +log = logging.getLogger("cibw") + +# Looking up the dir instead of using utils.resources_dir +# since we want to write to it. +DIR: Final[Path] = Path(__file__).parent.parent.resolve() +RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources" + +NODEJS_DIST: Final[str] = "https://nodejs.org/dist/" +NODEJS_INDEX: Final[str] = f"{NODEJS_DIST}index.json" + + +@dataclasses.dataclass(frozen=True, order=True) +class VersionTuple: + version: Version + version_string: str + + +def parse_nodejs_index() -> list[VersionTuple]: + versions: list[VersionTuple] = [] + response = requests.get(NODEJS_INDEX) + response.raise_for_status() + versions_info = response.json() + for version_info in versions_info: + version_string = version_info.get("version", "???") + if not version_info.get("lts", False): + log.debug("Ignoring non LTS release %r", version_string) + continue + if "linux-x64" not in version_info.get("files", []): + log.warning( + "Ignoring release %r which does not include a linux-x64 binary", version_string + ) + continue + try: + version = Version(version_string) + if version.is_devrelease: + log.info("Ignoring development release %r", str(version)) + continue + if version.is_prerelease: + log.info("Ignoring pre-release %r", str(version)) + continue + versions.append(VersionTuple(version, version_string)) + except InvalidVersion: + log.warning("Ignoring release %r", version_string) + versions.sort(reverse=True) + return versions + + +@click.command() +@click.option("--force", is_flag=True) +@click.option( + "--level", default="INFO", type=click.Choice(["WARNING", "INFO", "DEBUG"], case_sensitive=False) +) +def update_nodejs(force: bool, level: str) -> None: + logging.basicConfig( + level="INFO", + format="%(message)s", + datefmt="[%X]", + handlers=[RichHandler(rich_tracebacks=True, markup=True)], + ) + log.setLevel(level) + + toml_file_path = RESOURCES_DIR / "nodejs.toml" + + original_toml = toml_file_path.read_text() + with toml_file_path.open("rb") as f: + nodejs_data = tomllib.load(f) + + nodejs_data.pop("url") + + major_versions = [VersionTuple(Version(key), key) for key in nodejs_data] + major_versions.sort(reverse=True) + + versions = parse_nodejs_index() + + # update existing versions, 1 per LTS + for major_version in major_versions: + current = Version(nodejs_data[major_version.version_string]) + specifier = packaging.specifiers.SpecifierSet( + specifiers=f"=={major_version.version.major}.*" + ) + for version in versions: + if specifier.contains(version.version) and version.version > current: + nodejs_data[major_version.version_string] = version.version_string + break + + # check for a new major LTS to insert + if versions and versions[0].version.major > major_versions[0].version.major: + major_versions.insert( + 0, + VersionTuple(Version(str(versions[0].version.major)), f"v{versions[0].version.major}"), + ) + nodejs_data[major_versions[0].version_string] = versions[0].version_string + + versions_toml = "\n".join( + f'{major_version.version_string} = "{nodejs_data[major_version.version_string]}"' + for major_version in major_versions + ) + result_toml = f'url = "{NODEJS_DIST}"\n{versions_toml}\n' + + rich.print() # spacer + + if original_toml == result_toml: + rich.print("[green]Check complete, nodejs version unchanged.") + return + + rich.print("nodejs version updated.") + rich.print("Changes:") + rich.print() + + toml_relpath = toml_file_path.relative_to(DIR).as_posix() + diff_lines = difflib.unified_diff( + original_toml.splitlines(keepends=True), + result_toml.splitlines(keepends=True), + fromfile=toml_relpath, + tofile=toml_relpath, + ) + rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light")) + rich.print() + + if force: + toml_file_path.write_text(result_toml) + rich.print("[green]TOML file updated.") + else: + rich.print("[yellow]File left unchanged. Use --force flag to update.") + + +if __name__ == "__main__": + update_nodejs() diff --git a/bin/update_python_build_standalone.py b/bin/update_python_build_standalone.py new file mode 100755 index 000000000..b9ac37265 --- /dev/null +++ b/bin/update_python_build_standalone.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import json + +from cibuildwheel.extra import github_api_request +from cibuildwheel.util.python_build_standalone import ( + PythonBuildStandaloneAsset, + PythonBuildStandaloneReleaseData, +) +from cibuildwheel.util.resources import PYTHON_BUILD_STANDALONE_RELEASES + + +def main() -> None: + """ + This script updates the vendored list of release assets to the latest + version of astral-sh/python-build-standalone. + """ + + # Get the latest release tag from the GitHub API + latest_release = github_api_request("repos/astral-sh/python-build-standalone/releases/latest") + latest_tag = latest_release["tag_name"] + + # Get the list of assets for the latest release + github_assets = github_api_request( + f"repos/astral-sh/python-build-standalone/releases/tags/{latest_tag}" + )["assets"] + + assets = [ + PythonBuildStandaloneAsset(name=ga["name"], url=ga["browser_download_url"]) + for ga in github_assets + if ga["name"].endswith("install_only.tar.gz") + ] + + # Try to keep output order stable + assets = sorted(assets, key=lambda x: x["name"]) + + # Write the assets to the JSON file. One day, we might need to support + # multiple releases, but for now, we only support the latest one + json_file_contents = PythonBuildStandaloneReleaseData( + releases=[ + { + "tag": latest_tag, + "assets": assets, + } + ] + ) + + with PYTHON_BUILD_STANDALONE_RELEASES.open("w", encoding="utf-8") as f: + json.dump(json_file_contents, f, indent=2) + # Add a trailing newline, our pre-commit hook requires it + f.write("\n") + + print( + f"Updated {PYTHON_BUILD_STANDALONE_RELEASES.name} with {len(assets)} assets for tag {latest_tag}" + ) + + +if __name__ == "__main__": + main() diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 30859259a..d1058f452 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -1,24 +1,25 @@ #!/usr/bin/env python3 -from __future__ import annotations import copy import difflib import logging +import operator +import re +import tomllib +from collections.abc import Mapping, MutableMapping from pathlib import Path -from typing import Any, Union +from typing import Any, Final, Literal, TypedDict import click import requests import rich -import tomli from packaging.specifiers import Specifier from packaging.version import Version from rich.logging import RichHandler from rich.syntax import Syntax -from cibuildwheel.extra import dump_python_configurations -from cibuildwheel.typing import Final, Literal, TypedDict +from cibuildwheel.extra import dump_python_configurations, get_pyodide_xbuildenv_info log = logging.getLogger("cibw") @@ -44,13 +45,26 @@ class ConfigWinPP(TypedDict): url: str -class ConfigMacOS(TypedDict): +class ConfigWinGP(TypedDict): identifier: str version: str url: str -AnyConfig = Union[ConfigWinCP, ConfigWinPP, ConfigMacOS] +class ConfigApple(TypedDict): + identifier: str + version: str + url: str + + +class ConfigPyodide(TypedDict): + identifier: str + version: str + default_pyodide_version: str + node_version: str + + +AnyConfig = ConfigWinCP | ConfigWinPP | ConfigWinGP | ConfigApple | ConfigPyodide # The following set of "Versions" classes allow the initial call to the APIs to @@ -58,8 +72,7 @@ class ConfigMacOS(TypedDict): class WindowsVersions: - def __init__(self, arch_str: ArchStr) -> None: - + def __init__(self, arch_str: ArchStr, free_threaded: bool) -> None: response = requests.get("https://api.nuget.org/v3/index.json") response.raise_for_status() api_info = response.json() @@ -73,7 +86,11 @@ def __init__(self, arch_str: ArchStr) -> None: self.arch_str = arch_str self.arch = ARCH_DICT[arch_str] + self.free_threaded = free_threaded + package = PACKAGE_DICT[arch_str] + if free_threaded: + package = f"{package}-freethreaded" response = requests.get(f"{endpoint}{package}/index.json") response.raise_for_status() @@ -82,20 +99,20 @@ def __init__(self, arch_str: ArchStr) -> None: self.version_dict = {Version(v): v for v in cp_info["versions"]} def update_version_windows(self, spec: Specifier) -> ConfigWinCP | None: - # Specifier.filter selects all non pre-releases that match the spec, # unless there are only pre-releases, then it selects pre-releases # instead (like pip) unsorted_versions = spec.filter(self.version_dict) versions = sorted(unsorted_versions, reverse=True) - log.debug(f"Windows {self.arch} {spec} has {', '.join(str(v) for v in versions)}") + log.debug("Windows %s %s has %s", self.arch, spec, ", ".join(str(v) for v in versions)) if not versions: return None + flags = "t" if self.free_threaded else "" version = versions[0] - identifier = f"cp{version.major}{version.minor}-{self.arch}" + identifier = f"cp{version.major}{version.minor}{flags}-{self.arch}" return ConfigWinCP( identifier=identifier, version=self.version_dict[version], @@ -103,9 +120,78 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP | None: ) +class GraalPyVersions: + def __init__(self) -> None: + response = requests.get("https://api.github.com/repos/oracle/graalpython/releases") + response.raise_for_status() + + releases = response.json() + gp_version_re = re.compile(r"-(\d+\.\d+\.\d+)$") + cp_version_re = re.compile(r"Python (\d+\.\d+(?:\.\d+)?)") + for release in releases: + m = gp_version_re.search(release["tag_name"]) + if m: + release["graalpy_version"] = Version(m.group(1)) + m = cp_version_re.search(release["body"]) + if m: + release["python_version"] = Version(m.group(1)) + + self.releases = [r for r in releases if "graalpy_version" in r and "python_version" in r] + + def update_version(self, identifier: str, spec: Specifier) -> AnyConfig: + if "x86_64" in identifier or "amd64" in identifier: + arch = "x86_64" + elif "arm64" in identifier or "aarch64" in identifier: + arch = "aarch64" + else: + msg = f"{identifier} not supported yet on GraalPy" + raise RuntimeError(msg) + + gpspec_str = identifier.split("-")[0].split("_")[1] + gpspec = Specifier("==24.2.*") if gpspec_str == "242" else Specifier(f"=={gpspec_str}.*") + + releases = [r for r in self.releases if spec.contains(r["python_version"])] + releases = [r for r in self.releases if gpspec.contains(r["graalpy_version"])] + releases = sorted(releases, key=lambda r: r["graalpy_version"]) + + if not releases: + msg = f"GraalPy {arch} not found for {spec}!" + raise RuntimeError(msg) + + release = releases[-1] + version = release["python_version"] + gpversion = release["graalpy_version"] + + if "macosx" in identifier: + arch = "x86_64" if "x86_64" in identifier else "arm64" + config = ConfigApple + platform = "macos" + elif "win" in identifier: + arch = "aarch64" if "arm64" in identifier else "x86_64" + config = ConfigWinGP + platform = "windows" + else: + msg = "GraalPy provides downloads for macOS and Windows and is included for manylinux" + raise RuntimeError(msg) + + arch = "amd64" if arch == "x86_64" else "aarch64" + ext = "zip" if "win" in identifier else "tar.gz" + (url,) = ( + rf["browser_download_url"] + for rf in release["assets"] + if rf["name"].endswith(f"{platform}-{arch}.{ext}") + and rf["name"].startswith(f"graalpy-{gpversion.major}") + ) + + return config( + identifier=identifier, + version=f"{version.major}.{version.minor}", + url=url, + ) + + class PyPyVersions: def __init__(self, arch_str: ArchStr): - response = requests.get("https://downloads.python.org/pypy/versions.json") response.raise_for_status() @@ -121,7 +207,7 @@ def __init__(self, arch_str: ArchStr): ] self.arch = arch_str - def get_arch_file(self, release: dict[str, Any]) -> str: + def get_arch_file(self, release: Mapping[str, Any]) -> str: urls: list[str] = [ rf["download_url"] for rf in release["files"] @@ -131,11 +217,12 @@ def get_arch_file(self, release: dict[str, Any]) -> str: def update_version_windows(self, spec: Specifier) -> ConfigWinCP: releases = [r for r in self.releases if spec.contains(r["python_version"])] - releases = sorted(releases, key=lambda r: r["pypy_version"]) # type: ignore[no-any-return] + releases = sorted(releases, key=operator.itemgetter("pypy_version")) releases = [r for r in releases if self.get_arch_file(r)] if not releases: - raise RuntimeError(f"PyPy Win {self.arch} not found for {spec}! {self.releases}") + msg = f"PyPy Win {self.arch} not found for {spec}! {self.releases}" + raise RuntimeError(msg) version_arch = "win32" if self.arch == "32" else "win_amd64" @@ -151,27 +238,31 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP: url=url, ) - def update_version_macos(self, spec: Specifier) -> ConfigMacOS: - if self.arch != "64": - raise RuntimeError("Other archs not supported yet on macOS") + def update_version_macos(self, spec: Specifier) -> ConfigApple: + if self.arch not in {"64", "ARM64"}: + msg = f"'{self.arch}' arch not supported yet on macOS" + raise RuntimeError(msg) releases = [r for r in self.releases if spec.contains(r["python_version"])] - releases = sorted(releases, key=lambda r: r["pypy_version"]) # type: ignore[no-any-return] + releases = sorted(releases, key=operator.itemgetter("pypy_version")) if not releases: - raise RuntimeError(f"PyPy macOS {self.arch} not found for {spec}!") + msg = f"PyPy macOS {self.arch} not found for {spec}!" + raise RuntimeError(msg) release = releases[-1] version = release["python_version"] - identifier = f"pp{version.major}{version.minor}-macosx_x86_64" + arch = "x86_64" if self.arch == "64" else self.arch.lower() + identifier = f"pp{version.major}{version.minor}-macosx_{arch}" + arch = "x64" if self.arch == "64" else self.arch.lower() (url,) = ( rf["download_url"] for rf in release["files"] - if "" in rf["platform"] == "darwin" and rf["arch"] == "x64" + if "" in rf["platform"] == "darwin" and rf["arch"] == arch ) - return ConfigMacOS( + return ConfigApple( identifier=identifier, version=f"{version.major}.{version.minor}", url=url, @@ -180,7 +271,6 @@ def update_version_macos(self, spec: Specifier) -> ConfigMacOS: class CPythonVersions: def __init__(self) -> None: - response = requests.get( "https://www.python.org/api/v2/downloads/release/?is_published=true" ) @@ -190,24 +280,25 @@ def __init__(self) -> None: self.versions_dict: dict[Version, int] = {} for release in releases_info: - # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ") - version = Version(release["name"][7:]) + # Skip the pymanager releases + if not release["slug"].startswith("python"): + continue + + # Removing the prefix + version = Version(release["name"].removeprefix("Python ")) uri = int(release["resource_uri"].rstrip("/").split("/")[-1]) self.versions_dict[version] = uri def update_version_macos( self, identifier: str, version: Version, spec: Specifier - ) -> ConfigMacOS | None: - + ) -> ConfigApple | None: # see note above on Specifier.filter unsorted_versions = spec.filter(self.versions_dict) sorted_versions = sorted(unsorted_versions, reverse=True) - if version <= Version("3.8.9999"): - file_ident = "macosx10.9.pkg" - else: - file_ident = "macos11.pkg" + macver = "x10.9" if version <= Version("3.8.9999") else "11" + file_ident = f"macos{macver}.pkg" for new_version in sorted_versions: # Find the first patch version that contains the requested file @@ -220,7 +311,7 @@ def update_version_macos( urls = [rf["url"] for rf in file_info if file_ident in rf["url"]] if urls: - return ConfigMacOS( + return ConfigApple( identifier=identifier, version=f"{new_version.major}.{new_version.minor}", url=urls[0], @@ -229,51 +320,153 @@ def update_version_macos( return None +class CPythonIOSVersions: + def __init__(self) -> None: + response = requests.get( + "https://api.github.com/repos/beeware/Python-Apple-support/releases", + headers={ + "Accept": "application/vnd.github+json", + "X-Github-Api-Version": "2022-11-28", + }, + ) + response.raise_for_status() + + releases_info = response.json() + self.versions_dict: dict[Version, dict[int, str]] = {} + + # Each release has a name like "3.13-b4" + for release in releases_info: + py_version, build = release["name"].split("-") + version = Version(py_version) + self.versions_dict.setdefault(version, {}) + + # There are several release assets associated with each release; + # The name of the asset will be something like + # "Python-3.11-iOS-support.b4.tar.gz". Store all builds that are + # "-iOS-support" builds, retaining the download URL. + for asset in release["assets"]: + filename, build, _, _ = asset["name"].rsplit(".", 3) + if filename.endswith("-iOS-support"): + self.versions_dict[version][int(build[1:])] = asset["browser_download_url"] + + def update_version_ios(self, identifier: str, version: Version) -> ConfigApple | None: + # Return a config using the highest build number for the given version. + urls = [url for _, url in sorted(self.versions_dict.get(version, {}).items())] + if urls: + return ConfigApple( + identifier=identifier, + version=str(version), + url=urls[-1], + ) + + return None + + +class PyodideVersions: + def __init__(self) -> None: + xbuildenv_info = get_pyodide_xbuildenv_info() + self.releases = xbuildenv_info["releases"] + + def update_version_pyodide( + self, identifier: str, version: Version, spec: Specifier, node_version: str + ) -> ConfigPyodide | None: + # get releases that match the python version + releases = [ + r for r in self.releases.values() if spec.contains(Version(r["python_version"])) + ] + # sort by version, latest first + releases.sort(key=lambda r: Version(r["version"]), reverse=True) + + if not releases: + msg = f"Pyodide not found for {spec}!" + raise ValueError(msg) + + final_releases = [r for r in releases if not Version(r["version"]).is_prerelease] + + # prefer a final release if available, otherwise use the latest + # pre-release + release = final_releases[0] if final_releases else releases[0] + + return ConfigPyodide( + identifier=identifier, + version=str(version), + default_pyodide_version=release["version"], + node_version=node_version, + ) + + # This is a universal interface to all the above Versions classes. Given an # identifier, it updates a config dict. class AllVersions: def __init__(self) -> None: - self.windows_32 = WindowsVersions("32") - self.windows_64 = WindowsVersions("64") - self.windows_arm64 = WindowsVersions("ARM64") + self.windows_32 = WindowsVersions("32", False) + self.windows_t_32 = WindowsVersions("32", True) + self.windows_64 = WindowsVersions("64", False) + self.windows_t_64 = WindowsVersions("64", True) + self.windows_arm64 = WindowsVersions("ARM64", False) + self.windows_t_arm64 = WindowsVersions("ARM64", True) self.windows_pypy_64 = PyPyVersions("64") self.macos_cpython = CPythonVersions() self.macos_pypy = PyPyVersions("64") + self.macos_pypy_arm64 = PyPyVersions("ARM64") + + self.ios_cpython = CPythonIOSVersions() + + self.graalpy = GraalPyVersions() - def update_config(self, config: dict[str, str]) -> None: + self.pyodide = PyodideVersions() + + def update_config(self, config: MutableMapping[str, str]) -> None: identifier = config["identifier"] version = Version(config["version"]) spec = Specifier(f"=={version.major}.{version.minor}.*") - log.info(f"Reading in '{identifier}' -> {spec} @ {version}") + log.info("Reading in %r -> %s @ %s", str(identifier), spec, version) orig_config = copy.copy(config) config_update: AnyConfig | None = None # We need to use ** in update due to MyPy (probably a bug) - if "macos" in identifier: + if "macosx" in identifier: if identifier.startswith("cp"): config_update = self.macos_cpython.update_version_macos(identifier, version, spec) elif identifier.startswith("pp"): - config_update = self.macos_pypy.update_version_macos(spec) - elif "win32" in identifier: - if identifier.startswith("cp"): - config_update = self.windows_32.update_version_windows(spec) + if "macosx_x86_64" in identifier: + config_update = self.macos_pypy.update_version_macos(spec) + elif "macosx_arm64" in identifier: + config_update = self.macos_pypy_arm64.update_version_macos(spec) + elif identifier.startswith("gp"): + config_update = self.graalpy.update_version(identifier, spec) + elif "t-win32" in identifier and identifier.startswith("cp"): + config_update = self.windows_t_32.update_version_windows(spec) + elif "win32" in identifier and identifier.startswith("cp"): + config_update = self.windows_32.update_version_windows(spec) + elif "t-win_amd64" in identifier and identifier.startswith("cp"): + config_update = self.windows_t_64.update_version_windows(spec) elif "win_amd64" in identifier: if identifier.startswith("cp"): config_update = self.windows_64.update_version_windows(spec) elif identifier.startswith("pp"): config_update = self.windows_pypy_64.update_version_windows(spec) - elif "win_arm64" in identifier: - if identifier.startswith("cp"): - config_update = self.windows_arm64.update_version_windows(spec) + elif identifier.startswith("gp"): + config_update = self.graalpy.update_version(identifier, spec) + elif "t-win_arm64" in identifier and identifier.startswith("cp"): + config_update = self.windows_t_arm64.update_version_windows(spec) + elif "win_arm64" in identifier and identifier.startswith("cp"): + config_update = self.windows_arm64.update_version_windows(spec) + elif "ios" in identifier: + config_update = self.ios_cpython.update_version_ios(identifier, version) + elif "pyodide" in identifier: + config_update = self.pyodide.update_version_pyodide( + identifier, version, spec, config["node_version"] + ) assert config_update is not None, f"{identifier} not found!" config.update(**config_update) if config != orig_config: - log.info(f" Updated {orig_config} to {config}") + log.info(" Updated %s to %s", orig_config, config) @click.command() @@ -282,7 +475,6 @@ def update_config(self, config: dict[str, str]) -> None: "--level", default="INFO", type=click.Choice(["WARNING", "INFO", "DEBUG"], case_sensitive=False) ) def update_pythons(force: bool, level: str) -> None: - logging.basicConfig( level="INFO", format="%(message)s", @@ -296,7 +488,7 @@ def update_pythons(force: bool, level: str) -> None: original_toml = toml_file_path.read_text() with toml_file_path.open("rb") as f: - configs = tomli.load(f) + configs = tomllib.load(f) for config in configs["windows"]["python_configurations"]: all_versions.update_config(config) @@ -304,6 +496,12 @@ def update_pythons(force: bool, level: str) -> None: for config in configs["macos"]["python_configurations"]: all_versions.update_config(config) + for config in configs["ios"]["python_configurations"]: + all_versions.update_config(config) + + for config in configs["pyodide"]["python_configurations"]: + all_versions.update_config(config) + result_toml = dump_python_configurations(configs) rich.print() # spacer diff --git a/bin/update_readme_changelog.py b/bin/update_readme_changelog.py deleted file mode 100755 index 90b2ffd8b..000000000 --- a/bin/update_readme_changelog.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 - -import re -import sys -from pathlib import Path - -PROJECT_ROOT = Path(__file__).parent / ".." -CHANGELOG_FILE = PROJECT_ROOT / "docs" / "changelog.md" -README_FILE = PROJECT_ROOT / "README.md" - -# https://regexr.com/622ds -FIRST_5_CHANGELOG_ENTRIES_REGEX = re.compile(r"""(^###.*?(?=###)){5}""", re.DOTALL | re.MULTILINE) - -# https://regexr.com/622e5 -README_CHANGELOG_SECTION = re.compile( - r"""(?<=\n).*(?=)""", - re.DOTALL, -) - - -def main(): - changelog_text = CHANGELOG_FILE.read_text() - readme_text = README_FILE.read_text() - - mini_changelog_match = FIRST_5_CHANGELOG_ENTRIES_REGEX.search(changelog_text) - assert mini_changelog_match, "Failed to find the first few changelog entries" - - mini_changelog = "\n".join( - [ - "", - "", - "", - mini_changelog_match.group(0).strip(), - "", - "", - ] - ) - - if not re.search(README_CHANGELOG_SECTION, readme_text): - sys.exit("Changelog section not found in README") - - readme_text = re.sub(README_CHANGELOG_SECTION, mini_changelog, readme_text) - README_FILE.write_text(readme_text) - - -if __name__ == "__main__": - main() diff --git a/bin/update_virtualenv.py b/bin/update_virtualenv.py index 6cd293a31..6e38eadb6 100755 --- a/bin/update_virtualenv.py +++ b/bin/update_virtualenv.py @@ -1,22 +1,20 @@ #!/usr/bin/env python3 -from __future__ import annotations +import dataclasses import difflib import logging import subprocess +import tomllib from pathlib import Path -from typing import NamedTuple +from typing import Final import click import rich -import tomli from packaging.version import InvalidVersion, Version from rich.logging import RichHandler from rich.syntax import Syntax -from cibuildwheel.typing import Final - log = logging.getLogger("cibw") # Looking up the dir instead of using utils.resources_dir @@ -25,17 +23,18 @@ RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources" GET_VIRTUALENV_GITHUB: Final[str] = "https://github.com/pypa/get-virtualenv" -GET_VIRTUALENV_URL_TEMPLATE: Final[ - str -] = f"{GET_VIRTUALENV_GITHUB}/blob/{{version}}/public/virtualenv.pyz?raw=true" +GET_VIRTUALENV_URL_TEMPLATE: Final[str] = ( + f"{GET_VIRTUALENV_GITHUB}/blob/{{version}}/public/virtualenv.pyz?raw=true" +) -class VersionTuple(NamedTuple): +@dataclasses.dataclass(frozen=True, order=True) +class VersionTuple: version: Version version_string: str -def git_ls_remote_versions(url) -> list[VersionTuple]: +def git_ls_remote_versions(url: str) -> list[VersionTuple]: versions: list[VersionTuple] = [] tags = subprocess.run( ["git", "ls-remote", "--tags", url], check=True, text=True, capture_output=True @@ -47,14 +46,14 @@ def git_ls_remote_versions(url) -> list[VersionTuple]: try: version = Version(version_string) if version.is_devrelease: - log.info(f"Ignoring development release '{version}'") + log.info("Ignoring development release %r", str(version)) continue if version.is_prerelease: - log.info(f"Ignoring pre-release '{version}'") + log.info("Ignoring pre-release %r", str(version)) continue versions.append(VersionTuple(version, version_string)) except InvalidVersion: - log.warning(f"Ignoring ref '{ref}'") + log.warning("Ignoring ref %r", ref) versions.sort(reverse=True) return versions @@ -65,7 +64,6 @@ def git_ls_remote_versions(url) -> list[VersionTuple]: "--level", default="INFO", type=click.Choice(["WARNING", "INFO", "DEBUG"], case_sensitive=False) ) def update_virtualenv(force: bool, level: str) -> None: - logging.basicConfig( level="INFO", format="%(message)s", @@ -78,15 +76,20 @@ def update_virtualenv(force: bool, level: str) -> None: original_toml = toml_file_path.read_text() with toml_file_path.open("rb") as f: - loaded_file = tomli.load(f) - version = str(loaded_file["version"]) + configurations = tomllib.load(f) + default = configurations.pop("default") + version = str(default["version"]) versions = git_ls_remote_versions(GET_VIRTUALENV_GITHUB) if versions[0].version > Version(version): version = versions[0].version_string - result_toml = ( - f'version = "{version}"\n' - f'url = "{GET_VIRTUALENV_URL_TEMPLATE.format(version=version)}"\n' + configurations["default"] = { + "version": version, + "url": GET_VIRTUALENV_URL_TEMPLATE.format(version=version), + } + result_toml = "".join( + f'{key} = {{ version = "{value["version"]}", url = "{value["url"]}" }}\n' + for key, value in configurations.items() ) rich.print() # spacer diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py index 3a5935a2d..528787cfc 100644 --- a/cibuildwheel/__init__.py +++ b/cibuildwheel/__init__.py @@ -1 +1 @@ -__version__ = "2.3.1" +__version__ = "3.0.0" diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index d5323e69a..934d80c4e 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -1,33 +1,89 @@ import argparse +import contextlib +import dataclasses +import functools import os import shutil import sys +import tarfile import textwrap +import time +import traceback +import typing +from collections.abc import Generator, Iterable, Sequence from pathlib import Path from tempfile import mkdtemp -from typing import List, Set, Union +from typing import Any, Literal, TextIO import cibuildwheel -import cibuildwheel.linux -import cibuildwheel.macos import cibuildwheel.util -import cibuildwheel.windows +from cibuildwheel import errors from cibuildwheel.architecture import Architecture, allowed_architectures_check +from cibuildwheel.ci import CIProvider, detect_ci_provider, fix_ansi_codes_for_github_actions from cibuildwheel.logger import log from cibuildwheel.options import CommandLineArguments, Options, compute_options -from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never -from cibuildwheel.util import ( - CIBW_CACHE_PATH, - BuildSelector, - Unbuffered, - detect_ci_provider, -) +from cibuildwheel.platforms import ALL_PLATFORM_MODULES, get_build_identifiers +from cibuildwheel.selector import BuildSelector, EnableGroup, selector_matches +from cibuildwheel.typing import PLATFORMS, PlatformName +from cibuildwheel.util.file import CIBW_CACHE_PATH +from cibuildwheel.util.helpers import strtobool + + +@dataclasses.dataclass +class GlobalOptions: + print_traceback_on_error: bool = True # decides what happens when errors are hit. + + +@dataclasses.dataclass(frozen=True) +class FileReport: + name: str + size: str + + +# Taken from https://stackoverflow.com/a/107717 +class Unbuffered: + def __init__(self, stream: TextIO) -> None: + self.stream = stream + + def write(self, data: str) -> None: + self.stream.write(data) + self.stream.flush() + + def writelines(self, data: Iterable[str]) -> None: + self.stream.writelines(data) + self.stream.flush() + + def __getattr__(self, attr: str) -> Any: + return getattr(self.stream, attr) def main() -> None: - platform: PlatformName + global_options = GlobalOptions() + try: + main_inner(global_options) + except errors.FatalError as e: + message = e.args[0] + if log.step_active: + log.step_end_with_error(message) + else: + log.error(message) - parser = argparse.ArgumentParser( + if global_options.print_traceback_on_error: + traceback.print_exc(file=sys.stderr) + + sys.exit(e.return_code) + + +def main_inner(global_options: GlobalOptions) -> None: + """ + `main_inner` is the same as `main`, but it raises FatalError exceptions + rather than exiting directly. + """ + + make_parser = functools.partial(argparse.ArgumentParser, allow_abbrev=False) + if sys.version_info >= (3, 14): + make_parser = functools.partial(make_parser, color=True, suggest_on_error=True) + parser = make_parser( description="Build wheels for all the platforms.", epilog=""" Most options are supplied via environment variables or in @@ -38,14 +94,14 @@ def main() -> None: parser.add_argument( "--platform", - choices=["auto", "linux", "macos", "windows"], - default=os.environ.get("CIBW_PLATFORM", "auto"), + choices=["auto", "linux", "macos", "windows", "pyodide", "ios"], + default=None, help=""" - Platform to build for. Use this option to override the - auto-detected platform or to run cibuildwheel on your development - machine. Specifying "macos" or "windows" only works on that - operating system, but "linux" works on all three, as long as - Docker is installed. Default: auto. + Platform to build for. Use this option to override the auto-detected + platform. Specifying "macos" or "windows" only works on that + operating system. "linux" works on any desktop OS, as long as + Docker/Podman is installed. "pyodide" only works on linux and macOS. + "ios" only work on macOS. Default: auto. """, ) @@ -63,8 +119,31 @@ def main() -> None: """, ) + enable_groups_str = ", ".join(g.value for g in EnableGroup) + parser.add_argument( + "--enable", + action="http://github.com/append", + default=[], + metavar="GROUP", + help=f""" + Enable an additional category of builds. Use multiple times to select multiple groups. Choices: {enable_groups_str}. + """, + ) + + parser.add_argument( + "--only", + default=None, + help=""" + Force a single wheel build when given an identifier. Overrides + CIBW_BUILD/CIBW_SKIP. --platform and --arch cannot be specified + if this is given. + """, + ) + parser.add_argument( "--output-dir", + type=Path, + default=Path(os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse")), help="Destination folder for the wheels. Default: wheelhouse.", ) @@ -72,20 +151,26 @@ def main() -> None: "--config-file", default="", help=""" - TOML config file. Default: "", meaning {package}/pyproject.toml, - if it exists. + TOML config file. Default: "", meaning {package}/pyproject.toml, if + it exists. To refer to a project inside your project, use {package}; + this matters if you build from an SDist. """, ) parser.add_argument( "package_dir", - default=".", + metavar="PACKAGE", + default=Path(), + type=Path, nargs="?", help=""" - Path to the package that you want wheels for. Must be a subdirectory of - the working directory. When set, the working directory is still - considered the 'project' and is copied into the Docker container on - Linux. Default: the working directory. + Path to the package that you want wheels for. Default: the working + directory. Can be a directory inside the working directory, or an + sdist. When set to a directory, the working directory is still + considered the 'project' and is copied into the build container + on Linux. When set to a tar.gz sdist file, --config-file + and --output-dir are relative to the current directory, and other + paths are relative to the expanded SDist directory. """, ) @@ -102,61 +187,160 @@ def main() -> None: ) parser.add_argument( - "--prerelease-pythons", + "--debug-traceback", action="http://github.com/store_true", - help="Enable pre-release Python versions if available.", + default=strtobool(os.environ.get("CIBW_DEBUG_TRACEBACK", "0")), + help="Print a full traceback for all errors", ) - args = parser.parse_args(namespace=CommandLineArguments()) + args = CommandLineArguments(**vars(parser.parse_args())) + + global_options.print_traceback_on_error = args.debug_traceback + + args.package_dir = args.package_dir.resolve() + + # This are always relative to the base directory, even in SDist builds + args.output_dir = args.output_dir.resolve() + + # Standard builds if a directory or non-existent path is given + if not args.package_dir.is_file() and not args.package_dir.name.endswith("tar.gz"): + build_in_directory(args) + return + + # Tarfile builds require extraction and changing the directory + temp_dir = Path(mkdtemp(prefix="cibw-sdist-")).resolve(strict=True) + try: + with tarfile.open(args.package_dir) as tar: + tar.extractall(path=temp_dir) + + # The extract directory is now the project dir + try: + (project_dir,) = temp_dir.iterdir() + except ValueError: + msg = "invalid sdist: didn't contain a single dir" + raise SystemExit(msg) from None + + # This is now the new package dir + args.package_dir = project_dir.resolve() - if args.platform != "auto": - platform = args.platform + with contextlib.chdir(project_dir): + build_in_directory(args) + finally: + # avoid https://github.com/python/cpython/issues/86962 by performing + # cleanup manually + shutil.rmtree(temp_dir, ignore_errors=sys.platform.startswith("win")) + if temp_dir.exists(): + log.warning(f"Can't delete temporary folder '{temp_dir}'") + + +def _compute_platform_only(only: str) -> PlatformName: + if "linux_" in only: + return "linux" + if "macosx_" in only: + return "macos" + if "win_" in only or "win32" in only: + return "windows" + if "pyodide_" in only: + return "pyodide" + if "ios_" in only: + return "ios" + msg = f"Invalid --only='{only}', must be a build selector with a known platform" + raise errors.ConfigurationError(msg) + + +def _compute_platform_auto() -> PlatformName: + if sys.platform.startswith("linux"): + return "linux" + elif sys.platform == "darwin": + return "macos" + elif sys.platform == "win32": + return "windows" else: - ci_provider = detect_ci_provider() - if ci_provider is None: - print( - textwrap.dedent( - """ - cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server; - Travis CI, AppVeyor, Azure Pipelines, GitHub Actions, CircleCI, and Gitlab are - supported. You can run on your development machine or other CI providers using the - --platform argument. Check --help output for more information. - """ - ), - file=sys.stderr, - ) - sys.exit(2) - if sys.platform.startswith("linux"): - platform = "linux" - elif sys.platform == "darwin": - platform = "macos" - elif sys.platform == "win32": - platform = "windows" - else: - print( - 'cibuildwheel: Unable to detect platform from "sys.platform" in a CI environment. You can run ' - "cibuildwheel using the --platform argument. Check --help output for more information.", - file=sys.stderr, - ) - sys.exit(2) + msg = ( + 'Unable to detect platform from "sys.platform". cibuildwheel doesn\'t ' + "support building wheels for this platform. You might be able to build for a different " + "platform using the --platform argument. Check --help output for more information." + ) + raise errors.ConfigurationError(msg) + + +def _compute_platform(args: CommandLineArguments) -> PlatformName: + platform_option_value = args.platform or os.environ.get("CIBW_PLATFORM", "") or "auto" + + if args.only and args.platform is not None: + msg = "--platform cannot be specified with --only, it is computed from --only" + raise errors.ConfigurationError(msg) + if args.only and args.archs is not None: + msg = "--arch cannot be specified with --only, it is computed from --only" + raise errors.ConfigurationError(msg) + + if platform_option_value not in PLATFORMS | {"auto"}: + msg = f"Unsupported platform: {platform_option_value}" + raise errors.ConfigurationError(msg) - if platform not in PLATFORMS: - print(f"cibuildwheel: Unsupported platform: {platform}", file=sys.stderr) - sys.exit(2) + if args.only: + return _compute_platform_only(args.only) + elif platform_option_value != "auto": + return typing.cast(PlatformName, platform_option_value) - options = compute_options(platform=platform, command_line_arguments=args) + return _compute_platform_auto() + + +@contextlib.contextmanager +def print_new_wheels(msg: str, output_dir: Path) -> Generator[None, None, None]: + """ + Prints the new items in a directory upon exiting. The message to display + can include {n} for number of wheels, {s} for total number of seconds, + and/or {m} for total number of minutes. Does not print anything if this + exits via exception. + """ + + start_time = time.time() + existing_contents = set(output_dir.iterdir()) + yield + final_contents = set(output_dir.iterdir()) + + new_contents = [ + FileReport(wheel.name, f"{(wheel.stat().st_size + 1023) // 1024:,d}") + for wheel in final_contents - existing_contents + ] + + if not new_contents: + return + + max_name_len = max(len(f.name) for f in new_contents) + max_size_len = max(len(f.size) for f in new_contents) + n = len(new_contents) + s = time.time() - start_time + m = s / 60 + print( + msg.format(n=n, s=s, m=m), + *sorted( + f" {f.name:<{max_name_len}s} {f.size:>{max_size_len}s} kB" for f in new_contents + ), + sep="\n", + ) + + +def build_in_directory(args: CommandLineArguments) -> None: + platform: PlatformName = _compute_platform(args) + if platform == "pyodide" and sys.platform == "win32": + msg = "Building for pyodide is not supported on Windows" + raise errors.ConfigurationError(msg) + + options = compute_options(platform=platform, command_line_arguments=args, env=os.environ) package_dir = options.globals.package_dir package_files = {"setup.py", "setup.cfg", "pyproject.toml"} if not any(package_dir.joinpath(name).exists() for name in package_files): names = ", ".join(sorted(package_files, reverse=True)) - msg = f"cibuildwheel: Could not find any of {{{names}}} at root of package" - print(msg, file=sys.stderr) - sys.exit(2) + msg = f"Could not find any of {{{names}}} at root of package" + raise errors.ConfigurationError(msg) + platform_module = ALL_PLATFORM_MODULES[platform] identifiers = get_build_identifiers( - platform=platform, + platform_module=platform_module, build_selector=options.globals.build_selector, architectures=options.globals.architectures, ) @@ -167,11 +351,12 @@ def main() -> None: sys.exit(0) # Add CIBUILDWHEEL environment variable - # This needs to be passed on to the docker container in linux.py os.environ["CIBUILDWHEEL"] = "1" - # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' - sys.stdout = Unbuffered(sys.stdout) # type: ignore[assignment] + # Python is buffering by default when running on the CI platforms, giving + # problems interleaving subprocess call output with unflushed calls to + # 'print' + sys.stdout = Unbuffered(sys.stdout) # create the cache dir before it gets printed & builds performed CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True) @@ -182,42 +367,32 @@ def main() -> None: options.check_for_invalid_configuration(identifiers) allowed_architectures_check(platform, options.globals.architectures) except ValueError as err: - print("cibuildwheel:", *err.args, file=sys.stderr) - sys.exit(4) + raise errors.DeprecationError(*err.args) from err if not identifiers: - print( - f"cibuildwheel: No build identifiers selected: {options.globals.build_selector}", - file=sys.stderr, - ) - if not args.allow_empty: - sys.exit(3) + message = f"No build identifiers selected: {options.globals.build_selector}" + if options.globals.allow_empty: + print(f"cibuildwheel: {message}", file=sys.stderr) + else: + raise errors.NothingToDoError(message) output_dir = options.globals.output_dir - if not output_dir.exists(): - output_dir.mkdir(parents=True) + output_dir.mkdir(parents=True, exist_ok=True) tmp_path = Path(mkdtemp(prefix="cibw-run-")).resolve(strict=True) try: - with cibuildwheel.util.print_new_wheels( - "\n{n} wheels produced in {m:.0f} minutes:", output_dir - ): - if platform == "linux": - cibuildwheel.linux.build(options, tmp_path) - elif platform == "windows": - cibuildwheel.windows.build(options, tmp_path) - elif platform == "macos": - cibuildwheel.macos.build(options, tmp_path) - else: - assert_never(platform) + with print_new_wheels("\n{n} wheels produced in {m:.0f} minutes:", output_dir): + platform_module.build(options, tmp_path) finally: + # avoid https://github.com/python/cpython/issues/86962 by performing + # cleanup manually shutil.rmtree(tmp_path, ignore_errors=sys.platform.startswith("win")) if tmp_path.exists(): - log.warning(f"Can't delete temporary folder '{str(tmp_path)}'") + log.warning(f"Can't delete temporary folder '{tmp_path}'") -def print_preamble(platform: str, options: Options, identifiers: List[str]) -> None: +def print_preamble(platform: str, options: Options, identifiers: Sequence[str]) -> None: print( textwrap.dedent( """ @@ -232,49 +407,35 @@ def print_preamble(platform: str, options: Options, identifiers: List[str]) -> N print(f"cibuildwheel version {cibuildwheel.__version__}\n") print("Build options:") - print(f" platform: {platform!r}") - print(textwrap.indent(options.summary(identifiers), " ")) + print(f" platform: {platform}") + options_summary = textwrap.indent(options.summary(identifiers), " ") + if detect_ci_provider() == CIProvider.github_actions: + options_summary = fix_ansi_codes_for_github_actions(options_summary) + print(options_summary) + print() print(f"Cache folder: {CIBW_CACHE_PATH}") + print() - warnings = detect_warnings(platform=platform, options=options, identifiers=identifiers) - if warnings: - print("\nWarnings:") - for warning in warnings: - print(" " + warning) + warnings = detect_warnings(options=options, identifiers=identifiers) + for warning in warnings: + log.warning(warning) - print("\nHere we go!\n") + print("Here we go!\n") -def get_build_identifiers( - platform: PlatformName, build_selector: BuildSelector, architectures: Set[Architecture] -) -> List[str]: - python_configurations: Union[ - List[cibuildwheel.linux.PythonConfiguration], - List[cibuildwheel.windows.PythonConfiguration], - List[cibuildwheel.macos.PythonConfiguration], - ] +def detect_warnings(*, options: Options, identifiers: Iterable[str]) -> list[str]: + warnings = [] - if platform == "linux": - python_configurations = cibuildwheel.linux.get_python_configurations( - build_selector, architectures - ) - elif platform == "windows": - python_configurations = cibuildwheel.windows.get_python_configurations( - build_selector, architectures + python_version_deprecation = ((3, 11), 3) + if sys.version_info[:2] < python_version_deprecation[0]: + python_version = ".".join(map(str, python_version_deprecation[0])) + msg = ( + f"cibuildwheel {python_version_deprecation[1]} will require Python {python_version}+, " + "please upgrade the Python version used to run cibuildwheel. " + "This does not affect the versions you can target when building wheels. See: https://cibuildwheel.pypa.io/en/stable/#what-does-it-do" ) - elif platform == "macos": - python_configurations = cibuildwheel.macos.get_python_configurations( - build_selector, architectures - ) - else: - assert_never(platform) - - return [config.identifier for config in python_configurations] - - -def detect_warnings(platform: str, options: Options, identifiers: List[str]) -> List[str]: - warnings = [] + warnings.append(msg) # warn about deprecated {python} and {pip} for option_name in ["test_command", "before_build"]: @@ -283,9 +444,78 @@ def detect_warnings(platform: str, options: Options, identifiers: List[str]) -> if any(o and ("{python}" in o or "{pip}" in o) for o in option_values): # Reminder: in an f-string, double braces means literal single brace msg = ( - f"{option_name}: '{{python}}' and '{{pip}}' are no longer needed, " - "and will be removed in a future release. Simply use 'python' or 'pip' instead." + f"{option_name}: '{{python}}' and '{{pip}}' are no longer supported " + "and have been removed in cibuildwheel 3. Simply use 'python' or 'pip' instead." ) + raise errors.ConfigurationError(msg) + + build_selector = options.globals.build_selector + test_selector = options.globals.test_selector + + all_valid_identifiers = [ + config.identifier + for module in ALL_PLATFORM_MODULES.values() + for config in module.all_python_configurations() + ] + + enabled_selector = BuildSelector( + build_config="*", skip_config="", enable=options.globals.build_selector.enable + ) + all_enabled_identifiers = [ + identifier for identifier in all_valid_identifiers if enabled_selector(identifier) + ] + + warnings += check_for_invalid_selectors( + selector_name="build", + selector_value=build_selector.build_config, + all_valid_identifiers=all_valid_identifiers, + all_enabled_identifiers=all_enabled_identifiers, + ) + warnings += check_for_invalid_selectors( + selector_name="skip", + selector_value=build_selector.skip_config, + all_valid_identifiers=all_valid_identifiers, + all_enabled_identifiers=all_enabled_identifiers, + ) + warnings += check_for_invalid_selectors( + selector_name="test_skip", + selector_value=test_selector.skip_config, + all_valid_identifiers=all_valid_identifiers, + all_enabled_identifiers=all_enabled_identifiers, + ) + + return warnings + + +def check_for_invalid_selectors( + *, + selector_name: Literal["build", "skip", "test_skip"], + selector_value: str, + all_valid_identifiers: Sequence[str], + all_enabled_identifiers: Sequence[str], +) -> list[str]: + warnings = [] + + for selector in selector_value.split(): + if not any(selector_matches(selector, i) for i in all_enabled_identifiers): + msg = f"Invalid {selector_name} selector: {selector!r}. " + error_type: type = errors.ConfigurationError + + if any(selector_matches(selector, i) for i in all_valid_identifiers): + msg += "This selector matches a group that wasn't enabled. Enable it using the `enable` option or remove this selector. " + + if "p2" in selector or "p35" in selector: + msg += f"cibuildwheel 3.x no longer supports Python < 3.8. Please use the 1.x series or update `{selector_name}`. " + error_type = errors.DeprecationError + if "p36" in selector or "p37" in selector: + msg += f"cibuildwheel 3.x no longer supports Python < 3.8. Please use the 2.x series or update `{selector_name}`. " + error_type = errors.DeprecationError + + if selector_name == "build": + raise error_type(msg) + + msg += "This selector will have no effect. " + warnings.append(msg) return warnings diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index 9d39d5533..69c164b27 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -1,111 +1,216 @@ -import functools import platform as platform_module import re -from enum import Enum -from typing import Set - -from .typing import Literal, PlatformName, assert_never - -PRETTY_NAMES = {"linux": "Linux", "macos": "macOS", "windows": "Windows"} - - -@functools.total_ordering -class Architecture(Enum): - value: str - +import shutil +import subprocess +import sys +import typing +from collections.abc import Set +from enum import StrEnum, auto +from typing import Final, Literal + +from cibuildwheel import errors + +from .typing import PlatformName + +PRETTY_NAMES: Final[dict[PlatformName, str]] = { + "linux": "Linux", + "macos": "macOS", + "windows": "Windows", + "pyodide": "Pyodide", + "ios": "iOS", +} + +ARCH_SYNONYMS: Final[list[dict[PlatformName, str | None]]] = [ + {"linux": "x86_64", "macos": "x86_64", "windows": "AMD64"}, + {"linux": "i686", "macos": None, "windows": "x86"}, + {"linux": "aarch64", "macos": "arm64", "windows": "ARM64"}, +] + + +def _check_aarch32_el0() -> bool: + """Check if running armv7l natively on aarch64 is supported""" + if not sys.platform.startswith("linux"): + return False + if platform_module.machine() != "aarch64": + return False + executable = shutil.which("linux32") + if executable is None: + return False + check = subprocess.run([executable, "uname", "-m"], check=False, capture_output=True, text=True) + return check.returncode == 0 and check.stdout.startswith("armv") + + +@typing.final +class Architecture(StrEnum): # mac/linux archs - x86_64 = "x86_64" + x86_64 = auto() # linux archs - i686 = "i686" - aarch64 = "aarch64" - ppc64le = "ppc64le" - s390x = "s390x" + i686 = auto() + aarch64 = auto() + ppc64le = auto() + s390x = auto() + armv7l = auto() + riscv64 = auto() # mac archs - universal2 = "universal2" - arm64 = "arm64" + universal2 = auto() + arm64 = auto() # windows archs - x86 = "x86" + x86 = auto() AMD64 = "AMD64" ARM64 = "ARM64" - # Allow this to be sorted - def __lt__(self, other: "Architecture") -> bool: - return self.value < other.value + # WebAssembly + wasm32 = auto() + + # iOS "multiarch" architectures that include both + # the CPU architecture and the ABI. + arm64_iphoneos = auto() + arm64_iphonesimulator = auto() + x86_64_iphonesimulator = auto() @staticmethod - def parse_config(config: str, platform: PlatformName) -> "Set[Architecture]": + def parse_config(config: str, platform: PlatformName) -> "set[Architecture]": result = set() for arch_str in re.split(r"[\s,]+", config): - if arch_str == "auto": - result |= Architecture.auto_archs(platform=platform) - elif arch_str == "native": - result.add(Architecture(platform_module.machine())) - elif arch_str == "all": - result |= Architecture.all_archs(platform=platform) - elif arch_str == "auto64": - result |= Architecture.bitness_archs(platform=platform, bitness="64") - elif arch_str == "auto32": - result |= Architecture.bitness_archs(platform=platform, bitness="32") - else: - result.add(Architecture(arch_str)) + match arch_str: + case "auto": + result |= Architecture.auto_archs(platform=platform) + case "native": + if native_arch := Architecture.native_arch(platform=platform): + result.add(native_arch) + case "all": + result |= Architecture.all_archs(platform=platform) + case "auto64": + result |= Architecture.bitness_archs(platform=platform, bitness="64") + case "auto32": + result |= Architecture.bitness_archs(platform=platform, bitness="32") + case _: + try: + result.add(Architecture(arch_str)) + except ValueError as e: + msg = f"Invalid architecture '{arch_str}'" + raise errors.ConfigurationError(msg) from e return result @staticmethod - def auto_archs(platform: PlatformName) -> "Set[Architecture]": - native_architecture = Architecture(platform_module.machine()) - result = {native_architecture} + def native_arch(platform: PlatformName) -> "Architecture | None": + native_machine = platform_module.machine() + native_architecture = Architecture(native_machine) + + # Cross-platform support. Used for --print-build-identifiers or docker builds. + host_platform: PlatformName = ( + "windows" + if sys.platform.startswith("win") + else ("macos" if sys.platform.startswith("darwin") else "linux") + ) + + if platform == "pyodide": + return Architecture.wasm32 + elif platform == "ios": + # Can only build for iOS on macOS. The "native" architecture is the + # simulator for the macOS native platform. + if host_platform == "macos": + if native_architecture == Architecture.x86_64: + return Architecture.x86_64_iphonesimulator + else: + return Architecture.arm64_iphonesimulator + else: + return None + + # we might need to rename the native arch to the machine we're running + # on, as the same arch can have different names on different platforms + if host_platform != platform: + for arch_synonym in ARCH_SYNONYMS: + if native_machine == arch_synonym.get(host_platform): + synonym = arch_synonym[platform] + + if synonym is None: + # can't build anything on this platform + return None - if platform == "linux" and native_architecture == Architecture.x86_64: - # x86_64 machines can run i686 docker containers - result.add(Architecture.i686) + native_architecture = Architecture(synonym) - if platform == "windows" and native_architecture == Architecture.AMD64: - result.add(Architecture.x86) + return native_architecture - if platform == "macos" and native_architecture == Architecture.arm64: - # arm64 can build and test both archs of a universal2 wheel. - result.add(Architecture.universal2) + @staticmethod + def auto_archs(platform: PlatformName) -> "set[Architecture]": + native_arch = Architecture.native_arch(platform) + if native_arch is None: + return set() # can't build anything on this platform + result = {native_arch} + + match platform: + case "windows" if Architecture.AMD64 in result: + result.add(Architecture.x86) + case "ios" if native_arch == Architecture.arm64_iphonesimulator: + # Also build the device wheel if we're on ARM64. + result.add(Architecture.arm64_iphoneos) return result @staticmethod - def all_archs(platform: PlatformName) -> "Set[Architecture]": - if platform == "linux": - return { + def all_archs(platform: PlatformName) -> "set[Architecture]": + all_archs_map = { + "linux": { Architecture.x86_64, Architecture.i686, Architecture.aarch64, Architecture.ppc64le, Architecture.s390x, - } - elif platform == "macos": - return {Architecture.x86_64, Architecture.arm64, Architecture.universal2} - elif platform == "windows": - return {Architecture.x86, Architecture.AMD64, Architecture.ARM64} - else: - assert_never(platform) + Architecture.armv7l, + Architecture.riscv64, + }, + "macos": {Architecture.x86_64, Architecture.arm64, Architecture.universal2}, + "windows": {Architecture.x86, Architecture.AMD64, Architecture.ARM64}, + "pyodide": {Architecture.wasm32}, + "ios": { + Architecture.x86_64_iphonesimulator, + Architecture.arm64_iphonesimulator, + Architecture.arm64_iphoneos, + }, + } + return all_archs_map[platform] @staticmethod - def bitness_archs(platform: PlatformName, bitness: Literal["64", "32"]) -> "Set[Architecture]": - archs_32 = {Architecture.i686, Architecture.x86} - auto_archs = Architecture.auto_archs(platform) - - if bitness == "64": - return auto_archs - archs_32 - elif bitness == "32": - return auto_archs & archs_32 - else: - assert_never(bitness) + def bitness_archs(platform: PlatformName, bitness: Literal["64", "32"]) -> "set[Architecture]": + # This map maps 64-bit architectures to their 32-bit equivalents. + archs_map = { + Architecture.x86_64: Architecture.i686, + Architecture.AMD64: Architecture.x86, + Architecture.aarch64: Architecture.armv7l, + } + native_arch = Architecture.native_arch(platform) + + if native_arch is None: + return set() # can't build anything on this platform + + if native_arch == Architecture.wasm32: + return {native_arch} if bitness == "32" else set() + + match bitness: + case "64": + return {native_arch} if native_arch not in archs_map.values() else set() + case "32": + if native_arch in archs_map.values(): + return {native_arch} + elif native_arch in archs_map and platform in {"linux", "windows"}: + if native_arch == Architecture.aarch64 and not _check_aarch32_el0(): + # If we're on aarch64, skip if we cannot build armv7l wheels. + return set() + return {archs_map[native_arch]} + else: + return set() + case _: + typing.assert_never(bitness) def allowed_architectures_check( platform: PlatformName, architectures: Set[Architecture], ) -> None: - allowed_architectures = Architecture.all_archs(platform) msg = f"{PRETTY_NAMES[platform]} only supports {sorted(allowed_architectures)} at the moment." diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 9eb5eac34..3d1ac1185 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,26 +1,31 @@ +import dataclasses import subprocess -from typing import Callable, Dict, List, NamedTuple, Optional, Sequence +from collections.abc import ( + Callable, + Iterable, + Mapping, + Sequence, +) import bashlex # a function that takes a command and the environment, and returns the result -EnvironmentExecutor = Callable[[List[str], Dict[str, str]], str] +EnvironmentExecutor = Callable[[list[str], dict[str, str]], str] -def local_environment_executor(command: List[str], env: Dict[str, str]) -> str: - return subprocess.run( - command, env=env, universal_newlines=True, stdout=subprocess.PIPE, check=True - ).stdout +def local_environment_executor(command: Sequence[str], env: Mapping[str, str]) -> str: + return subprocess.run(command, env=env, text=True, stdout=subprocess.PIPE, check=True).stdout -class NodeExecutionContext(NamedTuple): - environment: Dict[str, str] +@dataclasses.dataclass(frozen=True, kw_only=True) +class NodeExecutionContext: + environment: dict[str, str] input: str executor: EnvironmentExecutor def evaluate( - value: str, environment: Dict[str, str], executor: Optional[EnvironmentExecutor] = None + value: str, environment: Mapping[str, str], executor: EnvironmentExecutor | None = None ) -> str: if not value: # empty string evaluates to empty string @@ -30,14 +35,17 @@ def evaluate( command_node = bashlex.parsesingle(value) if len(command_node.parts) != 1: - raise ValueError(f'"{value}" has too many parts') + msg = f"{value!r} has too many parts" + raise ValueError(msg) value_word_node = command_node.parts[0] return evaluate_node( value_word_node, context=NodeExecutionContext( - environment=environment, input=value, executor=executor or local_environment_executor + environment=dict(environment), + input=value, + executor=executor or local_environment_executor, ), ) @@ -52,7 +60,8 @@ def evaluate_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: elif node.kind == "parameter": return evaluate_parameter_node(node, context=context) else: - raise ValueError(f'Unsupported bash construct: "{node.kind}"') + msg = f"Unsupported bash construct: {node.kind!r}" + raise ValueError(msg) def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: @@ -63,10 +72,8 @@ def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> part_value = evaluate_node(part, context=context) if part_string not in value: - raise RuntimeError( - f'bash parse failed. part "{part_string}" not found in "{value}". ' - f'Word was "{node.word}". Full input was "{context.input}"' - ) + msg = f"bash parse failed. part {part_string!r} not found in {value!r}. Word was {node.word!r}. Full input was {context.input!r}" + raise RuntimeError(msg) value = value.replace(part_string, part_value, 1) @@ -93,15 +100,17 @@ def evaluate_nodes_as_compound_command( result += evaluate_command_node(node, context=context) elif node.kind == "operator": if node.op != ";": - raise ValueError(f'Unsupported bash operator: "{node.op}"') + msg = f"Unsupported bash operator: {node.op!r}" + raise ValueError(msg) else: - raise ValueError(f'Unsupported bash node in compound command: "{node.kind}"') + msg = f"Unsupported bash node in compound command: {node.kind!r}" + raise ValueError(msg) return result def evaluate_nodes_as_simple_command( - nodes: List[bashlex.ast.node], context: NodeExecutionContext + nodes: Iterable[bashlex.ast.node], context: NodeExecutionContext ) -> str: command = [evaluate_node(part, context=context) for part in nodes] return context.executor(command, context.environment) diff --git a/cibuildwheel/ci.py b/cibuildwheel/ci.py new file mode 100644 index 000000000..03205b7a4 --- /dev/null +++ b/cibuildwheel/ci.py @@ -0,0 +1,69 @@ +import os +import re +from enum import Enum + +from .util.helpers import strtobool + + +class CIProvider(Enum): + # official support + travis_ci = "travis" + circle_ci = "circle_ci" + azure_pipelines = "azure_pipelines" + github_actions = "github_actions" + gitlab = "gitlab" + cirrus_ci = "cirrus_ci" + + # unofficial support + appveyor = "appveyor" + + other = "other" + + +def detect_ci_provider() -> CIProvider | None: + if "TRAVIS" in os.environ: + return CIProvider.travis_ci + elif "APPVEYOR" in os.environ: + return CIProvider.appveyor + elif "CIRCLECI" in os.environ: + return CIProvider.circle_ci + elif "AZURE_HTTP_USER_AGENT" in os.environ: + return CIProvider.azure_pipelines + elif "GITHUB_ACTIONS" in os.environ: + return CIProvider.github_actions + elif "GITLAB_CI" in os.environ: + return CIProvider.gitlab + elif "CIRRUS_CI" in os.environ: + return CIProvider.cirrus_ci + elif strtobool(os.environ.get("CI", "false")): + return CIProvider.other + else: + return None + + +def fix_ansi_codes_for_github_actions(text: str) -> str: + """ + Github Actions forgets the current ANSI style on every new line. This + function repeats the current ANSI style on every new line. + """ + ansi_code_regex = re.compile(r"(\033\[[0-9;]*m)") + ansi_codes: list[str] = [] + output = "" + + for line in text.splitlines(keepends=True): + # add the current ANSI codes to the beginning of the line + output += "".join(ansi_codes) + line + + # split the line at each ANSI code + parts = ansi_code_regex.split(line) + # if there are any ANSI codes, save them + if len(parts) > 1: + # iterate over the ANSI codes in this line + for code in parts[1::2]: + if code == "\033[0m": + # reset the list of ANSI codes when the clear code is found + ansi_codes = [] + else: + ansi_codes.append(code) + + return output diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py deleted file mode 100644 index a145072e9..000000000 --- a/cibuildwheel/docker_container.py +++ /dev/null @@ -1,255 +0,0 @@ -import io -import json -import os -import platform -import shlex -import subprocess -import sys -import uuid -from pathlib import Path, PurePath -from types import TracebackType -from typing import IO, Dict, List, Optional, Sequence, Type, cast - -from cibuildwheel.util import CIProvider, detect_ci_provider - -from .typing import PathOrStr, PopenBytes - - -class DockerContainer: - """ - An object that represents a running Docker container. - - Intended for use as a context manager e.g. - `with DockerContainer(docker_image = 'ubuntu') as docker:` - - A bash shell is running in the remote container. When `call()` is invoked, - the command is relayed to the remote shell, and the results are streamed - back to cibuildwheel. - """ - - UTILITY_PYTHON = "/opt/python/cp38-cp38/bin/python" - - process: PopenBytes - bash_stdin: IO[bytes] - bash_stdout: IO[bytes] - - def __init__( - self, *, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[PathOrStr] = None - ): - if not docker_image: - raise ValueError("Must have a non-empty docker image to run.") - - self.docker_image = docker_image - self.simulate_32_bit = simulate_32_bit - self.cwd = cwd - self.name: Optional[str] = None - - def __enter__(self) -> "DockerContainer": - self.name = f"cibuildwheel-{uuid.uuid4()}" - cwd_args = ["-w", str(self.cwd)] if self.cwd else [] - - # work-around for Travis-CI PPC64le Docker runs since 2021: - # this avoids network splits - # https://github.com/pypa/cibuildwheel/issues/904 - # https://github.com/conda-forge/conda-smithy/pull/1520 - network_args = [] - if detect_ci_provider() == CIProvider.travis_ci and platform.machine() == "ppc64le": - network_args = ["--network=host"] - - shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"] - subprocess.run( - [ - "docker", - "create", - "--env=CIBUILDWHEEL", - f"--name={self.name}", - "--interactive", - "--volume=/:/host", # ignored on CircleCI - *network_args, - *cwd_args, - self.docker_image, - *shell_args, - ], - check=True, - ) - self.process = subprocess.Popen( - [ - "docker", - "start", - "--attach", - "--interactive", - self.name, - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - assert self.process.stdin and self.process.stdout - self.bash_stdin = self.process.stdin - self.bash_stdout = self.process.stdout - - # run a noop command to block until the container is responding - self.call(["/bin/true"]) - - return self - - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], - ) -> None: - - self.bash_stdin.close() - self.process.terminate() - self.process.wait() - - assert isinstance(self.name, str) - - subprocess.run(["docker", "rm", "--force", "-v", self.name], stdout=subprocess.DEVNULL) - self.name = None - - def copy_into(self, from_path: Path, to_path: PurePath) -> None: - # `docker cp` causes 'no space left on device' error when - # a container is running and the host filesystem is - # mounted. https://github.com/moby/moby/issues/38995 - # Use `docker exec` instead. - - if from_path.is_dir(): - self.call(["mkdir", "-p", to_path]) - subprocess.run( - f"tar cf - . | docker exec -i {self.name} tar -xC {shell_quote(to_path)} -f -", - shell=True, - check=True, - cwd=from_path, - ) - else: - subprocess.run( - f'cat {shell_quote(from_path)} | docker exec -i {self.name} sh -c "cat > {shell_quote(to_path)}"', - shell=True, - check=True, - ) - - def copy_out(self, from_path: PurePath, to_path: Path) -> None: - # note: we assume from_path is a dir - to_path.mkdir(parents=True, exist_ok=True) - - subprocess.run( - f"docker exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -", - shell=True, - check=True, - cwd=to_path, - ) - - def glob(self, path: PurePath, pattern: str) -> List[PurePath]: - glob_pattern = os.path.join(str(path), pattern) - - path_strings = json.loads( - self.call( - [ - self.UTILITY_PYTHON, - "-c", - f"import sys, json, glob; json.dump(glob.glob({glob_pattern!r}), sys.stdout)", - ], - capture_output=True, - ) - ) - - return [PurePath(p) for p in path_strings] - - def call( - self, - args: Sequence[PathOrStr], - env: Optional[Dict[str, str]] = None, - capture_output: bool = False, - cwd: Optional[PathOrStr] = None, - ) -> str: - - chdir = f"cd {cwd}" if cwd else "" - env_assignments = ( - " ".join(f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in env.items()) - if env is not None - else "" - ) - command = " ".join(shlex.quote(str(a)) for a in args) - end_of_message = str(uuid.uuid4()) - - # log the command we're executing - print(f" + {command}") - - # Write a command to the remote shell. First we change the - # cwd, if that's required. Then, we use the `env` utility to run - # `command` inside the specified environment. We use `env` because it - # can cope with spaces and strange characters in the name or value. - # Finally, the remote shell is told to write a footer - this will show - # up in the output so we know when to stop reading, and will include - # the return code of `command`. - self.bash_stdin.write( - bytes( - f"""( - {chdir} - env {env_assignments} {command} - printf "%04d%s\n" $? {end_of_message} - ) - """, - encoding="utf8", - errors="surrogateescape", - ) - ) - self.bash_stdin.flush() - - if capture_output: - output_io: IO[bytes] = io.BytesIO() - else: - output_io = sys.stdout.buffer - - while True: - line = self.bash_stdout.readline() - - if line.endswith(bytes(end_of_message, encoding="utf8") + b"\n"): - # fmt: off - footer_offset = ( - len(line) - - 1 # newline character - - len(end_of_message) # delimiter - - 4 # 4 return code decimals - ) - # fmt: on - return_code_str = line[footer_offset : footer_offset + 4] - return_code = int(return_code_str) - # add the last line to output, without the footer - output_io.write(line[0:footer_offset]) - break - else: - output_io.write(line) - - if isinstance(output_io, io.BytesIO): - output = str(output_io.getvalue(), encoding="utf8", errors="surrogateescape") - else: - output = "" - - if return_code != 0: - raise subprocess.CalledProcessError(return_code, args, output) - - return output - - def get_environment(self) -> Dict[str, str]: - env = json.loads( - self.call( - [ - self.UTILITY_PYTHON, - "-c", - "import sys, json, os; json.dump(os.environ.copy(), sys.stdout)", - ], - capture_output=True, - ) - ) - return cast(Dict[str, str], env) - - def environment_executor(self, command: List[str], environment: Dict[str, str]) -> str: - # used as an EnvironmentExecutor to evaluate commands and capture output - return self.call(command, env=environment, capture_output=True) - - -def shell_quote(path: PurePath) -> str: - return shlex.quote(str(path)) diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index ec8a8e360..a9a3f8627 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,9 +1,9 @@ import dataclasses -from typing import Any, Dict, List, Mapping, Optional, Sequence +from collections.abc import Mapping, Sequence +from typing import Any, Protocol import bashlex - -from cibuildwheel.typing import Protocol +import bashlex.errors from . import bashlex_eval @@ -12,7 +12,7 @@ class EnvironmentParseError(Exception): pass -def split_env_items(env_string: str) -> List[str]: +def split_env_items(env_string: str) -> list[str]: """Splits space-separated variable assignments into a list of individual assignments. >>> split_env_items('VAR=abc') @@ -31,7 +31,11 @@ def split_env_items(env_string: str) -> List[str]: if not env_string: return [] - command_node = bashlex.parsesingle(env_string) + try: + command_node = bashlex.parsesingle(env_string) + except bashlex.errors.ParsingError as e: + raise EnvironmentParseError(env_string) from e + result = [] for word_node in command_node.parts: @@ -47,11 +51,10 @@ class EnvironmentAssignment(Protocol): def evaluated_value( self, *, - environment: Dict[str, str], - executor: Optional[bashlex_eval.EnvironmentExecutor] = None, + environment: Mapping[str, str], + executor: bashlex_eval.EnvironmentExecutor | None = None, ) -> str: """Returns the value of this assignment, as evaluated in the environment""" - ... class EnvironmentAssignmentRaw: @@ -64,9 +67,9 @@ def __init__(self, name: str, value: str): self.value = value def __repr__(self) -> str: - return f"{self.name}: {self.value}" + return f"{self.name}={self.value}" - def evaluated_value(self, **kwargs: Any) -> str: + def evaluated_value(self, **_: Any) -> str: return self.value @@ -85,8 +88,8 @@ def __init__(self, assignment: str): def evaluated_value( self, - environment: Dict[str, str], - executor: Optional[bashlex_eval.EnvironmentExecutor] = None, + environment: Mapping[str, str], + executor: bashlex_eval.EnvironmentExecutor | None = None, ) -> str: return bashlex_eval.evaluate(self.value, environment=environment, executor=executor) @@ -99,9 +102,9 @@ def __eq__(self, other: object) -> bool: return False -@dataclasses.dataclass +@dataclasses.dataclass(kw_only=True) class ParsedEnvironment: - assignments: List[EnvironmentAssignment] + assignments: list[EnvironmentAssignment] def __init__(self, assignments: Sequence[EnvironmentAssignment]) -> None: self.assignments = list(assignments) @@ -109,9 +112,9 @@ def __init__(self, assignments: Sequence[EnvironmentAssignment]) -> None: def as_dictionary( self, prev_environment: Mapping[str, str], - executor: Optional[bashlex_eval.EnvironmentExecutor] = None, - ) -> Dict[str, str]: - environment = dict(**prev_environment) + executor: bashlex_eval.EnvironmentExecutor | None = None, + ) -> dict[str, str]: + environment = {**prev_environment} for assignment in self.assignments: value = assignment.evaluated_value(environment=environment, executor=executor) @@ -119,12 +122,19 @@ def as_dictionary( return environment - def add(self, name: str, value: str) -> None: - self.assignments.append(EnvironmentAssignmentRaw(name=name, value=value)) + def add(self, name: str, value: str, prepend: bool = False) -> None: + assignment = EnvironmentAssignmentRaw(name=name, value=value) + if prepend: + self.assignments.insert(0, assignment) + else: + self.assignments.append(assignment) def __repr__(self) -> str: return f"{self.__class__.__name__}({[repr(a) for a in self.assignments]!r})" + def options_summary(self) -> Any: + return self.assignments + def parse_environment(env_string: str) -> ParsedEnvironment: env_items = split_env_items(env_string) diff --git a/cibuildwheel/errors.py b/cibuildwheel/errors.py new file mode 100644 index 000000000..28939719f --- /dev/null +++ b/cibuildwheel/errors.py @@ -0,0 +1,87 @@ +""" +Errors that can cause the build to fail. Each subclass of FatalError has +a different return code, by defining them all here, we can ensure that they're +semantically clear and unique. +""" + +import textwrap + + +class FatalError(BaseException): + """ + Raising an error of this type will cause the message to be printed to + stderr and the process to be terminated. Within cibuildwheel, raising this + exception produces a better error message, and optional traceback. + """ + + return_code: int = 1 + + +class ConfigurationError(FatalError): + return_code = 2 + + +class NothingToDoError(FatalError): + return_code = 3 + + +class DeprecationError(FatalError): + return_code = 4 + + +class NonPlatformWheelError(FatalError): + def __init__(self) -> None: + message = textwrap.dedent( + """ + Build failed because a pure Python wheel was generated. + + If you intend to build a pure-Python wheel, you don't need + cibuildwheel - use `pip wheel .`, `pipx run build --wheel`, `uv + build --wheel`, etc. instead. You only need cibuildwheel if you + have compiled (not Python) code in your wheels making them depend + on the platform. + + If you expected a platform wheel, check your project configuration, + or run cibuildwheel with CIBW_BUILD_VERBOSITY=1 to view build logs. + """ + ) + super().__init__(message) + self.return_code = 5 + + +class AlreadyBuiltWheelError(FatalError): + def __init__(self, wheel_name: str) -> None: + message = textwrap.dedent( + f""" + Build failed because a wheel named {wheel_name} was already generated in the current run. + + If you expected another wheel to be generated, check your project configuration, or run + cibuildwheel with CIBW_BUILD_VERBOSITY=1 to view build logs. + """ + ) + super().__init__(message) + self.return_code = 6 + + +class OCIEngineTooOldError(FatalError): + def __init__(self, message: str) -> None: + super().__init__(message) + self.return_code = 7 + + +class RepairStepProducedNoWheelError(FatalError): + def __init__(self) -> None: + message = textwrap.dedent( + """ + Build failed because the repair step completed successfully but + did not produce a wheel. + + Your `repair-wheel-command` is expected to place the repaired + wheel in the {dest_dir} directory. See the documentation for + example configurations: + + https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command + """ + ) + super().__init__(message) + self.return_code = 8 diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index 60eecbad2..e13df6e0b 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -2,20 +2,27 @@ These are utilities for the `/bin` scripts, not for the `cibuildwheel` program. """ +import json +import time +import typing +import urllib.error +import urllib.request +from collections.abc import Mapping, Sequence from io import StringIO -from typing import Dict, List +from typing import Any, NotRequired, Protocol -from .typing import Protocol +from cibuildwheel import __version__ as cibw_version __all__ = ("Printable", "dump_python_configurations") class Printable(Protocol): - def __str__(self) -> str: - ... + def __str__(self) -> str: ... -def dump_python_configurations(inp: Dict[str, Dict[str, List[Dict[str, Printable]]]]) -> str: +def dump_python_configurations( + inp: Mapping[str, Mapping[str, Sequence[Mapping[str, Printable]]]], +) -> str: output = StringIO() for header, values in inp.items(): output.write(f"[{header}]\n") @@ -30,3 +37,65 @@ def dump_python_configurations(inp: Dict[str, Dict[str, List[Dict[str, Printable output.write("\n") # Strip the final newline, to avoid two blank lines at the end. return output.getvalue()[:-1] + + +def _json_request(request: urllib.request.Request, timeout: int = 30) -> dict[str, Any]: + with urllib.request.urlopen(request, timeout=timeout) as response: + return typing.cast(dict[str, Any], json.load(response)) + + +def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: + """ + Makes a GitHub API request to the given path and returns the JSON response. + """ + api_url = f"https://api.github.com/%7Bpath%7D" + headers = { + "Accept": "application/vnd.github.v3+json", + "User-Agent": f"cibuildwheel/{cibw_version}", + } + request = urllib.request.Request(api_url, headers=headers) + + for retry_count in range(max_retries): + try: + return _json_request(request) + except (urllib.error.URLError, TimeoutError) as e: + # pylint: disable=E1101 + if ( + isinstance(e, urllib.error.HTTPError) + and (e.code == 403 or e.code == 429) + and e.headers.get("x-ratelimit-remaining") == "0" + ): + reset_time = int(e.headers.get("x-ratelimit-reset", 0)) + wait_time = max(0, reset_time - int(e.headers.get("date", 0))) + print(f"Github rate limit exceeded. Waiting for {wait_time} seconds.") + time.sleep(wait_time) + else: + print(f"Retrying GitHub API request due to error: {e}") + + if retry_count == max_retries - 1: + print(f"GitHub API request failed (Network error: {e}). Check network connection.") + raise e + + # Should never be reached but to keep the type checker happy + msg = "Unexpected execution path in github_api_request" + raise RuntimeError(msg) + + +class PyodideXBuildEnvRelease(typing.TypedDict): + version: str + python_version: str + emscripten_version: str + min_pyodide_build_version: NotRequired[str] + max_pyodide_build_version: NotRequired[str] + + +class PyodideXBuildEnvInfo(typing.TypedDict): + releases: dict[str, PyodideXBuildEnvRelease] + + +def get_pyodide_xbuildenv_info() -> PyodideXBuildEnvInfo: + xbuildenv_info_url = ( + "https://pyodide.github.io/pyodide/api/pyodide-cross-build-environments.json" + ) + with urllib.request.urlopen(xbuildenv_info_url) as response: + return typing.cast(PyodideXBuildEnvInfo, json.loads(response.read().decode("utf-8"))) diff --git a/cibuildwheel/frontend.py b/cibuildwheel/frontend.py new file mode 100644 index 000000000..18157cdd0 --- /dev/null +++ b/cibuildwheel/frontend.py @@ -0,0 +1,68 @@ +import dataclasses +import shlex +import typing +from collections.abc import Sequence +from typing import Literal, Self, get_args + +from .logger import log +from .util.helpers import parse_key_value_string + +BuildFrontendName = Literal["pip", "build", "build[uv]"] + + +@dataclasses.dataclass(frozen=True) +class BuildFrontendConfig: + name: BuildFrontendName + args: Sequence[str] = () + + @classmethod + def from_config_string(cls, config_string: str) -> Self: + config_dict = parse_key_value_string(config_string, ["name"], ["args"]) + name = " ".join(config_dict["name"]) + if name not in get_args(BuildFrontendName): + names = ", ".join(repr(n) for n in get_args(BuildFrontendName)) + msg = f"Unrecognised build frontend {name!r}, must be one of {names}" + raise ValueError(msg) + + name = typing.cast(BuildFrontendName, name) + + args = config_dict.get("args") or [] + return cls(name=name, args=args) + + def options_summary(self) -> str | dict[str, str]: + if not self.args: + return self.name + else: + return {"name": self.name, "args": repr(self.args)} + + +def _get_verbosity_flags(level: int, frontend: BuildFrontendName) -> list[str]: + if level < 0: + if frontend == "pip": + return ["-" + -level * "q"] + + msg = f"build_verbosity {level} is not supported for {frontend} frontend. Ignoring." + log.warning(msg) + + if level > 0: + if frontend == "pip": + return ["-" + level * "v"] + if level > 1: + return ["-" + (level - 1) * "v"] + + return [] + + +def _split_config_settings(config_settings: str) -> list[str]: + config_settings_list = shlex.split(config_settings) + return [f"-C{setting}" for setting in config_settings_list] + + +def get_build_frontend_extra_flags( + build_frontend: BuildFrontendConfig, verbosity_level: int, config_settings: str +) -> list[str]: + return [ + *_split_config_settings(config_settings), + *build_frontend.args, + *_get_verbosity_flags(verbosity_level, build_frontend.name), + ] diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py deleted file mode 100644 index 35791059b..000000000 --- a/cibuildwheel/linux.py +++ /dev/null @@ -1,405 +0,0 @@ -import subprocess -import sys -import textwrap -from pathlib import Path, PurePath -from typing import Iterator, List, NamedTuple, Set, Tuple - -from .architecture import Architecture -from .docker_container import DockerContainer -from .logger import log -from .options import Options -from .typing import OrderedDict, PathOrStr, assert_never -from .util import ( - BuildSelector, - NonPlatformWheelError, - get_build_verbosity_extra_flags, - prepare_command, - read_python_configs, -) - - -class PythonConfiguration(NamedTuple): - version: str - identifier: str - path_str: str - - @property - def path(self) -> PurePath: - return PurePath(self.path_str) - - -class BuildStep(NamedTuple): - platform_configs: List[PythonConfiguration] - platform_tag: str - docker_image: str - - -def get_python_configurations( - build_selector: BuildSelector, - architectures: Set[Architecture], -) -> List[PythonConfiguration]: - - full_python_configs = read_python_configs("linux") - - python_configurations = [PythonConfiguration(**item) for item in full_python_configs] - - # return all configurations whose arch is in our `architectures` set, - # and match the build/skip rules - return [ - c - for c in python_configurations - if any(c.identifier.endswith(arch.value) for arch in architectures) - and build_selector(c.identifier) - ] - - -def docker_image_for_python_configuration(config: PythonConfiguration, options: Options) -> str: - build_options = options.build_options(config.identifier) - # e.g - # identifier is 'cp310-manylinux_x86_64' - # platform_tag is 'manylinux_x86_64' - # platform_arch is 'x86_64' - _, platform_tag = config.identifier.split("-", 1) - _, platform_arch = platform_tag.split("_", 1) - - assert build_options.manylinux_images is not None - assert build_options.musllinux_images is not None - - return ( - build_options.manylinux_images[platform_arch] - if platform_tag.startswith("manylinux") - else build_options.musllinux_images[platform_arch] - ) - - -def get_build_steps( - options: Options, python_configurations: List[PythonConfiguration] -) -> Iterator[BuildStep]: - """ - Groups PythonConfigurations into BuildSteps. Each BuildStep represents a - separate Docker container. - """ - steps = OrderedDict[Tuple[str, str, str], BuildStep]() - - for config in python_configurations: - _, platform_tag = config.identifier.split("-", 1) - - before_all = options.build_options(config.identifier).before_all - docker_image = docker_image_for_python_configuration(config, options) - - step_key = (platform_tag, docker_image, before_all) - - if step_key in steps: - steps[step_key].platform_configs.append(config) - else: - steps[step_key] = BuildStep( - platform_configs=[config], platform_tag=platform_tag, docker_image=docker_image - ) - - yield from steps.values() - - -def build_on_docker( - *, - options: Options, - platform_configs: List[PythonConfiguration], - docker: DockerContainer, - container_project_path: PurePath, - container_package_dir: PurePath, -) -> None: - container_output_dir = PurePath("/output") - - log.step("Copying project into Docker...") - docker.copy_into(Path.cwd(), container_project_path) - - before_all_options_identifier = platform_configs[0].identifier - before_all_options = options.build_options(before_all_options_identifier) - - if before_all_options.before_all: - log.step("Running before_all...") - - env = docker.get_environment() - env["PATH"] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}' - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - env = before_all_options.environment.as_dictionary( - env, executor=docker.environment_executor - ) - - before_all_prepared = prepare_command( - before_all_options.before_all, - project=container_project_path, - package=container_package_dir, - ) - docker.call(["sh", "-c", before_all_prepared], env=env) - - for config in platform_configs: - log.build_start(config.identifier) - build_options = options.build_options(config.identifier) - - dependency_constraint_flags: List[PathOrStr] = [] - - if build_options.dependency_constraints: - constraints_file = build_options.dependency_constraints.get_for_python_version( - config.version - ) - container_constraints_file = PurePath("/constraints.txt") - - docker.copy_into(constraints_file, container_constraints_file) - dependency_constraint_flags = ["-c", container_constraints_file] - - log.step("Setting up build environment...") - - env = docker.get_environment() - - # put this config's python top of the list - python_bin = config.path / "bin" - env["PATH"] = f'{python_bin}:{env["PATH"]}' - - env = build_options.environment.as_dictionary(env, executor=docker.environment_executor) - - # check config python is still on PATH - which_python = docker.call(["which", "python"], env=env, capture_output=True).strip() - if PurePath(which_python) != python_bin / "python": - print( - "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", - file=sys.stderr, - ) - sys.exit(1) - - which_pip = docker.call(["which", "pip"], env=env, capture_output=True).strip() - if PurePath(which_pip) != python_bin / "pip": - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) - - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, - project=container_project_path, - package=container_package_dir, - ) - docker.call(["sh", "-c", before_build_prepared], env=env) - - log.step("Building wheel...") - - temp_dir = PurePath("/tmp/cibuildwheel") - built_wheel_dir = temp_dir / "built_wheel" - docker.call(["rm", "-rf", built_wheel_dir]) - docker.call(["mkdir", "-p", built_wheel_dir]) - - verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) - - if build_options.build_frontend == "pip": - docker.call( - [ - "python", - "-m", - "pip", - "wheel", - container_package_dir, - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *verbosity_flags, - ], - env=env, - ) - elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) - docker.call( - [ - "python", - "-m", - "build", - container_package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", - ], - env=env, - ) - else: - assert_never(build_options.build_frontend) - - built_wheel = docker.glob(built_wheel_dir, "*.whl")[0] - - repaired_wheel_dir = temp_dir / "repaired_wheel" - docker.call(["rm", "-rf", repaired_wheel_dir]) - docker.call(["mkdir", "-p", repaired_wheel_dir]) - - if built_wheel.name.endswith("none-any.whl"): - raise NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - repair_command_prepared = prepare_command( - build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir - ) - docker.call(["sh", "-c", repair_command_prepared], env=env) - else: - docker.call(["mv", built_wheel, repaired_wheel_dir]) - - repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl") - - if build_options.test_command and build_options.test_selector(config.identifier): - log.step("Testing wheel...") - - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - docker.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env) - venv_dir = PurePath(docker.call(["mktemp", "-d"], capture_output=True).strip()) / "venv" - - docker.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=container_project_path, - package=container_package_dir, - ) - docker.call(["sh", "-c", before_test_prepared], env=virtualenv_env) - - # Install the wheel we just built - # Note: If auditwheel produced two wheels, it's because the earlier produced wheel - # conforms to multiple manylinux standards. These multiple versions of the wheel are - # functionally the same, differing only in name, wheel metadata, and possibly include - # different external shared libraries. so it doesn't matter which one we run the tests on. - # Let's just pick the first one. - wheel_to_test = repaired_wheels[0] - docker.call( - ["pip", "install", str(wheel_to_test) + build_options.test_extras], - env=virtualenv_env, - ) - - # Install any requirements to run the tests - if build_options.test_requires: - docker.call(["pip", "install", *build_options.test_requires], env=virtualenv_env) - - # Run the tests from a different directory - test_command_prepared = prepare_command( - build_options.test_command, - project=container_project_path, - package=container_package_dir, - ) - docker.call(["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env) - - # clean up test environment - docker.call(["rm", "-rf", venv_dir]) - - # move repaired wheels to output - docker.call(["mkdir", "-p", container_output_dir]) - docker.call(["mv", *repaired_wheels, container_output_dir]) - - log.build_end() - - log.step("Copying wheels back to host...") - # copy the output back into the host - docker.copy_out(container_output_dir, options.globals.output_dir) - log.step_end() - - -def build(options: Options, tmp_path: Path) -> None: - try: - # check docker is installed - subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL) - except Exception: - print( - "cibuildwheel: Docker not found. Docker is required to run Linux builds. " - "If you're building on Travis CI, add `services: [docker]` to your .travis.yml." - "If you're building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml", - file=sys.stderr, - ) - sys.exit(2) - - python_configurations = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) - - cwd = Path.cwd() - abs_package_dir = options.globals.package_dir.resolve() - if cwd != abs_package_dir and cwd not in abs_package_dir.parents: - raise Exception("package_dir must be inside the working directory") - - container_project_path = PurePath("/project") - container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) - - for build_step in get_build_steps(options, python_configurations): - try: - ids_to_build = [x.identifier for x in build_step.platform_configs] - log.step( - f"Starting Docker image {build_step.docker_image} for {', '.join(ids_to_build)}..." - ) - - with DockerContainer( - docker_image=build_step.docker_image, - simulate_32_bit=build_step.platform_tag.endswith("i686"), - cwd=container_project_path, - ) as docker: - - build_on_docker( - options=options, - platform_configs=build_step.platform_configs, - docker=docker, - container_project_path=container_project_path, - container_package_dir=container_package_dir, - ) - - except subprocess.CalledProcessError as error: - log.step_end_with_error( - f"Command {error.cmd} failed with code {error.returncode}. {error.stdout}" - ) - troubleshoot(options, error) - sys.exit(1) - - -def _matches_prepared_command(error_cmd: List[str], command_template: str) -> bool: - if len(error_cmd) < 3 or error_cmd[0:2] != ["sh", "-c"]: - return False - command_prefix = command_template.split("{", maxsplit=1)[0].strip() - return error_cmd[2].startswith(command_prefix) - - -def troubleshoot(options: Options, error: Exception) -> None: - - if isinstance(error, subprocess.CalledProcessError) and ( - error.cmd[0:4] == ["python", "-m", "pip", "wheel"] - or error.cmd[0:3] == ["python", "-m", "build"] - or _matches_prepared_command( - error.cmd, options.build_options(None).repair_command - ) # TODO allow matching of overrides too? - ): - # the wheel build step or the repair step failed - print("Checking for common errors...") - so_files = list(options.globals.package_dir.glob("**/*.so")) - - if so_files: - print( - textwrap.dedent( - """ - NOTE: Shared object (.so) files found in this project. - - These files might be built against the wrong OS, causing problems with - auditwheel. If possible, run cibuildwheel in a clean checkout. - - If you're using Cython and have previously done an in-place build, - remove those build files (*.so and *.c) before starting cibuildwheel. - - setuptools uses the build/ folder to store its build cache. It - may be necessary to remove those build files (*.so and *.o) before - starting cibuildwheel. - - Files that belong to a virtual environment are probably not an issue - unless you used a custom command telling cibuildwheel to activate it. - """ - ), - file=sys.stderr, - ) - - print(" Files detected:") - print("\n".join(f" {f}" for f in so_files)) - print("") diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index f79c51ada..cee94d666 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -3,45 +3,80 @@ import re import sys import time -from typing import IO, AnyStr, Optional, Union +from typing import IO, AnyStr, Final -from cibuildwheel.util import CIProvider, detect_ci_provider +from .ci import CIProvider, detect_ci_provider -DEFAULT_FOLD_PATTERN = ("{name}", "") -FOLD_PATTERNS = { +FoldPattern = tuple[str, str] +DEFAULT_FOLD_PATTERN: Final[FoldPattern] = ("{name}", "") +FOLD_PATTERNS: Final[dict[str, FoldPattern]] = { "azure": ("##[group]{name}", "##[endgroup]"), "travis": ("travis_fold:start:{identifier}\n{name}", "travis_fold:end:{identifier}"), "github": ("::group::{name}", "::endgroup::{name}"), } -PLATFORM_IDENTIFIER_DESCRIPTIONS = { +PLATFORM_IDENTIFIER_DESCRIPTIONS: Final[dict[str, str]] = { "manylinux_x86_64": "manylinux x86_64", "manylinux_i686": "manylinux i686", "manylinux_aarch64": "manylinux aarch64", "manylinux_ppc64le": "manylinux ppc64le", "manylinux_s390x": "manylinux s390x", + "manylinux_armv7l": "manylinux armv7l", + "manylinux_riscv64": "manylinux riscv64", "musllinux_x86_64": "musllinux x86_64", "musllinux_i686": "musllinux i686", "musllinux_aarch64": "musllinux aarch64", "musllinux_ppc64le": "musllinux ppc64le", - "musllinux_s390x": "manylinux s390x", + "musllinux_s390x": "musllinux s390x", + "musllinux_armv7l": "musllinux armv7l", + "musllinux_riscv64": "musllinux riscv64", "win32": "Windows 32bit", "win_amd64": "Windows 64bit", "win_arm64": "Windows on ARM 64bit", "macosx_x86_64": "macOS x86_64", "macosx_universal2": "macOS Universal 2 - x86_64 and arm64", "macosx_arm64": "macOS arm64 - Apple Silicon", + "pyodide_wasm32": "Pyodide", + "ios_arm64_iphoneos": "iOS Device (ARM64)", + "ios_arm64_iphonesimulator": "iOS Simulator (ARM64)", + "ios_x86_64_iphonesimulator": "iOS Simulator (x86_64)", } +class Colors: + def __init__(self, *, enabled: bool) -> None: + self.red = "\033[31m" if enabled else "" + self.green = "\033[32m" if enabled else "" + self.yellow = "\033[33m" if enabled else "" + self.blue = "\033[34m" if enabled else "" + self.cyan = "\033[36m" if enabled else "" + self.bright_red = "\033[91m" if enabled else "" + self.bright_green = "\033[92m" if enabled else "" + self.white = "\033[37m\033[97m" if enabled else "" + self.gray = "\033[38;5;244m" if enabled else "" + + self.bg_grey = "\033[48;5;235m" if enabled else "" + + self.bold = "\033[1m" if enabled else "" + self.faint = "\033[2m" if enabled else "" + + self.end = "\033[0m" if enabled else "" + + +class Symbols: + def __init__(self, *, unicode: bool) -> None: + self.done = "✓" if unicode else "done" + self.error = "✕" if unicode else "failed" + + class Logger: fold_mode: str colors_enabled: bool unicode_enabled: bool - active_build_identifier: Optional[str] = None - build_start_time: Optional[float] = None - step_start_time: Optional[float] = None - active_fold_group_name: Optional[str] = None + active_build_identifier: str | None = None + build_start_time: float | None = None + step_start_time: float | None = None + active_fold_group_name: str | None = None def __init__(self) -> None: if sys.platform == "win32" and hasattr(sys.stdout, "reconfigure"): @@ -119,23 +154,38 @@ def step_end(self, success: bool = True) -> None: self.step_start_time = None - def step_end_with_error(self, error: Union[BaseException, str]) -> None: + def step_end_with_error(self, error: BaseException | str) -> None: self.step_end(success=False) self.error(error) + def quiet(self, message: str) -> None: + c = self.colors + print(f"{c.gray}{message}{c.end}", file=sys.stderr) + + def notice(self, message: str) -> None: + if self.fold_mode == "github": + print(f"::notice::cibuildwheel: {message}\n", file=sys.stderr) + else: + c = self.colors + print(f"cibuildwheel: {c.bold}note{c.end}: {message}\n", file=sys.stderr) + def warning(self, message: str) -> None: if self.fold_mode == "github": - print(f"::warning::{message}\n", file=sys.stderr) + print(f"::warning::cibuildwheel: {message}\n", file=sys.stderr) else: c = self.colors - print(f"{c.yellow}Warning{c.end}: {message}\n", file=sys.stderr) + print(f"cibuildwheel: {c.yellow}warning{c.end}: {message}\n", file=sys.stderr) - def error(self, error: Union[BaseException, str]) -> None: + def error(self, error: BaseException | str) -> None: if self.fold_mode == "github": - print(f"::error::{error}\n", file=sys.stderr) + print(f"::error::cibuildwheel: {error}\n", file=sys.stderr) else: c = self.colors - print(f"{c.bright_red}Error{c.end}: {error}\n", file=sys.stderr) + print(f"cibuildwheel: {c.bright_red}error{c.end}: {error}\n", file=sys.stderr) + + @property + def step_active(self) -> bool: + return self.step_start_time is not None def _start_fold_group(self, name: str) -> None: self._end_fold_group() @@ -157,7 +207,8 @@ def _end_fold_group(self) -> None: sys.stdout.flush() self.active_fold_group_name = None - def _fold_group_identifier(self, name: str) -> str: + @staticmethod + def _fold_group_identifier(name: str) -> str: """ Travis doesn't like fold groups identifiers that have spaces in. This method converts them to ascii identifiers @@ -172,18 +223,12 @@ def _fold_group_identifier(self, name: str) -> str: return identifier.lower()[:20] @property - def colors(self) -> "Colors": - if self.colors_enabled: - return Colors(enabled=True) - else: - return Colors(enabled=False) + def colors(self) -> Colors: + return Colors(enabled=self.colors_enabled) @property - def symbols(self) -> "Symbols": - if self.unicode_enabled: - return Symbols(unicode=True) - else: - return Symbols(unicode=False) + def symbols(self) -> Symbols: + return Symbols(unicode=self.unicode_enabled) def build_description_from_identifier(identifier: str) -> str: @@ -192,50 +237,32 @@ def build_description_from_identifier(identifier: str) -> str: build_description = "" python_interpreter = python_identifier[0:2] - python_version = python_identifier[2:] + version_parts = python_identifier[2:].split("_") + python_version = version_parts[0] if python_interpreter == "cp": build_description += "CPython" elif python_interpreter == "pp": build_description += "PyPy" + elif python_interpreter == "gp": + build_description += "GraalPy" else: - raise Exception("unknown python") + msg = f"unknown python {python_interpreter!r}" + raise Exception(msg) build_description += f" {python_version[0]}.{python_version[1:]} " + if len(version_parts) > 1: + build_description += f"(ABI {version_parts[1]}) " try: build_description += PLATFORM_IDENTIFIER_DESCRIPTIONS[platform_identifier] except KeyError as e: - raise Exception("unknown platform") from e + msg = f"unknown platform {platform_identifier!r}" + raise Exception(msg) from e return build_description -class Colors: - def __init__(self, *, enabled: bool) -> None: - self.red = "\033[31m" if enabled else "" - self.green = "\033[32m" if enabled else "" - self.yellow = "\033[33m" if enabled else "" - self.blue = "\033[34m" if enabled else "" - self.cyan = "\033[36m" if enabled else "" - self.bright_red = "\033[91m" if enabled else "" - self.bright_green = "\033[92m" if enabled else "" - self.white = "\033[37m\033[97m" if enabled else "" - - self.bg_grey = "\033[48;5;235m" if enabled else "" - - self.bold = "\033[1m" if enabled else "" - self.faint = "\033[2m" if enabled else "" - - self.end = "\033[0m" if enabled else "" - - -class Symbols: - def __init__(self, *, unicode: bool) -> None: - self.done = "✓" if unicode else "done" - self.error = "✕" if unicode else "failed" - - def file_supports_color(file_obj: IO[AnyStr]) -> bool: """ Returns True if the running system's terminal supports color. @@ -262,8 +289,6 @@ def file_supports_unicode(file_obj: IO[AnyStr]) -> bool: return "utf" in codec_info.name -""" -Global instance of the Logger. -""" +# Global instance of the Logger. # (there's only one stdout per-process, so a global instance is justified) log = Logger() diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py deleted file mode 100644 index 420fa2a54..000000000 --- a/cibuildwheel/macos.py +++ /dev/null @@ -1,535 +0,0 @@ -import os -import platform -import re -import shutil -import subprocess -import sys -from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Sequence, Set, Tuple, cast - -from filelock import FileLock - -from .architecture import Architecture -from .environment import ParsedEnvironment -from .logger import log -from .options import Options -from .typing import Literal, PathOrStr, assert_never -from .util import ( - CIBW_CACHE_PATH, - BuildFrontend, - BuildSelector, - NonPlatformWheelError, - call, - detect_ci_provider, - download, - get_build_verbosity_extra_flags, - get_pip_version, - install_certifi_script, - prepare_command, - read_python_configs, - shell, - unwrap, - virtualenv, -) - - -def get_macos_version() -> Tuple[int, int]: - """ - Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) - - These tuples can be used in comparisons, e.g. - (10, 14) <= (11, 0) == True - (10, 14) <= (10, 16) == True - (11, 2) <= (11, 0) != True - """ - version_str, _, _ = platform.mac_ver() - version = tuple(map(int, version_str.split(".")[:2])) - return cast(Tuple[int, int], version) - - -def get_macos_sdks() -> List[str]: - output = call("xcodebuild", "-showsdks", capture_stdout=True) - return [m.group(1) for m in re.finditer(r"-sdk (macosx\S+)", output)] - - -class PythonConfiguration(NamedTuple): - version: str - identifier: str - url: str - - -def get_python_configurations( - build_selector: BuildSelector, architectures: Set[Architecture] -) -> List[PythonConfiguration]: - - full_python_configs = read_python_configs("macos") - - python_configurations = [PythonConfiguration(**item) for item in full_python_configs] - - # filter out configs that don't match any of the selected architectures - python_configurations = [ - c - for c in python_configurations - if any(c.identifier.endswith(a.value) for a in architectures) - ] - - # skip builds as required by BUILD/SKIP - return [c for c in python_configurations if build_selector(c.identifier)] - - -def install_cpython(tmp: Path, version: str, url: str) -> Path: - installation_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}") - with FileLock(CIBW_CACHE_PATH / f"cpython{version}.lock"): - installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines() - # if this version of python isn't installed, get it from python.org and install - python_package_identifier = f"org.python.Python.PythonFramework-{version}" - if python_package_identifier not in installed_system_packages: - if detect_ci_provider() is None: - # if running locally, we don't want to install CPython with sudo - # let the user know & provide a link to the installer - print( - f"Error: CPython {version} is not installed.\n" - "cibuildwheel will not perform system-wide installs when running outside of CI.\n" - f"To build locally, install CPython {version} on this machine, or, disable this version of Python using CIBW_SKIP=cp{version.replace('.', '')}-macosx_*\n" - f"\nDownload link: {url}", - file=sys.stderr, - ) - raise SystemExit(1) - pkg_path = tmp / "Python.pkg" - # download the pkg - download(url, pkg_path) - # install - call("sudo", "installer", "-pkg", pkg_path, "-target", "/") - pkg_path.unlink() - env = os.environ.copy() - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - call(installation_path / "bin" / "python3", install_certifi_script, env=env) - - return installation_path / "bin" / "python3" - - -def install_pypy(tmp: Path, version: str, url: str) -> Path: - pypy_tar_bz2 = url.rsplit("/", 1)[-1] - extension = ".tar.bz2" - assert pypy_tar_bz2.endswith(extension) - installation_path = CIBW_CACHE_PATH / pypy_tar_bz2[: -len(extension)] - with FileLock(str(installation_path) + ".lock"): - if not installation_path.exists(): - downloaded_tar_bz2 = tmp / pypy_tar_bz2 - download(url, downloaded_tar_bz2) - installation_path.parent.mkdir(parents=True, exist_ok=True) - call("tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2) - downloaded_tar_bz2.unlink() - return installation_path / "bin" / "pypy3" - - -def setup_python( - tmp: Path, - python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], - environment: ParsedEnvironment, - build_frontend: BuildFrontend, -) -> Dict[str, str]: - tmp.mkdir() - implementation_id = python_configuration.identifier.split("-")[0] - log.step(f"Installing Python {implementation_id}...") - if implementation_id.startswith("cp"): - base_python = install_cpython(tmp, python_configuration.version, python_configuration.url) - elif implementation_id.startswith("pp"): - base_python = install_pypy(tmp, python_configuration.version, python_configuration.url) - else: - raise ValueError("Unknown Python implementation") - assert base_python.exists() - - log.step("Setting up build environment...") - venv_path = tmp / "venv" - env = virtualenv(base_python, venv_path, dependency_constraint_flags) - venv_bin_path = venv_path / "bin" - assert venv_bin_path.exists() - # Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`, - # `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175 - # Also fix an issue with the shebang of installed scripts inside the - # testing virtualenv- see https://github.com/theacodes/nox/issues/44 and - # https://github.com/pypa/virtualenv/issues/620 - # Also see https://github.com/python/cpython/pull/9516 - env.pop("__PYVENV_LAUNCHER__", None) - - # we version pip ourselves, so we don't care about pip version checking - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - - # upgrade pip to the version matching our constraints - # if necessary, reinstall it to ensure that it's available on PATH as 'pip' - call( - "python", - "-m", - "pip", - "install", - "--upgrade", - "pip", - *dependency_constraint_flags, - env=env, - cwd=venv_path, - ) - - # Apply our environment after pip is ready - env = environment.as_dictionary(prev_environment=env) - - # check what pip version we're on - assert (venv_bin_path / "pip").exists() - call("which", "pip", env=env) - call("pip", "--version", env=env) - which_pip = call("which", "pip", env=env, capture_stdout=True).strip() - if which_pip != str(venv_bin_path / "pip"): - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) - - # check what Python version we're on - call("which", "python", env=env) - call("python", "--version", env=env) - which_python = call("which", "python", env=env, capture_stdout=True).strip() - if which_python != str(venv_bin_path / "python"): - print( - "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", - file=sys.stderr, - ) - sys.exit(1) - - # Set MACOSX_DEPLOYMENT_TARGET to 10.9, if the user didn't set it. - # PyPy defaults to 10.7, causing inconsistencies if it's left unset. - env.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.9") - - config_is_arm64 = python_configuration.identifier.endswith("arm64") - config_is_universal2 = python_configuration.identifier.endswith("universal2") - - if python_configuration.version not in {"3.6", "3.7"}: - if config_is_arm64: - # macOS 11 is the first OS with arm64 support, so the wheels - # have that as a minimum. - env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-11.0-arm64") - env.setdefault("ARCHFLAGS", "-arch arm64") - elif config_is_universal2: - env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-10.9-universal2") - env.setdefault("ARCHFLAGS", "-arch arm64 -arch x86_64") - elif python_configuration.identifier.endswith("x86_64"): - # even on the macos11.0 Python installer, on the x86_64 side it's - # compatible back to 10.9. - env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-10.9-x86_64") - env.setdefault("ARCHFLAGS", "-arch x86_64") - - building_arm64 = config_is_arm64 or config_is_universal2 - if building_arm64 and get_macos_version() < (10, 16) and "SDKROOT" not in env: - # xcode 12.2 or higher can build arm64 on macos 10.15 or below, but - # needs the correct SDK selected. - sdks = get_macos_sdks() - - # Different versions of Xcode contain different SDK versions... - # we're happy with anything newer than macOS 11.0 - arm64_compatible_sdks = [s for s in sdks if not s.startswith("macosx10.")] - - if not arm64_compatible_sdks: - log.warning( - unwrap( - """ - SDK for building arm64-compatible wheels not found. You need Xcode 12.2 or later - to build universal2 or arm64 wheels. - """ - ) - ) - else: - env.setdefault("SDKROOT", arm64_compatible_sdks[0]) - - log.step("Installing build tools...") - if build_frontend == "pip": - call( - "pip", - "install", - "--upgrade", - "setuptools", - "wheel", - "delocate", - *dependency_constraint_flags, - env=env, - ) - elif build_frontend == "build": - call( - "pip", - "install", - "--upgrade", - "delocate", - "build[virtualenv]", - *dependency_constraint_flags, - env=env, - ) - else: - assert_never(build_frontend) - - return env - - -def build(options: Options, tmp_path: Path) -> None: - python_configurations = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) - - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) - - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - env.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.9") - before_all_prepared = prepare_command( - before_all_options.before_all, project=".", package=before_all_options.package_dir - ) - shell(before_all_prepared, env=env) - - for config in python_configurations: - build_options = options.build_options(config.identifier) - log.build_start(config.identifier) - - identifier_tmp_dir = tmp_path / config.identifier - identifier_tmp_dir.mkdir() - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" - - config_is_arm64 = config.identifier.endswith("arm64") - config_is_universal2 = config.identifier.endswith("universal2") - - dependency_constraint_flags: Sequence[PathOrStr] = [] - if build_options.dependency_constraints: - dependency_constraint_flags = [ - "-c", - build_options.dependency_constraints.get_for_python_version(config.version), - ] - - env = setup_python( - identifier_tmp_dir / "build", - config, - dependency_constraint_flags, - build_options.environment, - build_options.build_frontend, - ) - - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, project=".", package=build_options.package_dir - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) - - if build_options.build_frontend == "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - "python", - "-m", - "pip", - "wheel", - build_options.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *verbosity_flags, - env=env, - ) - elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) - build_env = env.copy() - if build_options.dependency_constraints: - constraint_path = build_options.dependency_constraints.get_for_python_version( - config.version - ) - build_env["PIP_CONSTRAINT"] = constraint_path.as_uri() - build_env["VIRTUALENV_PIP"] = get_pip_version(env) - call( - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", - env=build_env, - ) - else: - assert_never(build_options.build_frontend) - - built_wheel = next(built_wheel_dir.glob("*.whl")) - - repaired_wheel_dir.mkdir() - - if built_wheel.name.endswith("none-any.whl"): - raise NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - - if config_is_universal2: - delocate_archs = "x86_64,arm64" - elif config_is_arm64: - delocate_archs = "arm64" - else: - delocate_archs = "x86_64" - - repair_command_prepared = prepare_command( - build_options.repair_command, - wheel=built_wheel, - dest_dir=repaired_wheel_dir, - delocate_archs=delocate_archs, - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - - log.step_end() - - if build_options.test_command and build_options.test_selector(config.identifier): - machine_arch = platform.machine() - testing_archs: List[Literal["x86_64", "arm64"]] - - if config_is_arm64: - testing_archs = ["arm64"] - elif config_is_universal2: - testing_archs = ["x86_64", "arm64"] - else: - testing_archs = ["x86_64"] - - for testing_arch in testing_archs: - if config_is_universal2: - arch_specific_identifier = f"{config.identifier}:{testing_arch}" - if not build_options.test_selector(arch_specific_identifier): - continue - - if machine_arch == "x86_64" and testing_arch == "arm64": - if config_is_arm64: - log.warning( - unwrap( - """ - While arm64 wheels can be built on x86_64, they cannot be - tested. The ability to test the arm64 wheels will be added in a - future release of cibuildwheel, once Apple Silicon CI runners - are widely available. To silence this warning, set - `CIBW_TEST_SKIP: *-macosx_arm64`. - """ - ) - ) - elif config_is_universal2: - log.warning( - unwrap( - """ - While universal2 wheels can be built on x86_64, the arm64 part - of them cannot currently be tested. The ability to test the - arm64 part of a universal2 wheel will be added in a future - release of cibuildwheel, once Apple Silicon CI runners are - widely available. To silence this warning, set - `CIBW_TEST_SKIP: *-macosx_universal2:arm64`. - """ - ) - ) - else: - raise RuntimeError("unreachable") - - # skip this test - continue - - log.step( - "Testing wheel..." - if testing_arch == machine_arch - else f"Testing wheel on {testing_arch}..." - ) - - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) - - venv_dir = identifier_tmp_dir / "venv-test" - - arch_prefix = [] - if testing_arch != machine_arch: - if machine_arch == "arm64" and testing_arch == "x86_64": - # rosetta2 will provide the emulation with just the arch prefix. - arch_prefix = ["arch", "-x86_64"] - else: - raise RuntimeError( - "don't know how to emulate {testing_arch} on {machine_arch}" - ) - - # define a custom 'call' function that adds the arch prefix each time - def call_with_arch(*args: PathOrStr, **kwargs: Any) -> None: - call(*arch_prefix, *args, **kwargs) - - def shell_with_arch(command: str, **kwargs: Any) -> None: - command = " ".join(arch_prefix) + " " + command - shell(command, **kwargs) - - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call_with_arch("python", "-m", "virtualenv", "--no-download", venv_dir, env=env) - - virtualenv_env = env.copy() - virtualenv_env["PATH"] = os.pathsep.join( - [ - str(venv_dir / "bin"), - virtualenv_env["PATH"], - ] - ) - - # check that we are using the Python from the virtual environment - call_with_arch("which", "python", env=virtualenv_env) - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - ) - shell_with_arch(before_test_prepared, env=virtualenv_env) - - # install the wheel - call_with_arch( - "pip", - "install", - f"{repaired_wheel}{build_options.test_extras}", - env=virtualenv_env, - ) - - # test the wheel - if build_options.test_requires: - call_with_arch( - "pip", "install", *build_options.test_requires, env=virtualenv_env - ) - - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - build_options.test_command, - project=Path(".").resolve(), - package=build_options.package_dir.resolve(), - ) - shell_with_arch( - test_command_prepared, cwd=os.environ["HOME"], env=virtualenv_env - ) - - # we're all done here; move it to output (overwrite existing) - shutil.move(str(repaired_wheel), build_options.output_dir) - - # clean up - shutil.rmtree(identifier_tmp_dir) - - log.build_end() - except subprocess.CalledProcessError as error: - log.step_end_with_error( - f"Command {error.cmd} failed with code {error.returncode}. {error.stdout}" - ) - sys.exit(1) diff --git a/cibuildwheel/oci_container.py b/cibuildwheel/oci_container.py new file mode 100644 index 000000000..59a1ca7e2 --- /dev/null +++ b/cibuildwheel/oci_container.py @@ -0,0 +1,509 @@ +import dataclasses +import io +import json +import os +import platform +import shlex +import shutil +import subprocess +import sys +import textwrap +import typing +import uuid +from collections.abc import Mapping, Sequence +from enum import Enum +from pathlib import Path, PurePath, PurePosixPath +from types import TracebackType +from typing import IO, Literal, Self, assert_never + +from .ci import CIProvider, detect_ci_provider +from .errors import OCIEngineTooOldError +from .logger import log +from .typing import PathOrStr +from .util.cmd import call +from .util.helpers import FlexibleVersion, parse_key_value_string, strtobool + +ContainerEngineName = Literal["docker", "podman"] + + +# Order of the enum matters for tests. 386 shall appear before amd64. +class OCIPlatform(Enum): + i386 = "linux/386" + AMD64 = "linux/amd64" + ARMV7 = "linux/arm/v7" + ARM64 = "linux/arm64" + PPC64LE = "linux/ppc64le" + RISCV64 = "linux/riscv64" + S390X = "linux/s390x" + + +@dataclasses.dataclass(frozen=True) +class OCIContainerEngineConfig: + name: ContainerEngineName + _: dataclasses.KW_ONLY + create_args: tuple[str, ...] = dataclasses.field(default_factory=tuple) + disable_host_mount: bool = False + + @classmethod + def from_config_string(cls, config_string: str) -> Self: + config_dict = parse_key_value_string( + config_string, + ["name"], + ["create_args", "create-args", "disable_host_mount", "disable-host-mount"], + ) + name = " ".join(config_dict["name"]) + if name not in {"docker", "podman"}: + msg = f"unknown container engine {name}" + raise ValueError(msg) + + name = typing.cast(ContainerEngineName, name) + # some flexibility in the option names to cope with TOML conventions + create_args = config_dict.get("create_args") or config_dict.get("create-args") or [] + disable_host_mount_options = ( + config_dict.get("disable_host_mount") or config_dict.get("disable-host-mount") or [] + ) + disable_host_mount = ( + strtobool(disable_host_mount_options[-1]) if disable_host_mount_options else False + ) + if "--platform" in create_args or any(arg.startswith("--platform=") for arg in create_args): + msg = "Using '--platform' in 'container-engine::create_args' is deprecated. It will be ignored." + log.warning(msg) + if "--platform" in create_args: + index = create_args.index("--platform") + create_args.pop(index) + create_args.pop(index) + else: + create_args = [arg for arg in create_args if not arg.startswith("--platform=")] + + return cls(name=name, create_args=tuple(create_args), disable_host_mount=disable_host_mount) + + def options_summary(self) -> str | dict[str, str]: + if not self.create_args: + return self.name + else: + return { + "name": self.name, + "create_args": repr(self.create_args), + "disable_host_mount": str(self.disable_host_mount), + } + + +DEFAULT_ENGINE = OCIContainerEngineConfig("docker") + + +def _check_engine_version(engine: OCIContainerEngineConfig) -> None: + try: + version_string = call(engine.name, "version", "-f", "{{json .}}", capture_stdout=True) + version_info = json.loads(version_string.strip()) + if engine.name == "docker": + client_api_version = FlexibleVersion(version_info["Client"]["ApiVersion"]) + server_api_version = FlexibleVersion(version_info["Server"]["ApiVersion"]) + # --platform support was introduced in 1.32 as experimental, 1.41 removed the experimental flag + version = min(client_api_version, server_api_version) + minimum_version = FlexibleVersion("1.41") + minimum_version_str = "20.10.0" # docker version + error_msg = textwrap.dedent( + f""" + Build failed because {engine.name} is too old. + + cibuildwheel requires {engine.name}>={minimum_version_str} running API version {minimum_version}. + The API version found by cibuildwheel is {version}. + """ + ) + elif engine.name == "podman": + # podman uses the same version string for "Version" & "ApiVersion" + client_version = FlexibleVersion(version_info["Client"]["Version"]) + if "Server" in version_info: + server_version = FlexibleVersion(version_info["Server"]["Version"]) + else: + server_version = client_version + # --platform support was introduced in v3 + version = min(client_version, server_version) + minimum_version = FlexibleVersion("3") + error_msg = textwrap.dedent( + f""" + Build failed because {engine.name} is too old. + + cibuildwheel requires {engine.name}>={minimum_version}. + The version found by cibuildwheel is {version}. + """ + ) + else: + assert_never(engine.name) + if version < minimum_version: + raise OCIEngineTooOldError(error_msg) from None + except (subprocess.CalledProcessError, KeyError, ValueError) as e: + msg = f"Build failed because {engine.name} is too old or is not working properly." + raise OCIEngineTooOldError(msg) from e + + +class OCIContainer: + """ + An object that represents a running OCI (e.g. Docker) container. + + Intended for use as a context manager e.g. + `with OCIContainer(image = 'ubuntu') as docker:` + + A bash shell is running in the remote container. When `call()` is invoked, + the command is relayed to the remote shell, and the results are streamed + back to cibuildwheel. + + Example: + >>> from cibuildwheel.oci_container import * # NOQA + >>> from cibuildwheel.options import _get_pinned_container_images + >>> image = _get_pinned_container_images()['x86_64']['manylinux2014'] + >>> # Test the default container + >>> with OCIContainer(image=image) as self: + ... self.call(["echo", "hello world"]) + ... self.call(["cat", "/proc/1/cgroup"]) + ... print(self.get_environment()) + ... print(self.debug_info()) + """ + + UTILITY_PYTHON = "/opt/python/cp39-cp39/bin/python" + + process: subprocess.Popen[bytes] + bash_stdin: IO[bytes] + bash_stdout: IO[bytes] + + def __init__( + self, + *, + image: str, + oci_platform: OCIPlatform, + cwd: PathOrStr | None = None, + engine: OCIContainerEngineConfig = DEFAULT_ENGINE, + ): + if not image: + msg = "Must have a non-empty image to run." + raise ValueError(msg) + + self.image = image + self.oci_platform = oci_platform + self.cwd = cwd + self.name: str | None = None + self.engine = engine + self.host_tar_format = "" + if sys.platform.startswith("darwin"): + self.host_tar_format = "--format gnutar" + + def _get_platform_args(self, *, oci_platform: OCIPlatform | None = None) -> tuple[str, str]: + if oci_platform is None: + oci_platform = self.oci_platform + + # we need '--pull=always' otherwise some images with the wrong platform get reused (e.g. 386 image for amd64) + # c.f. https://github.com/moby/moby/issues/48197#issuecomment-2282802313 + pull = "always" + try: + image_platform = call( + self.engine.name, + "image", + "inspect", + self.image, + "--format", + ( + "{{.Os}}/{{.Architecture}}/{{.Variant}}" + if len(oci_platform.value.split("/")) == 3 + else "{{.Os}}/{{.Architecture}}" + ), + capture_stdout=True, + ).strip() + if image_platform == oci_platform.value: + # in case the correct image is already present, don't pull + # this allows to run local only images + pull = "never" + except subprocess.CalledProcessError: + pass + return f"--platform={oci_platform.value}", f"--pull={pull}" + + def __enter__(self) -> Self: + self.name = f"cibuildwheel-{uuid.uuid4()}" + + _check_engine_version(self.engine) + + # work-around for Travis-CI PPC64le Docker runs since 2021: + # this avoids network splits + # https://github.com/pypa/cibuildwheel/issues/904 + # https://github.com/conda-forge/conda-smithy/pull/1520 + network_args = [] + if detect_ci_provider() == CIProvider.travis_ci and platform.machine() == "ppc64le": + network_args = ["--network=host"] + + platform_args = self._get_platform_args() + + simulate_32_bit = False + if self.oci_platform in {OCIPlatform.i386, OCIPlatform.ARMV7}: + # If the architecture running the image is already the right one + # or the image entrypoint takes care of enforcing this, then we don't need to + # simulate this + run_cmd = [self.engine.name, "run", "--rm"] + ctr_cmd = ["uname", "-m"] + try: + container_machine = call( + *run_cmd, *platform_args, self.image, *ctr_cmd, capture_stdout=True + ).strip() + except subprocess.CalledProcessError: + if self.oci_platform == OCIPlatform.i386: + # The image might have been built with amd64 architecture + # Let's try that + platform_args = self._get_platform_args(oci_platform=OCIPlatform.AMD64) + container_machine = call( + *run_cmd, *platform_args, self.image, *ctr_cmd, capture_stdout=True + ).strip() + else: + raise + simulate_32_bit = container_machine not in {"i686", "armv7l", "armv8l"} + + shell_args = ["linux32", "/bin/bash"] if simulate_32_bit else ["/bin/bash"] + + subprocess.run( + [ + self.engine.name, + "create", + "--env=CIBUILDWHEEL", + "--env=SOURCE_DATE_EPOCH", + f"--name={self.name}", + "--interactive", + *(["--volume=/:/host"] if not self.engine.disable_host_mount else []), + *network_args, + *platform_args, + *self.engine.create_args, + self.image, + *shell_args, + ], + check=True, + ) + + self.process = subprocess.Popen( + [ + self.engine.name, + "start", + "--attach", + "--interactive", + self.name, + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + assert self.process.stdin + assert self.process.stdout + self.bash_stdin = self.process.stdin + self.bash_stdout = self.process.stdout + + # run a noop command to block until the container is responding + self.call(["/bin/true"], cwd="/") + + if self.cwd: + # Although `docker create -w` does create the working dir if it + # does not exist, podman does not. There does not seem to be a way + # to setup a workdir for a container running in podman. + self.call(["mkdir", "-p", os.fspath(self.cwd)], cwd="/") + + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.bash_stdin.write(b"exit 0\n") + self.bash_stdin.flush() + self.process.wait(timeout=30) + self.bash_stdin.close() + self.bash_stdout.close() + + if self.engine.name == "podman": + # This works around what seems to be a race condition in the podman + # backend. The full reason is not understood. See PR #966 for a + # discussion on possible causes and attempts to remove this line. + # For now, this seems to work "well enough". + self.process.wait() + + assert isinstance(self.name, str) + + keep_container = strtobool(os.environ.get("CIBW_DEBUG_KEEP_CONTAINER", "")) + if not keep_container: + subprocess.run( + [self.engine.name, "rm", "--force", "-v", self.name], + stdout=subprocess.DEVNULL, + check=False, + ) + self.name = None + + def copy_into(self, from_path: Path, to_path: PurePath) -> None: + if from_path.is_dir(): + self.call(["mkdir", "-p", to_path]) + subprocess.run( + f"tar -c {self.host_tar_format} -f - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -", + shell=True, + check=True, + cwd=from_path, + ) + else: + exec_process: subprocess.Popen[bytes] + self.call(["mkdir", "-p", to_path.parent]) + with subprocess.Popen( + [ + self.engine.name, + "exec", + "-i", + str(self.name), + "sh", + "-c", + f"cat > {shell_quote(to_path)}", + ], + stdin=subprocess.PIPE, + ) as exec_process: + assert exec_process.stdin + with open(from_path, "rb") as from_file: + shutil.copyfileobj(from_file, exec_process.stdin) + + exec_process.stdin.close() + exec_process.wait() + + if exec_process.returncode: + raise subprocess.CalledProcessError( + exec_process.returncode, exec_process.args, None, None + ) + + def copy_out(self, from_path: PurePath, to_path: Path) -> None: + # note: we assume from_path is a dir + to_path.mkdir(parents=True, exist_ok=True) + call(self.engine.name, "cp", f"{self.name}:{from_path}/.", to_path) + + def glob(self, path: PurePosixPath, pattern: str) -> list[PurePosixPath]: + glob_pattern = path.joinpath(pattern) + + path_strings = json.loads( + self.call( + [ + self.UTILITY_PYTHON, + "-c", + f"import sys, json, glob; json.dump(glob.glob({str(glob_pattern)!r}), sys.stdout)", + ], + capture_output=True, + ) + ) + + return [PurePosixPath(p) for p in path_strings] + + def call( + self, + args: Sequence[PathOrStr], + env: Mapping[str, str] | None = None, + capture_output: bool = False, + cwd: PathOrStr | None = None, + ) -> str: + if cwd is None: + # Podman does not start the a container in a specific working dir + # so we always need to specify it when making calls. + cwd = self.cwd + + chdir = f"cd {cwd}" if cwd else "" + env_assignments = ( + " ".join(f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in env.items()) + if env is not None + else "" + ) + command = " ".join(shlex.quote(str(a)) for a in args) + end_of_message = str(uuid.uuid4()) + + # log the command we're executing + print(f" + {command}") + + # Write a command to the remote shell. First we change the + # cwd, if that's required. Then, we use the `env` utility to run + # `command` inside the specified environment. We use `env` because it + # can cope with spaces and strange characters in the name or value. + # Finally, the remote shell is told to write a footer - this will show + # up in the output so we know when to stop reading, and will include + # the return code of `command`. + self.bash_stdin.write( + bytes( + f"""( + {chdir} + env {env_assignments} {command} + printf "%04d%s\n" $? {end_of_message} + ) + """, + encoding="utf8", + errors="surrogateescape", + ) + ) + self.bash_stdin.flush() + + if capture_output: + output_io: IO[bytes] = io.BytesIO() + else: + output_io = sys.stdout.buffer + + while True: + line = self.bash_stdout.readline() + + if line.endswith(bytes(end_of_message, encoding="utf8") + b"\n"): + # fmt: off + footer_offset = ( + len(line) + - 1 # newline character + - len(end_of_message) # delimiter + - 4 # 4 return code decimals + ) + # fmt: on + return_code_str = line[footer_offset : footer_offset + 4] + return_code = int(return_code_str) + # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + output_io.flush() + break + else: + output_io.write(line) + output_io.flush() + + if isinstance(output_io, io.BytesIO): + output = str(output_io.getvalue(), encoding="utf8", errors="surrogateescape") + else: + output = "" + + if return_code != 0: + raise subprocess.CalledProcessError(return_code, args, output) + + return output + + def get_environment(self) -> dict[str, str]: + env = json.loads( + self.call( + [ + self.UTILITY_PYTHON, + "-c", + "import sys, json, os; json.dump(os.environ.copy(), sys.stdout)", + ], + capture_output=True, + ) + ) + return typing.cast(dict[str, str], env) + + def environment_executor(self, command: Sequence[str], environment: dict[str, str]) -> str: + # used as an EnvironmentExecutor to evaluate commands and capture output + return self.call(command, env=environment, capture_output=True) + + def debug_info(self) -> str: + if self.engine.name == "podman": + command = f"{self.engine.name} info --debug" + else: + command = f"{self.engine.name} info" + completed = subprocess.run( + command, + shell=True, + check=True, + cwd=self.cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + output = str(completed.stdout, encoding="utf8", errors="surrogateescape") + return output + + +def shell_quote(path: PurePath) -> str: + return shlex.quote(os.fspath(path)) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 0ef879902..7c98f1e21 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -1,77 +1,118 @@ -import os -import sys -import traceback -from configparser import ConfigParser -from contextlib import contextmanager +import collections +import configparser +import contextlib +import dataclasses +import difflib +import enum +import functools +import shlex +import textwrap +import tomllib +from collections.abc import Callable, Generator, Iterable, Mapping, Sequence, Set from pathlib import Path -from typing import ( - Any, - Dict, - Iterator, - List, - Mapping, - NamedTuple, - Optional, - Set, - Tuple, - Union, -) +from typing import Any, Final, Literal, Self, assert_never -import tomli from packaging.specifiers import SpecifierSet +from . import errors from .architecture import Architecture from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment -from .projectfiles import get_requires_python_str -from .typing import PLATFORMS, Literal, PlatformName, TypedDict -from .util import ( - MANYLINUX_ARCHS, - MUSLLINUX_ARCHS, - BuildFrontend, - BuildSelector, - DependencyConstraints, - TestSelector, - resources_dir, - selector_matches, - strtobool, - unwrap, +from .frontend import BuildFrontendConfig +from .logger import log +from .oci_container import OCIContainerEngineConfig +from .projectfiles import get_requires_python_str, resolve_dependency_groups +from .selector import BuildSelector, EnableGroup, TestSelector, selector_matches +from .typing import PLATFORMS, PlatformName +from .util import resources +from .util.helpers import format_safe, strtobool, unwrap +from .util.packaging import DependencyConstraints + +MANYLINUX_ARCHS: Final[tuple[str, ...]] = ( + "x86_64", + "i686", + "pypy_x86_64", + "aarch64", + "ppc64le", + "s390x", + "armv7l", + "riscv64", + "pypy_aarch64", + "pypy_i686", +) + +MUSLLINUX_ARCHS: Final[tuple[str, ...]] = ( + "x86_64", + "i686", + "aarch64", + "ppc64le", + "s390x", + "armv7l", + "riscv64", ) +@dataclasses.dataclass(kw_only=True) class CommandLineArguments: - platform: Literal["auto", "linux", "macos", "windows"] - archs: Optional[str] - output_dir: Optional[str] + platform: Literal["auto", "linux", "macos", "windows"] | None + archs: str | None + output_dir: Path + only: str | None config_file: str - package_dir: str + package_dir: Path print_build_identifiers: bool allow_empty: bool - prerelease_pythons: bool + debug_traceback: bool + enable: list[str] + + @classmethod + def defaults(cls) -> Self: + return cls( + platform="auto", + allow_empty=False, + archs=None, + only=None, + config_file="", + output_dir=Path("wheelhouse"), + package_dir=Path(), + print_build_identifiers=False, + debug_traceback=False, + enable=[], + ) -class GlobalOptions(NamedTuple): +@dataclasses.dataclass(frozen=True, kw_only=True) +class GlobalOptions: package_dir: Path output_dir: Path build_selector: BuildSelector test_selector: TestSelector - architectures: Set[Architecture] + architectures: set[Architecture] + allow_empty: bool -class BuildOptions(NamedTuple): +@dataclasses.dataclass(frozen=True, kw_only=True) +class BuildOptions: globals: GlobalOptions environment: ParsedEnvironment before_all: str - before_build: Optional[str] + before_build: str | None + xbuild_tools: list[str] | None repair_command: str - manylinux_images: Optional[Dict[str, str]] - musllinux_images: Optional[Dict[str, str]] - dependency_constraints: Optional[DependencyConstraints] - test_command: Optional[str] - before_test: Optional[str] - test_requires: List[str] + manylinux_images: dict[str, str] | None + musllinux_images: dict[str, str] | None + dependency_constraints: DependencyConstraints + test_command: str | None + before_test: str | None + test_sources: list[str] + test_requires: list[str] test_extras: str + test_groups: list[str] + test_environment: ParsedEnvironment build_verbosity: int - build_frontend: BuildFrontend + build_frontend: BuildFrontendConfig | None + config_settings: str + container_engine: OCIContainerEngineConfig + pyodide_version: str | None @property def package_dir(self) -> Path: @@ -90,16 +131,27 @@ def test_selector(self) -> TestSelector: return self.globals.test_selector @property - def architectures(self) -> Set[Architecture]: + def architectures(self) -> set[Architecture]: return self.globals.architectures -Setting = Union[Dict[str, str], List[str], str] +SettingLeaf = str | int | bool +SettingList = Sequence[SettingLeaf] +SettingTable = Mapping[str, SettingLeaf | SettingList] +SettingValue = SettingTable | SettingList | SettingLeaf + +class InheritRule(enum.Enum): + NONE = enum.auto() + APPEND = enum.auto() + PREPEND = enum.auto() -class Override(NamedTuple): + +@dataclasses.dataclass(frozen=True) +class Override: select_pattern: str - options: Dict[str, Setting] + options: dict[str, SettingValue] + inherit: dict[str, InheritRule] MANYLINUX_OPTIONS = {f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS} @@ -111,35 +163,220 @@ class Override(NamedTuple): } -class TableFmt(TypedDict): - item: str - sep: str +class OptionsReaderError(errors.ConfigurationError): + pass -class ConfigOptionError(KeyError): - pass +class OptionFormat: + """ + Base class for option format specifiers. These objects describe how values + can be parsed from rich TOML values and how they're merged together. + """ + + class NotSupported(Exception): + pass + + def format_list(self, value: SettingList) -> str: # noqa: ARG002 + raise OptionFormat.NotSupported + + def format_table(self, table: SettingTable) -> str: # noqa: ARG002 + raise OptionFormat.NotSupported + + def merge_values(self, before: str, after: str) -> str: # noqa: ARG002 + raise OptionFormat.NotSupported + + +class ListFormat(OptionFormat): + """ + A format that joins lists with a separator. + """ + + def __init__(self, sep: str, quote: Callable[[str], str] | None = None) -> None: + self.sep = sep + self.quote = quote or (lambda s: s) + + def format_list(self, value: SettingList) -> str: + return self.sep.join(self.quote(str(v)) for v in value) + + def merge_values(self, before: str, after: str) -> str: + return f"{before}{self.sep}{after}" + + +class ShlexTableFormat(OptionFormat): + """ + The standard table format uses shlex.quote to quote values and shlex.split + to unquote and split them. When merging values, keys in before are + replaced by keys in after. + """ + + def __init__(self, sep: str = " ", pair_sep: str = "=", allow_merge: bool = True) -> None: + self.sep = sep + self.pair_sep = pair_sep + self.allow_merge = allow_merge + + def format_table(self, table: SettingTable) -> str: + assignments: list[tuple[str, str]] = [] + + for k, v in table.items(): + if shlex.split(k) != [k]: + msg = f"Invalid table key: {k}" + raise OptionsReaderError(msg) + + if isinstance(v, str): + assignments.append((k, v)) + elif isinstance(v, Sequence): + assignments.extend((k, str(inner_v)) for inner_v in v) + else: + assignments.append((k, str(v))) + + return self.sep.join(f"{k}{self.pair_sep}{shlex.quote(v)}" for k, v in assignments) + + def merge_values(self, before: str, after: str) -> str: + if not self.allow_merge: + raise OptionFormat.NotSupported + + before_dict = self.parse_table(before) + after_dict = self.parse_table(after) + + return self.format_table({**before_dict, **after_dict}) + + def parse_table(self, table: str) -> Mapping[str, str | Sequence[str]]: + assignments: list[tuple[str, str]] = [] + for assignment_str in shlex.split(table): + key, sep, value = assignment_str.partition(self.pair_sep) -def _dig_first(*pairs: Tuple[Mapping[str, Setting], str], ignore_empty: bool = False) -> Setting: + if not sep: + msg = f"malformed option with value {assignment_str!r}" + raise OptionsReaderError(msg) + + assignments.append((key, value)) + + result: dict[str, str | list[str]] = {} + + for key, value in assignments: + if key in result: + existing_value = result[key] + if isinstance(existing_value, list): + result[key] = [*existing_value, value] + else: + result[key] = [existing_value, value] + else: + result[key] = value + + return result + + +class EnvironmentFormat(OptionFormat): + """ + The environment format accepts a table of environment variables, where the + values may contain variables or command substitutions. """ - Return the first dict item that matches from pairs of dicts and keys. - Will throw a KeyError if missing. - _dig_first((dict1, "key1"), (dict2, "key2"), ...) + def format_table(self, table: SettingTable) -> str: + return " ".join(f'{k}="{v}"' for k, v in table.items()) + + def merge_values(self, before: str, after: str) -> str: + return f"{before} {after}" + + +def _resolve_cascade( + *pairs: tuple[SettingValue | None, InheritRule], + ignore_empty: bool = False, + option_format: OptionFormat | None = None, +) -> str: + """ + Given a cascade of values with inherit rules, resolve them into a single + value. + + 'None' values mean that the option was not set at that level, and are + ignored. If `ignore_empty` is True, empty values are ignored too. + + Values start with defaults, followed by more specific rules. If rules are + NONE, the last non-null value is returned. If a rule is APPEND or PREPEND, + the value is concatenated with the previous value. + + The following idiom can be used to get the first matching value: + + _resolve_cascade(("value1", Inherit.NONE), ("value2", Inherit.NONE), ...))) """ if not pairs: - raise ValueError("pairs cannot be empty") + msg = "pairs cannot be empty" + raise ValueError(msg) - for dict_like, key in pairs: - if key in dict_like: - value = dict_like[key] + result: str | None = None - if ignore_empty and value == "": - continue + for value, rule in pairs: + if value is None: + continue - return value + if ignore_empty and not value and value is not False: + continue + + value_string = _stringify_setting(value, option_format=option_format) - raise KeyError(key) + result = _apply_inherit_rule(result, value_string, rule=rule, option_format=option_format) + + if result is None: + msg = "a setting should at least have a default value" + raise ValueError(msg) + + return result + + +def _apply_inherit_rule( + before: str | None, after: str, rule: InheritRule, option_format: OptionFormat | None +) -> str: + if rule == InheritRule.NONE: + return after + + if not before: + # if before is None, we can just return after + # if before is an empty string, we shouldn't add any separator + return after + + if not after: + # if after is an empty string, we shouldn't add any separator + return before + + if not option_format: + msg = f"Don't know how to merge {before!r} and {after!r} with {rule}" + raise OptionsReaderError(msg) + + if rule == InheritRule.APPEND: + return option_format.merge_values(before, after) + if rule == InheritRule.PREPEND: + return option_format.merge_values(after, before) + + assert_never(rule) + + +def _stringify_setting( + setting: SettingValue, + option_format: OptionFormat | None, +) -> str: + if isinstance(setting, Mapping): + try: + if option_format is None: + raise OptionFormat.NotSupported + return option_format.format_table(setting) + except OptionFormat.NotSupported: + msg = f"Error converting {setting!r} to a string: this setting doesn't accept a table" + raise OptionsReaderError(msg) from None + + if not isinstance(setting, str) and isinstance(setting, Sequence): + try: + if option_format is None: + raise OptionFormat.NotSupported + return option_format.format_list(setting) + except OptionFormat.NotSupported: + msg = f"Error converting {setting!r} to a string: this setting doesn't accept a list" + raise OptionsReaderError(msg) from None + + if isinstance(setting, bool | int): + return str(setting) + + return setting class OptionsReader: @@ -161,87 +398,106 @@ class OptionsReader: def __init__( self, - config_file_path: Optional[Path] = None, + config_file_path: Path | None = None, *, platform: PlatformName, - disallow: Optional[Dict[str, Set[str]]] = None, + env: Mapping[str, str], + disallow: Mapping[str, Set[str]] | None = None, ) -> None: self.platform = platform + self.env = env self.disallow = disallow or {} # Open defaults.toml, loading both global and platform sections - defaults_path = resources_dir / "defaults.toml" - self.default_options, self.default_platform_options = self._load_file(defaults_path) + self.default_options, self.default_platform_options = self._load_file(resources.DEFAULTS) # Load the project config file - config_options: Dict[str, Any] = {} - config_platform_options: Dict[str, Any] = {} + config_options: dict[str, Any] = {} + config_platform_options: dict[str, Any] = {} if config_file_path is not None: config_options, config_platform_options = self._load_file(config_file_path) # Validate project config for option_name in config_options: - if not self._is_valid_global_option(option_name): - raise ConfigOptionError(f'Option "{option_name}" not supported in a config file') + self._validate_global_option(option_name) for option_name in config_platform_options: - if not self._is_valid_platform_option(option_name): - raise ConfigOptionError( - f'Option "{option_name}" not supported in the "{self.platform}" section' - ) + self._validate_platform_option(option_name) self.config_options = config_options self.config_platform_options = config_platform_options - self.overrides: List[Override] = [] - self.current_identifier: Optional[str] = None + self.overrides: list[Override] = [] + self.current_identifier: str | None = None config_overrides = self.config_options.get("overrides") if config_overrides is not None: if not isinstance(config_overrides, list): - raise ConfigOptionError('"tool.cibuildwheel.overrides" must be a list') + msg = "'tool.cibuildwheel.overrides' must be a list" + raise OptionsReaderError(msg) for config_override in config_overrides: select = config_override.pop("select", None) if not select: - raise ConfigOptionError('"select" must be set in an override') + msg = "'select' must be set in an override" + raise OptionsReaderError(msg) if isinstance(select, list): select = " ".join(select) - self.overrides.append(Override(select, config_override)) + inherit = config_override.pop("inherit", {}) + if not isinstance(inherit, dict) or not all( + i in {"none", "append", "prepend"} for i in inherit.values() + ): + msg = "'inherit' must be a dict containing only {'none', 'append', 'prepend'} values" + raise OptionsReaderError(msg) + + inherit_enum = {k: InheritRule[v.upper()] for k, v in inherit.items()} - def _is_valid_global_option(self, name: str) -> bool: + self.overrides.append(Override(select, config_override, inherit_enum)) + + def _validate_global_option(self, name: str) -> None: """ - Returns True if an option with this name is allowed in the + Raises an error if an option with this name is not allowed in the [tool.cibuildwheel] section of a config file. """ allowed_option_names = self.default_options.keys() | PLATFORMS | {"overrides"} - return name in allowed_option_names + if name not in allowed_option_names: + msg = f"Option {name!r} not supported in a config file." + matches = difflib.get_close_matches(name, allowed_option_names, 1, 0.7) + if matches: + msg += f" Perhaps you meant {matches[0]!r}?" + raise OptionsReaderError(msg) - def _is_valid_platform_option(self, name: str) -> bool: + def _validate_platform_option(self, name: str) -> None: """ - Returns True if an option with this name is allowed in the + Raises an error if an option with this name is not allowed in the [tool.cibuildwheel.] section of a config file. """ disallowed_platform_options = self.disallow.get(self.platform, set()) if name in disallowed_platform_options: - return False + msg = f"{name!r} is not allowed in {disallowed_platform_options}" + raise OptionsReaderError(msg) allowed_option_names = self.default_options.keys() | self.default_platform_options.keys() - return name in allowed_option_names + if name not in allowed_option_names: + msg = f"Option {name!r} not supported in the {self.platform!r} section" + matches = difflib.get_close_matches(name, allowed_option_names, 1, 0.7) + if matches: + msg += f" Perhaps you meant {matches[0]!r}?" + raise OptionsReaderError(msg) - def _load_file(self, filename: Path) -> Tuple[Dict[str, Any], Dict[str, Any]]: + def _load_file(self, filename: Path) -> tuple[dict[str, Any], dict[str, Any]]: """ Load a toml file, returns global and platform as separate dicts. """ with filename.open("rb") as f: - config = tomli.load(f) + config = tomllib.load(f) global_options = config.get("tool", {}).get("cibuildwheel", {}) platform_options = global_options.get(self.platform, {}) @@ -249,15 +505,15 @@ def _load_file(self, filename: Path) -> Tuple[Dict[str, Any], Dict[str, Any]]: return global_options, platform_options @property - def active_config_overrides(self) -> List[Override]: + def active_config_overrides(self) -> list[Override]: if self.current_identifier is None: return [] return [ o for o in self.overrides if selector_matches(o.select_pattern, self.current_identifier) ] - @contextmanager - def identifier(self, identifier: Optional[str]) -> Iterator[None]: + @contextlib.contextmanager + def identifier(self, identifier: str | None) -> Generator[None, None, None]: self.current_identifier = identifier try: yield @@ -269,9 +525,9 @@ def get( name: str, *, env_plat: bool = True, - sep: Optional[str] = None, - table: Optional[TableFmt] = None, + option_format: OptionFormat | None = None, ignore_empty: bool = False, + env_rule: InheritRule = InheritRule.NONE, ) -> str: """ Get and return the value for the named option from environment, @@ -279,64 +535,77 @@ def get( accept platform versions of the environment variable. If this is an array it will be merged with "sep" before returning. If it is a table, it will be formatted with "table['item']" using {k} and {v} and merged - with "table['sep']". Empty variables will not override if ignore_empty - is True. + with "table['sep']". If sep is also given, it will be used for arrays + inside the table (must match table['sep']). Empty variables will not + override if ignore_empty is True. """ if name not in self.default_options and name not in self.default_platform_options: - raise ConfigOptionError(f"{name} must be in cibuildwheel/resources/defaults.toml file") + msg = f"{name!r} must be in cibuildwheel/resources/defaults.toml file to be accessed." + raise OptionsReaderError(msg) # Environment variable form envvar = f"CIBW_{name.upper().replace('-', '_')}" plat_envvar = f"{envvar}_{self.platform.upper()}" - # later overrides take precedence over earlier ones, so reverse the list - active_config_overrides = reversed(self.active_config_overrides) - - # get the option from the environment, then the config file, then finally the default. + # get the option from the default, then the config file, then finally the environment. # platform-specific options are preferred, if they're allowed. - result = _dig_first( - (os.environ if env_plat else {}, plat_envvar), # type: ignore[arg-type] - (os.environ, envvar), - *[(o.options, name) for o in active_config_overrides], - (self.config_platform_options, name), - (self.config_options, name), - (self.default_platform_options, name), - (self.default_options, name), + return _resolve_cascade( + (self.default_options.get(name), InheritRule.NONE), + (self.default_platform_options.get(name), InheritRule.NONE), + (self.config_options.get(name), InheritRule.NONE), + (self.config_platform_options.get(name), InheritRule.NONE), + *[ + (o.options.get(name), o.inherit.get(name, InheritRule.NONE)) + for o in self.active_config_overrides + ], + (self.env.get(envvar), env_rule), + (self.env.get(plat_envvar) if env_plat else None, env_rule), ignore_empty=ignore_empty, + option_format=option_format, ) - if isinstance(result, dict): - if table is None: - raise ConfigOptionError(f"{name} does not accept a table") - return table["sep"].join(table["item"].format(k=k, v=v) for k, v in result.items()) - elif isinstance(result, list): - if sep is None: - raise ConfigOptionError(f"{name} does not accept a list") - return sep.join(result) - elif isinstance(result, int): - return str(result) - else: - return result - class Options: - def __init__(self, platform: PlatformName, command_line_arguments: CommandLineArguments): + pyproject_toml: dict[str, Any] | None + + def __init__( + self, + platform: PlatformName, + command_line_arguments: CommandLineArguments, + env: Mapping[str, str], + defaults: bool = False, + ): self.platform = platform self.command_line_arguments = command_line_arguments + self.env = env + self._defaults = defaults + self._image_warnings = set[str]() self.reader = OptionsReader( - self.config_file_path, + None if defaults else self.config_file_path, platform=platform, + env=env, disallow=DISALLOWED_OPTIONS, ) - @property - def config_file_path(self) -> Optional[Path]: + self.package_dir = Path(command_line_arguments.package_dir) + try: + with self.package_dir.joinpath("pyproject.toml").open("rb") as f: + self.pyproject_toml = tomllib.load(f) + except FileNotFoundError: + self.pyproject_toml = None + + # cache the build options method so repeated calls don't need to + # resolve the options again + self.build_options = functools.cache(self._compute_build_options) + + @functools.cached_property + def config_file_path(self) -> Path | None: args = self.command_line_arguments if args.config_file: - return Path(args.config_file.format(package=args.package_dir)) + return Path(format_safe(args.config_file, package=args.package_dir)) # return pyproject.toml, if it's available pyproject_toml_path = Path(args.package_dir) / "pyproject.toml" @@ -345,117 +614,192 @@ def config_file_path(self) -> Optional[Path]: return None - @property - def package_requires_python_str(self) -> Optional[str]: - if not hasattr(self, "_package_requires_python_str"): - args = self.command_line_arguments - self._package_requires_python_str = get_requires_python_str(Path(args.package_dir)) - return self._package_requires_python_str + @functools.cached_property + def package_requires_python_str(self) -> str | None: + return get_requires_python_str(self.package_dir, self.pyproject_toml) - @property + @functools.cached_property def globals(self) -> GlobalOptions: args = self.command_line_arguments - package_dir = Path(args.package_dir) - output_dir = Path( - args.output_dir - if args.output_dir is not None - else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse") + package_dir = args.package_dir + output_dir = args.output_dir + + build_config = ( + self.reader.get("build", env_plat=False, option_format=ListFormat(sep=" ")) or "*" ) + skip_config = self.reader.get("skip", env_plat=False, option_format=ListFormat(sep=" ")) + test_skip = self.reader.get("test-skip", env_plat=False, option_format=ListFormat(sep=" ")) - build_config = self.reader.get("build", env_plat=False, sep=" ") or "*" - skip_config = self.reader.get("skip", env_plat=False, sep=" ") - test_skip = self.reader.get("test-skip", env_plat=False, sep=" ") + allow_empty = args.allow_empty or strtobool(self.env.get("CIBW_ALLOW_EMPTY", "0")) - prerelease_pythons = args.prerelease_pythons or strtobool( - os.environ.get("CIBW_PRERELEASE_PYTHONS", "0") + enable_groups = self.reader.get( + "enable", env_plat=False, option_format=ListFormat(sep=" "), env_rule=InheritRule.APPEND ) + try: + enable = { + *EnableGroup.parse_option_value(enable_groups), + *EnableGroup.parse_option_value(" ".join(args.enable)), + } + except ValueError as e: + msg = f"Failed to parse enable group. {e}. Valid group names are: {', '.join(g.value for g in EnableGroup)}" + raise errors.ConfigurationError(msg) from e # This is not supported in tool.cibuildwheel, as it comes from a standard location. # Passing this in as an environment variable will override pyproject.toml, setup.cfg, or setup.py - requires_python_str: Optional[str] = ( - os.environ.get("CIBW_PROJECT_REQUIRES_PYTHON") or self.package_requires_python_str + requires_python_str: str | None = ( + self.env.get("CIBW_PROJECT_REQUIRES_PYTHON") or self.package_requires_python_str ) requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str) + archs_config_str = args.archs or self.reader.get("archs", option_format=ListFormat(sep=" ")) + architectures = Architecture.parse_config(archs_config_str, platform=self.platform) + + # Process `--only` + if args.only: + build_config = args.only + skip_config = "" + architectures = Architecture.all_archs(self.platform) + enable = set(EnableGroup) + build_selector = BuildSelector( build_config=build_config, skip_config=skip_config, requires_python=requires_python, - prerelease_pythons=prerelease_pythons, + enable=frozenset(enable), ) test_selector = TestSelector(skip_config=test_skip) - archs_config_str = args.archs or self.reader.get("archs", sep=" ") - architectures = Architecture.parse_config(archs_config_str, platform=self.platform) - return GlobalOptions( package_dir=package_dir, output_dir=output_dir, build_selector=build_selector, test_selector=test_selector, architectures=architectures, + allow_empty=allow_empty, ) - def build_options(self, identifier: Optional[str]) -> BuildOptions: + def _check_pinned_image(self, value: str, pinned_images: Mapping[str, str]) -> None: + error_set = {"manylinux1", "manylinux2010", "manylinux_2_24", "musllinux_1_1"} + warning_set: set[str] = set() + + if value in error_set: + msg = ( + f"cibuildwheel 3.x does not support the image {value!r}. Either upgrade to a " + "supported image or continue using the image by pinning it directly with" + " its full OCI registry '{:|@}'." + ) + raise errors.DeprecationError(msg) + + if value in warning_set and value not in self._image_warnings: + self._image_warnings.add(value) + msg = ( + f"Deprecated image {value!r}. This value will not work" + " in a future version of cibuildwheel. Either upgrade to a supported" + " image or continue using the deprecated image by pinning directly" + f" to {pinned_images[value]!r}." + ) + log.warning(msg) + + def _compute_build_options(self, identifier: str | None) -> BuildOptions: """ - Compute BuildOptions for a single run configuration. + Compute BuildOptions for a single run configuration. Normally accessed + through the `build_options` method, which is the same but the result + is cached. """ with self.reader.identifier(identifier): - before_all = self.reader.get("before-all", sep=" && ") + before_all = self.reader.get("before-all", option_format=ListFormat(sep=" && ")) + + environment_config = self.reader.get("environment", option_format=EnvironmentFormat()) + environment_pass = self.reader.get( + "environment-pass", option_format=ListFormat(sep=" ") + ).split() + before_build = self.reader.get("before-build", option_format=ListFormat(sep=" && ")) + repair_command = self.reader.get( + "repair-wheel-command", option_format=ListFormat(sep=" && ") + ) + config_settings = self.reader.get( + "config-settings", option_format=ShlexTableFormat(sep=" ", pair_sep="=") + ) - build_frontend_str = self.reader.get("build-frontend", env_plat=False) - environment_config = self.reader.get( - "environment", table={"item": '{k}="{v}"', "sep": " "} + test_command = self.reader.get("test-command", option_format=ListFormat(sep=" && ")) + before_test = self.reader.get("before-test", option_format=ListFormat(sep=" && ")) + xbuild_tools: list[str] | None = shlex.split( + self.reader.get( + "xbuild-tools", option_format=ListFormat(sep=" ", quote=shlex.quote) + ) + ) + # ["\u0000"] is a sentinel value used as a default, because TOML + # doesn't have an explicit NULL value. If xbuild-tools is set to the + # sentinel, it indicates that the user hasn't defined xbuild-tools + # *at all* (not even an `xbuild-tools = []` definition). + if xbuild_tools == ["\u0000"]: + xbuild_tools = None + + test_sources = shlex.split( + self.reader.get( + "test-sources", option_format=ListFormat(sep=" ", quote=shlex.quote) + ) + ) + test_environment_config = self.reader.get( + "test-environment", option_format=EnvironmentFormat() + ) + try: + test_environment = parse_environment(test_environment_config) + except (EnvironmentParseError, ValueError) as e: + msg = f"Malformed environment option {test_environment_config!r}" + raise errors.ConfigurationError(msg) from e + + test_requires = self.reader.get( + "test-requires", option_format=ListFormat(sep=" ") + ).split() + test_extras = self.reader.get("test-extras", option_format=ListFormat(sep=",")) + test_groups_str = self.reader.get("test-groups", option_format=ListFormat(sep=" ")) + test_groups = [x for x in test_groups_str.split() if x] + test_requirements_from_groups = resolve_dependency_groups( + self.pyproject_toml, *test_groups ) - environment_pass = self.reader.get("environment-pass", sep=" ").split() - before_build = self.reader.get("before-build", sep=" && ") - repair_command = self.reader.get("repair-wheel-command", sep=" && ") - - dependency_versions = self.reader.get("dependency-versions") - test_command = self.reader.get("test-command", sep=" && ") - before_test = self.reader.get("before-test", sep=" && ") - test_requires = self.reader.get("test-requires", sep=" ").split() - test_extras = self.reader.get("test-extras", sep=",") build_verbosity_str = self.reader.get("build-verbosity") - build_frontend: BuildFrontend - if build_frontend_str == "build": - build_frontend = "build" - elif build_frontend_str == "pip": - build_frontend = "pip" + build_frontend_str = self.reader.get( + "build-frontend", + env_plat=False, + option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), + ) + build_frontend: BuildFrontendConfig | None + if not build_frontend_str or build_frontend_str == "default": + build_frontend = None else: - msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend_str}', only 'pip' and 'build' are supported" - print(msg, file=sys.stderr) - sys.exit(2) + try: + build_frontend = BuildFrontendConfig.from_config_string(build_frontend_str) + except ValueError as e: + msg = f"Failed to parse build frontend. {e}" + raise errors.ConfigurationError(msg) from e try: environment = parse_environment(environment_config) - except (EnvironmentParseError, ValueError): - print( - f'cibuildwheel: Malformed environment option "{environment_config}"', - file=sys.stderr, - ) - traceback.print_exc(None, sys.stderr) - sys.exit(2) + except (EnvironmentParseError, ValueError) as e: + msg = f"Malformed environment option {environment_config!r}" + raise errors.ConfigurationError(msg) from e # Pass through environment variables if self.platform == "linux": - for env_var_name in environment_pass: - try: - environment.add(env_var_name, os.environ[env_var_name]) - except KeyError: - pass - - if dependency_versions == "pinned": - dependency_constraints: Optional[ - DependencyConstraints - ] = DependencyConstraints.with_defaults() - elif dependency_versions == "latest": - dependency_constraints = None - else: - dependency_versions_path = Path(dependency_versions) - dependency_constraints = DependencyConstraints(dependency_versions_path) + for env_var_name in reversed(environment_pass): + with contextlib.suppress(KeyError): + environment.add(env_var_name, self.env[env_var_name], prepend=True) + + dependency_versions_str = self.reader.get( + "dependency-versions", + env_plat=True, + option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), + ) + try: + dependency_constraints = DependencyConstraints.from_config_string( + dependency_versions_str + ) + except (ValueError, OSError) as e: + msg = f"Failed to parse dependency versions. {e}" + raise errors.ConfigurationError(msg) from e if test_extras: test_extras = f"[{test_extras}]" @@ -465,62 +809,76 @@ def build_options(self, identifier: Optional[str]) -> BuildOptions: except ValueError: build_verbosity = 0 - manylinux_images: Dict[str, str] = {} - musllinux_images: Dict[str, str] = {} + manylinux_images: dict[str, str] = {} + musllinux_images: dict[str, str] = {} + container_engine: OCIContainerEngineConfig | None = None + if self.platform == "linux": - all_pinned_docker_images = _get_pinned_docker_images() + all_pinned_container_images = _get_pinned_container_images() for build_platform in MANYLINUX_ARCHS: - pinned_images = all_pinned_docker_images[build_platform] - + pinned_images = all_pinned_container_images[build_platform] config_value = self.reader.get( f"manylinux-{build_platform}-image", ignore_empty=True ) - - if not config_value: - # default to manylinux2014 - image = pinned_images.get("manylinux2014") - elif config_value in pinned_images: + self._check_pinned_image(config_value, pinned_images) + if config_value in pinned_images: image = pinned_images[config_value] else: image = config_value - - assert image is not None manylinux_images[build_platform] = image for build_platform in MUSLLINUX_ARCHS: - pinned_images = all_pinned_docker_images[build_platform] - - config_value = self.reader.get(f"musllinux-{build_platform}-image") - - if config_value is None: - image = pinned_images["musllinux_1_1"] - elif config_value in pinned_images: + pinned_images = all_pinned_container_images[build_platform] + config_value = self.reader.get( + f"musllinux-{build_platform}-image", ignore_empty=True + ) + self._check_pinned_image(config_value, pinned_images) + if config_value in pinned_images: image = pinned_images[config_value] else: image = config_value - musllinux_images[build_platform] = image + container_engine_str = self.reader.get( + "container-engine", + option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), + ) + + try: + container_engine = OCIContainerEngineConfig.from_config_string(container_engine_str) + except ValueError as e: + msg = f"Failed to parse container config. {e}" + raise errors.ConfigurationError(msg) from e + + pyodide_version = self.reader.get("pyodide-version", env_plat=False) + return BuildOptions( globals=self.globals, test_command=test_command, - test_requires=test_requires, + test_sources=test_sources, + test_environment=test_environment, + test_requires=[*test_requires, *test_requirements_from_groups], test_extras=test_extras, + test_groups=test_groups, before_test=before_test, before_build=before_build, before_all=before_all, build_verbosity=build_verbosity, + xbuild_tools=xbuild_tools, repair_command=repair_command, environment=environment, dependency_constraints=dependency_constraints, manylinux_images=manylinux_images or None, musllinux_images=musllinux_images or None, build_frontend=build_frontend, + config_settings=config_settings, + container_engine=container_engine, + pyodide_version=pyodide_version or None, ) - def check_for_invalid_configuration(self, identifiers: List[str]) -> None: - if self.platform in ["macos", "windows"]: + def check_for_invalid_configuration(self, identifiers: Iterable[str]) -> None: + if self.platform in {"macos", "windows"}: before_all_values = {self.build_options(i).before_all for i in identifiers} if len(before_all_values) > 1: @@ -534,50 +892,125 @@ def check_for_invalid_configuration(self, identifiers: List[str]) -> None: ) ) - def check_for_deprecated_options(self) -> None: - build_selector = self.globals.build_selector - test_selector = self.globals.test_selector + @functools.cached_property + def defaults(self) -> Self: + return self.__class__( + platform=self.platform, + command_line_arguments=CommandLineArguments.defaults(), + env={}, + defaults=True, + ) - deprecated_selectors("CIBW_BUILD", build_selector.build_config, error=True) - deprecated_selectors("CIBW_SKIP", build_selector.skip_config) - deprecated_selectors("CIBW_TEST_SKIP", test_selector.skip_config) + def summary(self, identifiers: Iterable[str]) -> str: + lines = [] + global_option_names = sorted(f.name for f in dataclasses.fields(self.globals)) - def summary(self, identifiers: List[str]) -> str: - lines = [ - f"{option_name}: {option_value!r}" - for option_name, option_value in sorted(self.globals._asdict().items()) - ] + for option_name in global_option_names: + option_value = getattr(self.globals, option_name) + default_value = getattr(self.defaults.globals, option_name) + lines.append(self.option_summary(option_name, option_value, default_value)) - build_option_defaults = self.build_options(identifier=None) + build_options = self.build_options(identifier=None) + build_options_defaults = self.defaults.build_options(identifier=None) + build_options_for_identifier = { + identifier: self.build_options(identifier) for identifier in identifiers + } - for option_name, default_value in sorted(build_option_defaults._asdict().items()): + build_option_names = sorted(f.name for f in dataclasses.fields(build_options)) + + for option_name in build_option_names: if option_name == "globals": continue - lines.append(f"{option_name}: {default_value!r}") + option_value = getattr(build_options, option_name) + default_value = getattr(build_options_defaults, option_name) + overrides = { + i: getattr(build_options_for_identifier[i], option_name) for i in identifiers + } - # if any identifiers have an overridden value, print that too - for identifier in identifiers: - option_value = self.build_options(identifier=identifier)._asdict()[option_name] - if option_value != default_value: - lines.append(f" {identifier}: {option_value!r}") + lines.append( + self.option_summary(option_name, option_value, default_value, overrides=overrides) + ) return "\n".join(lines) + def option_summary( + self, + option_name: str, + option_value: Any, + default_value: Any, + overrides: Mapping[str, Any] | None = None, + ) -> str: + """ + Return a summary of the option value, including any overrides, with + ANSI 'dim' color if it's the default. + """ + value_str = self.option_summary_value(option_value) + default_value_str = self.option_summary_value(default_value) + overrides_value_strs = { + k: self.option_summary_value(v) for k, v in (overrides or {}).items() + } + # if the override value is the same as the non-overridden value, don't print it + overrides_value_strs = {k: v for k, v in overrides_value_strs.items() if v != value_str} + + has_been_set = (value_str != default_value_str) or overrides_value_strs + c = log.colors + + result = c.gray if not has_been_set else "" + result += f"{option_name}: " + + if overrides_value_strs: + overrides_groups = collections.defaultdict(list) + for k, v in overrides_value_strs.items(): + overrides_groups[v].append(k) + + result += "\n *: " + result += self.indent_if_multiline(value_str, " ") + + for override_value_str, identifiers in overrides_groups.items(): + result += f"\n {', '.join(identifiers)}: " + result += self.indent_if_multiline(override_value_str, " ") + else: + result += self.indent_if_multiline(value_str, " ") + + result += c.end + + return result + + @staticmethod + def indent_if_multiline(value: str, indent: str) -> str: + if "\n" in value: + return "\n" + textwrap.indent(value.strip(), indent) + else: + return value + + @staticmethod + def option_summary_value(option_value: Any) -> str: + if hasattr(option_value, "options_summary"): + option_value = option_value.options_summary() + + if isinstance(option_value, list): + return "".join(f"{el}\n" for el in option_value) + + if isinstance(option_value, set): + return ", ".join(str(el) for el in sorted(option_value)) + + if isinstance(option_value, dict): + return "".join(f"{k}: {v}\n" for k, v in option_value.items()) + + return str(option_value) + def compute_options( platform: PlatformName, command_line_arguments: CommandLineArguments, + env: Mapping[str, str], ) -> Options: - options = Options(platform=platform, command_line_arguments=command_line_arguments) - options.check_for_deprecated_options() - return options - + return Options(platform=platform, command_line_arguments=command_line_arguments, env=env) -_all_pinned_docker_images: Optional[ConfigParser] = None - -def _get_pinned_docker_images() -> Mapping[str, Mapping[str, str]]: +@functools.cache +def _get_pinned_container_images() -> Mapping[str, Mapping[str, str]]: """ This looks like a dict of dicts, e.g. { 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'}, @@ -585,18 +1018,6 @@ def _get_pinned_docker_images() -> Mapping[str, Mapping[str, str]]: 'pypy_x86_64': {'manylinux2010': '...' } ... } """ - global _all_pinned_docker_images - - if _all_pinned_docker_images is None: - pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg" - _all_pinned_docker_images = ConfigParser() - _all_pinned_docker_images.read(pinned_docker_images_file) - return _all_pinned_docker_images - - -def deprecated_selectors(name: str, selector: str, *, error: bool = False) -> None: - if "p2" in selector or "p35" in selector: - msg = f"cibuildwheel 2.x no longer supports Python < 3.6. Please use the 1.x series or update {name}" - print(msg, file=sys.stderr) - if error: - sys.exit(4) + all_pinned_images = configparser.ConfigParser() + all_pinned_images.read(resources.PINNED_DOCKER_IMAGES) + return all_pinned_images diff --git a/cibuildwheel/platforms/__init__.py b/cibuildwheel/platforms/__init__.py new file mode 100644 index 000000000..a56fb9036 --- /dev/null +++ b/cibuildwheel/platforms/__init__.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path +from typing import Final, Protocol + +from cibuildwheel.architecture import Architecture +from cibuildwheel.options import Options +from cibuildwheel.platforms import ios, linux, macos, pyodide, windows +from cibuildwheel.selector import BuildSelector +from cibuildwheel.typing import GenericPythonConfiguration, PlatformName + + +class PlatformModule(Protocol): + # note that as per PEP544, the self argument is ignored when the protocol + # is applied to a module + def all_python_configurations(self) -> Sequence[GenericPythonConfiguration]: ... + + def get_python_configurations( + self, build_selector: BuildSelector, architectures: set[Architecture] + ) -> Sequence[GenericPythonConfiguration]: ... + + def build(self, options: Options, tmp_path: Path) -> None: ... + + +ALL_PLATFORM_MODULES: Final[dict[PlatformName, PlatformModule]] = { + "linux": linux, + "windows": windows, + "macos": macos, + "pyodide": pyodide, + "ios": ios, +} + + +def get_build_identifiers( + platform_module: PlatformModule, + build_selector: BuildSelector, + architectures: set[Architecture], +) -> list[str]: + python_configurations = platform_module.get_python_configurations(build_selector, architectures) + return [config.identifier for config in python_configurations] diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py new file mode 100644 index 000000000..1a3d2d274 --- /dev/null +++ b/cibuildwheel/platforms/ios.py @@ -0,0 +1,689 @@ +from __future__ import annotations + +import dataclasses +import os +import shlex +import shutil +import subprocess +import sys +import textwrap +from collections.abc import Sequence, Set +from pathlib import Path +from typing import assert_never + +from filelock import FileLock + +from .. import errors +from ..architecture import Architecture +from ..environment import ParsedEnvironment +from ..frontend import ( + BuildFrontendConfig, + BuildFrontendName, + get_build_frontend_extra_flags, +) +from ..logger import log +from ..options import Options +from ..selector import BuildSelector +from ..util import resources +from ..util.cmd import call, shell, split_command +from ..util.file import ( + CIBW_CACHE_PATH, + copy_test_sources, + download, + move_file, +) +from ..util.helpers import prepare_command, unwrap_preserving_paragraphs +from ..util.packaging import ( + combine_constraints, + find_compatible_wheel, + get_pip_version, +) +from ..venv import constraint_flags, virtualenv +from .macos import install_cpython as install_build_cpython + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class PythonConfiguration: + version: str + identifier: str + url: str + build_url: str + + @property + def sdk(self) -> str: + return self.multiarch.rsplit("-", 1)[1] + + @property + def arch(self) -> str: + return self.multiarch.rsplit("-", 1)[0] + + @property + def multiarch(self) -> str: + # The multiarch identifier, as reported by `sys.implementation._multiarch` + return "-".join(self.identifier.split("-ios_")[1].rsplit("_", 1)) + + @property + def is_simulator(self) -> bool: + return self.identifier.endswith("_iphonesimulator") + + @property + def xcframework_slice(self) -> str: + "XCframeworks include binaries for multiple ABIs; which ABI section should be used?" + return "ios-arm64_x86_64-simulator" if self.is_simulator else "ios-arm64" + + +def all_python_configurations() -> list[PythonConfiguration]: + # iOS builds are always cross builds; we need to install a macOS Python as + # well. Rather than duplicate the location of the URL of macOS installers, + # load the macos configurations, determine the macOS configuration that + # matches the platform we're building, and embed that URL in the parsed iOS + # configuration. + macos_python_configs = resources.read_python_configs("macos") + + def build_url(http://github.com/config_dict:%20dict[str,%20str]) -> str: + # The iOS identifier will be something like cp313-ios_arm64_iphoneos. + # Drop the iphoneos suffix, then replace ios with macosx to yield + # cp313-macosx_arm64, which will be a macOS build identifier. + modified_ios_identifier = config_dict["identifier"].rsplit("_", 1)[0] + macos_identifier = modified_ios_identifier.replace("ios", "macosx") + matching = [ + config for config in macos_python_configs if config["identifier"] == macos_identifier + ] + return matching[0]["url"] + + # Load the platform configuration + full_python_configs = resources.read_python_configs("ios") + # Build the configurations, annotating with macOS URL details. + return [ + PythonConfiguration( + **item, + build_url=build_url(http://github.com/item), + ) + for item in full_python_configs + ] + + +def get_python_configurations( + build_selector: BuildSelector, + architectures: Set[Architecture], +) -> list[PythonConfiguration]: + python_configurations = all_python_configurations() + + # Filter out configs that don't match any of the selected architectures + python_configurations = [ + c + for c in python_configurations + if any(c.identifier.endswith(f"-ios_{a.value}") for a in architectures) + ] + + # Skip builds as required by BUILD/SKIP + python_configurations = [c for c in python_configurations if build_selector(c.identifier)] + + return python_configurations + + +def install_target_cpython(tmp: Path, config: PythonConfiguration, free_threading: bool) -> Path: + if free_threading: + msg = "Free threading builds aren't available for iOS (yet)" + raise errors.FatalError(msg) + + # Install an iOS build of CPython + ios_python_tar_gz = config.url.rsplit("/", 1)[-1] + extension = ".tar.gz" + assert ios_python_tar_gz.endswith(extension) + installation_path = CIBW_CACHE_PATH / ios_python_tar_gz[: -len(extension)] + with FileLock(str(installation_path) + ".lock"): + if not installation_path.exists(): + downloaded_tar_gz = tmp / ios_python_tar_gz + download(config.url, downloaded_tar_gz) + installation_path.mkdir(parents=True, exist_ok=True) + call("tar", "-C", installation_path, "-xf", downloaded_tar_gz) + downloaded_tar_gz.unlink() + + return installation_path + + +def cross_virtualenv( + *, + py_version: str, + target_python: Path, + multiarch: str, + build_python: Path, + venv_path: Path, + dependency_constraint: Path | None, + xbuild_tools: Sequence[str] | None, +) -> dict[str, str]: + """Create a cross-compilation virtual environment. + + In a cross-compilation environment, the *target* is the platform where the + code will ultimately run, and the *build* is the platform where you're + running the compilation. When building iOS wheels, iOS is the target machine + and macOS is the build machine. The terminology around these machines varies + between build tools (configure uses "host" and "build"; cmake uses "target" and + "build host"). + + A cross-compilation virtualenv is an environment that is based on the + *build* python (so that binaries can execute); but it modifies the + environment at startup so that any request for platform details (such as + `sys.platform` or `sysconfig.get_platform()`) return details of the target + platform. It also applies a loader patch so that any virtualenv created by + the cross-compilation environment will also be a cross-compilation + environment. + + :param py_version: The Python version (major.minor) in use + :param target_python: The path to the python binary for the target platform + :param multiarch: The multiarch tag for the target platform (i.e., the value + of `sys.implementation._multiarch`) + :param build_python: The path to the python binary for the build platform + :param venv_path: The path where the cross virtual environment should be + created. + :param dependency_constraint: A path to a constraint file that should be + used when constraining dependencies in the environment. + :param xbuild_tools: A list of executable names (without paths) that are + on the path, but must be preserved in the cross environment. + """ + # Create an initial macOS virtual environment + env = virtualenv( + py_version, + build_python, + venv_path, + dependency_constraint, + use_uv=False, + ) + + # Convert the macOS virtual environment into an iOS virtual environment + # using the cross-platform conversion script in the iOS distribution. + + # target_python is the path to the Python binary; + # determine the root of the XCframework slice that is being used. + slice_path = target_python.parent.parent + call( + "python", + str(slice_path / f"platform-config/{multiarch}/make_cross_venv.py"), + str(venv_path), + env=env, + cwd=venv_path, + ) + + # When running on macOS, it's easy for the build environment to leak into + # the target environment, especially when building for ARM64 (because the + # build architecture is the same as the target architecture). The primary + # culprit for this is Homebrew libraries leaking in as dependencies for iOS + # libraries. + # + # To prevent problems, set the PATH to isolate the build environment from + # sources that could introduce incompatible binaries. + # + # However, there may be some tools on the path that are needed for the + # build. Find their location on the path, and link the underlying binaries + # (fully resolving symlinks) to a "safe" location that will *only* contain + # those tools. This avoids needing to add *all* of Homebrew to the path just + # to get access to (for example) cmake for build purposes. A value of None + # means the user hasn't provided a list of xbuild tools. + xbuild_tools_path = venv_path / "cibw_xbuild_tools" + xbuild_tools_path.mkdir() + if xbuild_tools is None: + log.warning( + textwrap.dedent( + """ + Your project configuration does not define any cross-build tools. + + iOS builds use an isolated build environment; if your build process requires any + third-party tools (such as cmake, ninja, or rustc), you must explicitly declare + that those tools are required using xbuild-tools/CIBW_XBUILD_TOOLS. This will + likely manifest as a "somebuildtool: command not found" error. + + If the build succeeds, you can silence this warning by setting adding + `xbuild-tools = []` to your pyproject.toml configuration, or exporting + CIBW_XBUILD_TOOLS as an empty string into your environment. + """ + ) + ) + else: + for tool in xbuild_tools: + tool_path = shutil.which(tool) + if tool_path is None: + msg = f"Could not find a {tool!r} executable on the path." + raise errors.FatalError(msg) + + # Link the binary into the safe tools directory + original = Path(tool_path).resolve() + print(f"{tool!r} will be included in the cross-build environment (using {original})") + (xbuild_tools_path / tool).symlink_to(original) + + env["PATH"] = os.pathsep.join( + [ + # The target python's binary directory + str(target_python.parent), + # The cross-platform environment's binary directory + str(venv_path / "bin"), + # The directory of cross-build tools + str(xbuild_tools_path), + # The bare minimum Apple system paths. + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/Library/Apple/usr/bin", + ] + ) + # Also unset DYLD_LIBRARY_PATH to ensure that no macOS libraries will be + # found and linked. + env.pop("DYLD_LIBRARY_PATH", None) + + return env + + +def setup_python( + tmp: Path, + *, + python_configuration: PythonConfiguration, + dependency_constraint: Path | None, + environment: ParsedEnvironment, + build_frontend: BuildFrontendName, + xbuild_tools: Sequence[str] | None, +) -> tuple[Path, dict[str, str]]: + if build_frontend == "build[uv]": + msg = "uv doesn't support iOS" + raise errors.FatalError(msg) + + # An iOS environment requires 2 python installs - one for the build machine + # (macOS), and one for the target (iOS). We'll only ever interact with the + # *target* python, but the build Python needs to exist to act as the base + # for a cross venv. + tmp.mkdir() + implementation_id = python_configuration.identifier.split("-")[0] + log.step(f"Installing Build Python {implementation_id}...") + if implementation_id.startswith("cp"): + free_threading = "t-ios" in python_configuration.identifier + build_python = install_build_cpython( + tmp, + python_configuration.version, + python_configuration.build_url, + free_threading, + ) + else: + msg = f"Unknown Python implementation: {implementation_id}" + raise errors.FatalError(msg) + + assert build_python.exists(), ( + f"{build_python.name} not found, has {list(build_python.parent.iterdir())}" + ) + + log.step(f"Installing Target Python {implementation_id}...") + target_install_path = install_target_cpython(tmp, python_configuration, free_threading) + target_python = ( + target_install_path + / "Python.xcframework" + / python_configuration.xcframework_slice + / "bin" + / f"python{python_configuration.version}" + ) + + assert target_python.exists(), ( + f"{target_python.name} not found, has {list(target_install_path.iterdir())}" + ) + + log.step("Creating cross build environment...") + + venv_path = tmp / "venv" + env = cross_virtualenv( + py_version=python_configuration.version, + target_python=target_python, + multiarch=python_configuration.multiarch, + build_python=build_python, + venv_path=venv_path, + dependency_constraint=dependency_constraint, + xbuild_tools=xbuild_tools, + ) + venv_bin_path = venv_path / "bin" + assert venv_bin_path.exists() + + # We version pip ourselves, so we don't care about pip version checking + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + # upgrade pip to the version matching our constraints + # if necessary, reinstall it to ensure that it's available on PATH as 'pip' + pip = ["python", "-m", "pip"] + call( + *pip, + "install", + "--upgrade", + "pip", + *constraint_flags(dependency_constraint), + env=env, + cwd=venv_path, + ) + + # Apply our environment after pip is ready + env = environment.as_dictionary(prev_environment=env) + + # Check what Python version we're on + which_python = call("which", "python", env=env, capture_stdout=True).strip() + print(which_python) + if which_python != str(venv_bin_path / "python"): + msg = ( + "cibuildwheel: python available on PATH doesn't match our installed instance. " + "If you have modified PATH, ensure that you don't overwrite cibuildwheel's " + "entry or insert python above it." + ) + raise errors.FatalError(msg) + call("python", "--version", env=env) + + # Check what pip version we're on + assert (venv_bin_path / "pip").exists() + which_pip = call("which", "pip", env=env, capture_stdout=True).strip() + print(which_pip) + if which_pip != str(venv_bin_path / "pip"): + msg = ( + "cibuildwheel: pip available on PATH doesn't match our installed instance. " + "If you have modified PATH, ensure that you don't overwrite cibuildwheel's " + "entry or insert pip above it." + ) + raise errors.FatalError(msg) + call("pip", "--version", env=env) + + # Ensure that IPHONEOS_DEPLOYMENT_TARGET is set in the environment + env.setdefault("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + + log.step("Installing build tools...") + if build_frontend == "pip": + # No additional build tools required + pass + elif build_frontend == "build": + call( + "pip", + "install", + "--upgrade", + "build[virtualenv]", + *constraint_flags(dependency_constraint), + env=env, + ) + else: + assert_never(build_frontend) + + return target_install_path, env + + +def build(options: Options, tmp_path: Path) -> None: + if sys.platform != "darwin": + msg = "iOS binaries can only be built on macOS" + raise errors.FatalError(msg) + + python_configurations = get_python_configurations( + build_selector=options.globals.build_selector, + architectures=options.globals.architectures, + ) + + if not python_configurations: + return + + try: + before_all_options_identifier = python_configurations[0].identifier + before_all_options = options.build_options(before_all_options_identifier) + + if before_all_options.before_all: + log.step("Running before_all...") + env = before_all_options.environment.as_dictionary(prev_environment=os.environ) + env.setdefault("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + before_all_prepared = prepare_command( + before_all_options.before_all, + project=".", + package=before_all_options.package_dir, + ) + shell(before_all_prepared, env=env) + + built_wheels: list[Path] = [] + + for config in python_configurations: + build_options = options.build_options(config.identifier) + build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + # uv doesn't support iOS + if build_frontend.name == "build[uv]": + msg = "uv doesn't support iOS" + raise errors.FatalError(msg) + + log.build_start(config.identifier) + + identifier_tmp_dir = tmp_path / config.identifier + identifier_tmp_dir.mkdir() + built_wheel_dir = identifier_tmp_dir / "built_wheel" + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=config.version, tmp_dir=identifier_tmp_dir + ) + + target_install_path, env = setup_python( + identifier_tmp_dir / "build", + python_configuration=config, + dependency_constraint=constraints_path, + environment=build_options.environment, + build_frontend=build_frontend.name, + xbuild_tools=build_options.xbuild_tools, + ) + pip_version = get_pip_version(env) + + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name} " + f"that is compatible with {config.identifier}. " + "Skipping build step..." + ) + test_wheel = compatible_wheel + else: + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, + project=".", + package=build_options.package_dir, + ) + shell(before_build_prepared, env=env) + + log.step("Building wheel...") + built_wheel_dir.mkdir() + + extra_flags = get_build_frontend_extra_flags( + build_frontend, build_options.build_verbosity, build_options.config_settings + ) + + build_env = env.copy() + build_env["VIRTUALENV_PIP"] = pip_version + if constraints_path: + combine_constraints(build_env, constraints_path, None) + + if build_frontend.name == "pip": + # Path.resolve() is needed. Without it pip wheel may try to + # fetch package from pypi.org. See + # https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *extra_flags, + env=build_env, + ) + elif build_frontend.name == "build": + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + *extra_flags, + env=build_env, + ) + else: + assert_never(build_frontend) + + test_wheel = built_wheel = next(built_wheel_dir.glob("*.whl")) + + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + + log.step_end() + + if build_options.test_command and build_options.test_selector(config.identifier): + if not config.is_simulator: + log.step("Skipping tests on non-simulator SDK") + elif config.arch != os.uname().machine: + log.step("Skipping tests on non-native simulator architecture") + else: + test_env = build_options.test_environment.as_dictionary( + prev_environment=build_env + ) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, + ) + shell(before_test_prepared, env=test_env) + + log.step("Setting up test harness...") + # Clone the testbed project into the build directory + testbed_path = identifier_tmp_dir / "testbed" + call( + "python", + target_install_path / "testbed", + "clone", + testbed_path, + env=test_env, + ) + + testbed_app_path = testbed_path / "iOSTestbed" / "app" + + # Copy the test sources to the testbed app + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + Path.cwd(), + testbed_app_path, + ) + else: + (testbed_app_path / "test_fail.py").write_text( + resources.TEST_FAIL_CWD_FILE.read_text() + ) + + log.step("Installing test requirements...") + # Install the compiled wheel (with any test extras), plus + # the test requirements. Use the --platform tag to force + # the installation of iOS wheels; this requires the use of + # --only-binary=:all: + ios_version = test_env["IPHONEOS_DEPLOYMENT_TARGET"] + platform_tag = f"ios_{ios_version.replace('.', '_')}_{config.arch}_{config.sdk}" + + call( + "python", + "-m", + "pip", + "install", + "--only-binary=:all:", + "--platform", + platform_tag, + "--target", + testbed_path / "iOSTestbed" / "app_packages", + f"{test_wheel}{build_options.test_extras}", + *build_options.test_requires, + env=test_env, + ) + + log.step("Running test suite...") + + # iOS doesn't support placeholders in the test command, + # because the source dir isn't visible on the simulator. + if ( + "{project}" in build_options.test_command + or "{package}" in build_options.test_command + ): + msg = unwrap_preserving_paragraphs( + f""" + iOS tests configured with a test command that uses the "{{project}}" or + "{{package}}" placeholder. iOS tests cannot use placeholders, because the + source directory is not visible on the simulator. + + In addition, iOS tests must run as a Python module, so the test command + must begin with 'python -m'. + + Test command: {build_options.test_command!r} + """ + ) + raise errors.FatalError(msg) + + test_command_list = shlex.split(build_options.test_command) + try: + for test_command_parts in split_command(test_command_list): + match test_command_parts: + case ["python", "-m", *rest]: + final_command = rest + case ["pytest", *rest]: + # pytest works exactly the same as a module, so we + # can just run it as a module. + msg = unwrap_preserving_paragraphs(f""" + iOS tests configured with a test command which doesn't start + with 'python -m'. iOS tests must execute python modules - other + entrypoints are not supported. + + cibuildwheel will try to execute it as if it started with + 'python -m'. If this works, all you need to do is add that to + your test command. + + Test command: {build_options.test_command!r} + """) + log.warning(msg) + final_command = ["pytest", *rest] + case _: + msg = unwrap_preserving_paragraphs( + f""" + iOS tests configured with a test command which doesn't start + with 'python -m'. iOS tests must execute python modules - other + entrypoints are not supported. + + Test command: {build_options.test_command!r} + """ + ) + raise errors.FatalError(msg) + + call( + "python", + testbed_path, + "run", + *(["--verbose"] if build_options.build_verbosity > 0 else []), + "--", + *final_command, + env=test_env, + ) + except subprocess.CalledProcessError: + # catches the first test command failure in the loop, + # implementing short-circuiting + log.step_end(success=False) + log.error(f"Test suite failed on {config.identifier}") + sys.exit(1) + + log.step_end() + + # We're all done here; move it to output (overwrite existing) + if compatible_wheel is None: + output_wheel = build_options.output_dir.joinpath(built_wheel.name) + moved_wheel = move_file(built_wheel, output_wheel) + if moved_wheel != output_wheel.resolve(): + log.warning( + f"{built_wheel} was moved to {moved_wheel} instead of {output_wheel}" + ) + built_wheels.append(output_wheel) + + # Clean up + shutil.rmtree(identifier_tmp_dir) + + log.build_end() + except subprocess.CalledProcessError as error: + msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" + raise errors.FatalError(msg) from error diff --git a/cibuildwheel/platforms/linux.py b/cibuildwheel/platforms/linux.py new file mode 100644 index 000000000..82a695e69 --- /dev/null +++ b/cibuildwheel/platforms/linux.py @@ -0,0 +1,542 @@ +import contextlib +import dataclasses +import subprocess +import sys +import textwrap +from collections import OrderedDict +from collections.abc import Iterable, Iterator, Sequence, Set +from pathlib import Path, PurePath, PurePosixPath +from typing import assert_never + +from .. import errors +from ..architecture import Architecture +from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags +from ..logger import log +from ..oci_container import OCIContainer, OCIContainerEngineConfig, OCIPlatform +from ..options import BuildOptions, Options +from ..selector import BuildSelector +from ..typing import PathOrStr +from ..util import resources +from ..util.file import copy_test_sources +from ..util.helpers import prepare_command, unwrap +from ..util.packaging import find_compatible_wheel + +ARCHITECTURE_OCI_PLATFORM_MAP = { + Architecture.x86_64: OCIPlatform.AMD64, + Architecture.i686: OCIPlatform.i386, + Architecture.aarch64: OCIPlatform.ARM64, + Architecture.ppc64le: OCIPlatform.PPC64LE, + Architecture.s390x: OCIPlatform.S390X, + Architecture.armv7l: OCIPlatform.ARMV7, + Architecture.riscv64: OCIPlatform.RISCV64, +} + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class PythonConfiguration: + version: str + identifier: str + path_str: str + + @property + def path(self) -> PurePosixPath: + return PurePosixPath(self.path_str) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class BuildStep: + platform_configs: list[PythonConfiguration] + platform_tag: str + container_engine: OCIContainerEngineConfig + container_image: str + + +def all_python_configurations() -> list[PythonConfiguration]: + config_dicts = resources.read_python_configs("linux") + return [PythonConfiguration(**item) for item in config_dicts] + + +def get_python_configurations( + build_selector: BuildSelector, + architectures: Set[Architecture], +) -> list[PythonConfiguration]: + python_configurations = all_python_configurations() + + # return all configurations whose arch is in our `architectures` set, + # and match the build/skip rules + return [ + c + for c in python_configurations + if any(c.identifier.endswith(arch.value) for arch in architectures) + and build_selector(c.identifier) + ] + + +def container_image_for_python_configuration( + config: PythonConfiguration, build_options: BuildOptions +) -> str: + # e.g + # identifier is 'cp310-manylinux_x86_64' + # platform_tag is 'manylinux_x86_64' + # platform_arch is 'x86_64' + _, platform_tag = config.identifier.split("-", 1) + _, platform_arch = platform_tag.split("_", 1) + + assert build_options.manylinux_images is not None + assert build_options.musllinux_images is not None + + return ( + build_options.manylinux_images[platform_arch] + if platform_tag.startswith("manylinux") + else build_options.musllinux_images[platform_arch] + ) + + +def get_build_steps( + options: Options, python_configurations: list[PythonConfiguration] +) -> Iterator[BuildStep]: + """ + Groups PythonConfigurations into BuildSteps. Each BuildStep represents a + separate container instance. + """ + steps = OrderedDict[tuple[str, str, str, OCIContainerEngineConfig], BuildStep]() + + for config in python_configurations: + _, platform_tag = config.identifier.split("-", 1) + + build_options = options.build_options(config.identifier) + + before_all = build_options.before_all + container_image = container_image_for_python_configuration(config, build_options) + container_engine = build_options.container_engine + + step_key = (platform_tag, container_image, before_all, container_engine) + + if step_key in steps: + steps[step_key].platform_configs.append(config) + else: + steps[step_key] = BuildStep( + platform_configs=[config], + platform_tag=platform_tag, + container_engine=container_engine, + container_image=container_image, + ) + + yield from steps.values() + + +def check_all_python_exist( + *, platform_configs: Iterable[PythonConfiguration], container: OCIContainer +) -> None: + exist = True + has_manylinux_interpreters = False + messages = [] + + with contextlib.suppress(subprocess.CalledProcessError): + # use capture_output to keep quiet + container.call(["manylinux-interpreters", "--help"], capture_output=True) + has_manylinux_interpreters = True + + for config in platform_configs: + python_path = config.path / "bin" / "python" + if has_manylinux_interpreters: + try: + container.call(["manylinux-interpreters", "ensure", config.path.name]) + except subprocess.CalledProcessError: + messages.append( + f" 'manylinux-interpreters ensure {config.path.name}' needed to build '{config.identifier}' failed in container running image '{container.image}'." + " Either the installation failed or this interpreter is not available in that image. Please check the logs." + ) + exist = False + else: + try: + container.call(["test", "-x", python_path]) + except subprocess.CalledProcessError: + messages.append( + f" '{python_path}' executable doesn't exist in image '{container.image}' to build '{config.identifier}'." + ) + exist = False + if not exist: + message = "\n".join(messages) + raise errors.FatalError(message) + + +def build_in_container( + *, + options: Options, + platform_configs: Sequence[PythonConfiguration], + container: OCIContainer, + container_project_path: PurePath, + container_package_dir: PurePath, + local_tmp_dir: Path, +) -> None: + container_output_dir = PurePosixPath("/output") + + check_all_python_exist(platform_configs=platform_configs, container=container) + + log.step("Copying project into container...") + container.copy_into(Path.cwd(), container_project_path) + + before_all_options_identifier = platform_configs[0].identifier + before_all_options = options.build_options(before_all_options_identifier) + + if before_all_options.before_all: + log.step("Running before_all...") + + env = container.get_environment() + env["PATH"] = f"/opt/python/cp39-cp39/bin:{env['PATH']}" + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + env["PIP_ROOT_USER_ACTION"] = "ignore" + env = before_all_options.environment.as_dictionary( + env, executor=container.environment_executor + ) + + before_all_prepared = prepare_command( + before_all_options.before_all, + project=container_project_path, + package=container_package_dir, + ) + container.call(["sh", "-c", before_all_prepared], env=env) + + built_wheels: list[PurePosixPath] = [] + + for config in platform_configs: + log.build_start(config.identifier) + local_identifier_tmp_dir = local_tmp_dir / config.identifier + build_options = options.build_options(config.identifier) + build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + use_uv = build_frontend.name == "build[uv]" + pip = ["uv", "pip"] if use_uv else ["pip"] + + log.step("Setting up build environment...") + + dependency_constraint_flags: list[PathOrStr] = [] + local_constraints_file = build_options.dependency_constraints.get_for_python_version( + version=config.version, + tmp_dir=local_identifier_tmp_dir, + ) + if local_constraints_file: + container_constraints_file = PurePosixPath("/constraints.txt") + container.copy_into(local_constraints_file, container_constraints_file) + dependency_constraint_flags = ["-c", container_constraints_file] + + env = container.get_environment() + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + env["PIP_ROOT_USER_ACTION"] = "ignore" + + # put this config's python top of the list + python_bin = config.path / "bin" + env["PATH"] = f"{python_bin}:{env['PATH']}" + + env = build_options.environment.as_dictionary(env, executor=container.environment_executor) + + # check config python is still on PATH + which_python = container.call(["which", "python"], env=env, capture_output=True).strip() + if PurePosixPath(which_python) != python_bin / "python": + msg = "python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." + raise errors.FatalError(msg) + + if use_uv: + which_uv = container.call(["which", "uv"], env=env, capture_output=True).strip() + if not which_uv: + msg = "uv not found on PATH. You must use a supported manylinux or musllinux environment with uv." + raise errors.FatalError(msg) + else: + which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() + if PurePosixPath(which_pip) != python_bin / "pip": + msg = "pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." + raise errors.FatalError(msg) + + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + ) + repaired_wheels = [compatible_wheel] + else: + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, + project=container_project_path, + package=container_package_dir, + ) + container.call(["sh", "-c", before_build_prepared], env=env) + + log.step("Building wheel...") + + temp_dir = PurePosixPath("/tmp/cibuildwheel") + built_wheel_dir = temp_dir / "built_wheel" + container.call(["rm", "-rf", built_wheel_dir]) + container.call(["mkdir", "-p", built_wheel_dir]) + + extra_flags = get_build_frontend_extra_flags( + build_frontend, build_options.build_verbosity, build_options.config_settings + ) + + if build_frontend.name == "pip": + container.call( + [ + "python", + "-m", + "pip", + "wheel", + container_package_dir, + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *extra_flags, + ], + env=env, + ) + elif build_frontend.name == "build" or build_frontend.name == "build[uv]": + if use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: + extra_flags += ["--installer=uv"] + container.call( + [ + "python", + "-m", + "build", + container_package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + *extra_flags, + ], + env=env, + ) + else: + assert_never(build_frontend) + + built_wheel = container.glob(built_wheel_dir, "*.whl")[0] + + repaired_wheel_dir = temp_dir / "repaired_wheel" + container.call(["rm", "-rf", repaired_wheel_dir]) + container.call(["mkdir", "-p", repaired_wheel_dir]) + + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + + if build_options.repair_command: + log.step("Repairing wheel...") + repair_command_prepared = prepare_command( + build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir + ) + container.call(["sh", "-c", repair_command_prepared], env=env) + else: + container.call(["mv", built_wheel, repaired_wheel_dir]) + + repaired_wheels = container.glob(repaired_wheel_dir, "*.whl") + + if not repaired_wheels: + raise errors.RepairStepProducedNoWheelError() + + for repaired_wheel in repaired_wheels: + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise errors.AlreadyBuiltWheelError(repaired_wheel.name) + + if build_options.test_command and build_options.test_selector(config.identifier): + log.step("Testing wheel...") + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + if not use_uv: + container.call( + ["pip", "install", "virtualenv", *dependency_constraint_flags], env=env + ) + + testing_temp_dir = PurePosixPath( + container.call(["mktemp", "-d"], capture_output=True).strip() + ) + venv_dir = testing_temp_dir / "venv" + + if use_uv: + container.call(["uv", "venv", venv_dir, "--python", python_bin / "python"], env=env) + else: + # Use embedded dependencies from virtualenv to ensure determinism + venv_args = ["--no-periodic-update", "--pip=embed", "--no-setuptools"] + if "38" in config.identifier: + venv_args.append("--no-wheel") + container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" + virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=container_project_path, + package=container_package_dir, + ) + container.call(["sh", "-c", before_test_prepared], env=virtualenv_env) + + # Install the wheel we just built + # Note: If auditwheel produced two wheels, it's because the earlier produced wheel + # conforms to multiple manylinux standards. These multiple versions of the wheel are + # functionally the same, differing only in name, wheel metadata, and possibly include + # different external shared libraries. so it doesn't matter which one we run the tests on. + # Let's just pick the first one. + wheel_to_test = repaired_wheels[0] + container.call( + [*pip, "install", str(wheel_to_test) + build_options.test_extras], + env=virtualenv_env, + ) + + # Install any requirements to run the tests + if build_options.test_requires: + container.call([*pip, "install", *build_options.test_requires], env=virtualenv_env) + + # Run the tests from a different directory + test_command_prepared = prepare_command( + build_options.test_command, + project=container_project_path, + package=container_package_dir, + wheel=wheel_to_test, + ) + + test_cwd = testing_temp_dir / "test_cwd" + container.call(["mkdir", "-p", test_cwd]) + + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + Path.cwd(), + test_cwd, + copy_into=container.copy_into, + ) + else: + # Use the test_fail.py file to raise a nice error if the user + # tries to run tests in the cwd + container.copy_into(resources.TEST_FAIL_CWD_FILE, test_cwd / "test_fail.py") + + container.call(["sh", "-c", test_command_prepared], cwd=test_cwd, env=virtualenv_env) + + # clean up test environment + container.call(["rm", "-rf", testing_temp_dir]) + + # move repaired wheels to output + if compatible_wheel is None: + container.call(["mkdir", "-p", container_output_dir]) + container.call(["mv", *repaired_wheels, container_output_dir]) + built_wheels.extend( + container_output_dir / repaired_wheel.name for repaired_wheel in repaired_wheels + ) + + log.build_end() + + log.step("Copying wheels back to host...") + # copy the output back into the host + container.copy_out(container_output_dir, options.globals.output_dir) + log.step_end() + + +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) + + cwd = Path.cwd() + abs_package_dir = options.globals.package_dir.resolve() + if cwd != abs_package_dir and cwd not in abs_package_dir.parents: + msg = "package_dir must be inside the working directory" + raise Exception(msg) + + container_project_path = PurePosixPath("/project") + container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) + + for build_step in get_build_steps(options, python_configurations): + try: + # check the container engine is installed + subprocess.run( + [build_step.container_engine.name, "--version"], + check=True, + stdout=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError as error: + msg = unwrap( + f""" + {build_step.container_engine.name} not found. An OCI exe like + Docker or Podman is required to run Linux builds. If you're + building on Travis CI, add `services: [docker]` to your + .travis.yml. If you're building on Circle CI in Linux, add a + `setup_remote_docker` step to your .circleci/config.yml. If + you're building on Cirrus CI, use `docker_builder` task. + """ + ) + raise errors.ConfigurationError(msg) from error + + try: + ids_to_build = [x.identifier for x in build_step.platform_configs] + log.step(f"Starting container image {build_step.container_image}...") + + print(f"info: This container will host the build for {', '.join(ids_to_build)}...") + architecture = Architecture(build_step.platform_tag.split("_", 1)[1]) + + with OCIContainer( + image=build_step.container_image, + oci_platform=ARCHITECTURE_OCI_PLATFORM_MAP[architecture], + cwd=container_project_path, + engine=build_step.container_engine, + ) as container: + build_in_container( + options=options, + platform_configs=build_step.platform_configs, + container=container, + container_project_path=container_project_path, + container_package_dir=container_package_dir, + local_tmp_dir=tmp_path, + ) + + except subprocess.CalledProcessError as error: + troubleshoot(options, error) + msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" + raise errors.FatalError(msg) from error + + +def _matches_prepared_command(error_cmd: Sequence[str], command_template: str) -> bool: + if len(error_cmd) < 3 or error_cmd[0:2] != ["sh", "-c"]: + return False + command_prefix = command_template.split("{", maxsplit=1)[0].strip() + return error_cmd[2].startswith(command_prefix) + + +def troubleshoot(options: Options, error: Exception) -> None: + if isinstance(error, subprocess.CalledProcessError) and ( + error.cmd[0:4] == ["python", "-m", "pip", "wheel"] + or error.cmd[0:3] == ["python", "-m", "build"] + or _matches_prepared_command( + error.cmd, options.build_options(None).repair_command + ) # TODO allow matching of overrides too? + ): + # the wheel build step or the repair step failed + so_files = list(options.globals.package_dir.glob("**/*.so")) + + if so_files: + print( + textwrap.dedent( + """ + NOTE: Shared object (.so) files found in this project. + + These files might be built against the wrong OS, causing problems with + auditwheel. If possible, run cibuildwheel in a clean checkout. + + If you're using Cython and have previously done an in-place build, + remove those build files (*.so and *.c) before starting cibuildwheel. + + setuptools uses the build/ folder to store its build cache. It + may be necessary to remove those build files (*.so and *.o) before + starting cibuildwheel. + + Files that belong to a virtual environment are probably not an issue + unless you used a custom command telling cibuildwheel to activate it. + """ + ), + file=sys.stderr, + ) + + print(" Files detected:") + print("\n".join(f" {f}" for f in so_files)) + print() diff --git a/cibuildwheel/platforms/macos.py b/cibuildwheel/platforms/macos.py new file mode 100644 index 000000000..99ca915b8 --- /dev/null +++ b/cibuildwheel/platforms/macos.py @@ -0,0 +1,744 @@ +import dataclasses +import functools +import inspect +import os +import platform +import re +import shutil +import subprocess +import sys +import typing +from collections.abc import Set +from pathlib import Path +from typing import Literal, assert_never + +from filelock import FileLock +from packaging.version import Version + +from .. import errors +from ..architecture import Architecture +from ..ci import detect_ci_provider +from ..environment import ParsedEnvironment +from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_frontend_extra_flags +from ..logger import log +from ..options import Options +from ..selector import BuildSelector +from ..util import resources +from ..util.cmd import call, shell +from ..util.file import ( + CIBW_CACHE_PATH, + copy_test_sources, + download, + move_file, +) +from ..util.helpers import prepare_command, unwrap +from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version +from ..venv import constraint_flags, find_uv, virtualenv + + +@functools.cache +def get_macos_version() -> tuple[int, int]: + """ + Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) + + These tuples can be used in comparisons, e.g. + (10, 14) <= (11, 0) == True + (10, 14) <= (10, 16) == True + (11, 2) <= (11, 0) != True + """ + version_str, _, _ = platform.mac_ver() + version = tuple(map(int, version_str.split(".")[:2])) + if (10, 15) < version < (11, 0): + # When built against an older macOS SDK, Python will report macOS 10.16 + # instead of the real version. + version_str = call( + sys.executable, + "-sS", + "-c", + "import platform; print(platform.mac_ver()[0])", + env={"SYSTEM_VERSION_COMPAT": "0"}, + capture_stdout=True, + ) + version = tuple(map(int, version_str.split(".")[:2])) + return typing.cast(tuple[int, int], version) + + +@functools.cache +def get_test_macosx_deployment_target() -> str: + version = get_macos_version() + if version >= (11, 0): + return f"{version[0]}.0" + return f"{version[0]}.{version[1]}" + + +def get_macos_sdks() -> list[str]: + output = call("xcodebuild", "-showsdks", capture_stdout=True) + return [m.group(1) for m in re.finditer(r"-sdk (macosx\S+)", output)] + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class PythonConfiguration: + version: str + identifier: str + url: str + + +def all_python_configurations() -> list[PythonConfiguration]: + config_dicts = resources.read_python_configs("macos") + return [PythonConfiguration(**item) for item in config_dicts] + + +def get_python_configurations( + build_selector: BuildSelector, architectures: Set[Architecture] +) -> list[PythonConfiguration]: + python_configurations = all_python_configurations() + + # filter out configs that don't match any of the selected architectures + python_configurations = [ + c + for c in python_configurations + if any(c.identifier.endswith(a.value) for a in architectures) + ] + + # skip builds as required by BUILD/SKIP + python_configurations = [c for c in python_configurations if build_selector(c.identifier)] + + # filter-out some cross-compilation configs with PyPy and GraalPy: + # can't build arm64 on x86_64 + # rosetta allows to build x86_64 on arm64 + if platform.machine() == "x86_64": + python_configurations_before = set(python_configurations) + python_configurations = [ + c + for c in python_configurations + if not (c.identifier.startswith(("pp", "gp")) and c.identifier.endswith("arm64")) + ] + removed_elements = python_configurations_before - set(python_configurations) + if removed_elements: + ids = ", ".join(c.identifier for c in removed_elements) + log.quiet( + unwrap( + f""" + Note: {ids} {"was" if len(removed_elements) == 1 else "were"} + selected, but can't be built on x86_64 so will be skipped automatically. + """ + ) + ) + + return python_configurations + + +def install_cpython(_tmp: Path, version: str, url: str, free_threading: bool) -> Path: + ft = "T" if free_threading else "" + installation_path = Path(f"/Library/Frameworks/Python{ft}.framework/Versions/{version}") + with FileLock(CIBW_CACHE_PATH / f"cpython{version}.lock"): + installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines() + # if this version of python isn't installed, get it from python.org and install + python_package_identifier = f"org.python.Python.Python{ft}Framework-{version}" + if python_package_identifier not in installed_system_packages: + if detect_ci_provider() is None: + # if running locally, we don't want to install CPython with sudo + # let the user know & provide a link to the installer + msg = inspect.cleandoc( + f""" + Error: CPython {version} is not installed. + cibuildwheel will not perform system-wide installs when running outside of CI. + To build locally, install CPython {version} on this machine, or, disable this + version of Python using CIBW_SKIP=cp{version.replace(".", "")}-macosx_* + For portable builds, cibuildwheel needs the official builds from python.org. + Download link: {url} + """ + ) + raise errors.FatalError(msg) + python_filename = url.split("/")[-1] + pkg_path = CIBW_CACHE_PATH / "cpython-installer" / python_filename + if not pkg_path.exists(): + download(url, pkg_path) + args = [] + if version.startswith("3.13"): + # Python 3.13 is the first version to have a free-threading option + args += ["-applyChoiceChangesXML", str(resources.FREE_THREAD_ENABLE_313.resolve())] + elif version.startswith("3.14"): + args += ["-applyChoiceChangesXML", str(resources.FREE_THREAD_ENABLE_314.resolve())] + call("sudo", "installer", "-pkg", pkg_path, *args, "-target", "/") + pkg_path.unlink() + env = os.environ.copy() + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + if free_threading: + call(installation_path / f"bin/python{version}t", "-m", "ensurepip", env=env) + call( + installation_path / f"bin/python{version}t", + resources.INSTALL_CERTIFI_SCRIPT, + env=env, + ) + else: + call(installation_path / "bin/python3", resources.INSTALL_CERTIFI_SCRIPT, env=env) + + return installation_path / "bin" / (f"python{version}t" if free_threading else "python3") + + +def install_pypy(tmp: Path, url: str) -> Path: + pypy_tar_bz2 = url.rsplit("/", 1)[-1] + extension = ".tar.bz2" + assert pypy_tar_bz2.endswith(extension) + installation_path = CIBW_CACHE_PATH / pypy_tar_bz2[: -len(extension)] + with FileLock(str(installation_path) + ".lock"): + if not installation_path.exists(): + downloaded_tar_bz2 = tmp / pypy_tar_bz2 + download(url, downloaded_tar_bz2) + installation_path.parent.mkdir(parents=True, exist_ok=True) + call("tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2) + downloaded_tar_bz2.unlink() + return installation_path / "bin" / "pypy3" + + +def install_graalpy(tmp: Path, url: str) -> Path: + graalpy_archive = url.rsplit("/", 1)[-1] + extension = ".tar.gz" + assert graalpy_archive.endswith(extension) + installation_path = CIBW_CACHE_PATH / graalpy_archive[: -len(extension)] + with FileLock(str(installation_path) + ".lock"): + if not installation_path.exists(): + downloaded_archive = tmp / graalpy_archive + download(url, downloaded_archive) + installation_path.mkdir(parents=True) + # GraalPy top-folder name is inconsistent with archive name + call("tar", "-C", installation_path, "--strip-components=1", "-xzf", downloaded_archive) + downloaded_archive.unlink() + return installation_path / "bin" / "graalpy" + + +def setup_python( + tmp: Path, + python_configuration: PythonConfiguration, + dependency_constraint: Path | None, + environment: ParsedEnvironment, + build_frontend: BuildFrontendName, +) -> tuple[Path, dict[str, str]]: + uv_path = find_uv() + use_uv = build_frontend == "build[uv]" + + tmp.mkdir() + implementation_id = python_configuration.identifier.split("-")[0] + log.step(f"Installing Python {implementation_id}...") + if implementation_id.startswith("cp"): + free_threading = "t-macos" in python_configuration.identifier + base_python = install_cpython( + tmp, python_configuration.version, python_configuration.url, free_threading + ) + + elif implementation_id.startswith("pp"): + base_python = install_pypy(tmp, python_configuration.url) + elif implementation_id.startswith("gp"): + base_python = install_graalpy(tmp, python_configuration.url) + else: + msg = "Unknown Python implementation" + raise ValueError(msg) + assert base_python.exists(), ( + f"{base_python.name} not found, has {list(base_python.parent.iterdir())}" + ) + + log.step("Setting up build environment...") + venv_path = tmp / "venv" + env = virtualenv( + python_configuration.version, + base_python, + venv_path, + dependency_constraint, + use_uv=use_uv, + ) + venv_bin_path = venv_path / "bin" + assert venv_bin_path.exists() + # Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`, + # `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175 + # Also fix an issue with the shebang of installed scripts inside the + # testing virtualenv- see https://github.com/theacodes/nox/issues/44 and + # https://github.com/pypa/virtualenv/issues/620 + # Also see https://github.com/python/cpython/pull/9516 + env.pop("__PYVENV_LAUNCHER__", None) + + # we version pip ourselves, so we don't care about pip version checking + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + # Apply our environment after pip is ready + env = environment.as_dictionary(prev_environment=env) + + # check what Python version we're on + which_python = call("which", "python", env=env, capture_stdout=True).strip() + print(which_python) + if which_python != str(venv_bin_path / "python"): + msg = "python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." + raise errors.FatalError(msg) + call("python", "--version", env=env) + + # check what pip version we're on + if not use_uv: + assert (venv_bin_path / "pip").exists() + which_pip = call("which", "pip", env=env, capture_stdout=True).strip() + print(which_pip) + if which_pip != str(venv_bin_path / "pip"): + msg = "pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." + raise errors.FatalError(msg) + call("pip", "--version", env=env) + + config_is_arm64 = python_configuration.identifier.endswith("arm64") + config_is_universal2 = python_configuration.identifier.endswith("universal2") + + # Set MACOSX_DEPLOYMENT_TARGET, if the user didn't set it. + # For arm64, the minimal deployment target is 11.0. + # On x86_64 (or universal2), use 10.9 as a default. + # CPython 3.12.6+ needs 10.13. + if config_is_arm64: + default_target = "11.0" + elif Version(python_configuration.version) >= Version("3.12"): + default_target = "10.13" + elif python_configuration.identifier.startswith("pp") and Version( + python_configuration.version + ) >= Version("3.9"): + default_target = "10.15" + else: + default_target = "10.9" + env.setdefault("MACOSX_DEPLOYMENT_TARGET", default_target) + + # This is a floor, it can't be set lower than the default_target. + if Version(env["MACOSX_DEPLOYMENT_TARGET"]) < Version(default_target): + log.warning( + f"Bumping MACOSX_DEPLOYMENT_TARGET ({env['MACOSX_DEPLOYMENT_TARGET']}) to the minimum required ({default_target})." + ) + env["MACOSX_DEPLOYMENT_TARGET"] = default_target + + if config_is_arm64: + # macOS 11 is the first OS with arm64 support, so the wheels + # have that as a minimum. + env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-11.0-arm64") + env.setdefault("ARCHFLAGS", "-arch arm64") + elif config_is_universal2: + env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-10.9-universal2") + env.setdefault("ARCHFLAGS", "-arch arm64 -arch x86_64") + elif python_configuration.identifier.endswith("x86_64"): + # even on the macos11.0 Python installer, on the x86_64 side it's + # compatible back to 10.9. + env.setdefault("_PYTHON_HOST_PLATFORM", "macosx-10.9-x86_64") + env.setdefault("ARCHFLAGS", "-arch x86_64") + + building_arm64 = config_is_arm64 or config_is_universal2 + if building_arm64 and get_macos_version() < (10, 16) and "SDKROOT" not in env: + # xcode 12.2 or higher can build arm64 on macos 10.15 or below, but + # needs the correct SDK selected. + sdks = get_macos_sdks() + + # Different versions of Xcode contain different SDK versions... + # we're happy with anything newer than macOS 11.0 + arm64_compatible_sdks = [s for s in sdks if not s.startswith("macosx10.")] + + if not arm64_compatible_sdks: + log.warning( + unwrap( + """ + SDK for building arm64-compatible wheels not found. You need Xcode 12.2 or later + to build universal2 or arm64 wheels. + """ + ) + ) + else: + env.setdefault("SDKROOT", arm64_compatible_sdks[0]) + + log.step("Installing build tools...") + if build_frontend == "pip": + call( + "pip", + "install", + "--upgrade", + "delocate", + *constraint_flags(dependency_constraint), + env=env, + ) + elif build_frontend == "build": + call( + "pip", + "install", + "--upgrade", + "delocate", + "build[virtualenv]", + *constraint_flags(dependency_constraint), + env=env, + ) + elif build_frontend == "build[uv]": + assert uv_path is not None + call( + uv_path, + "pip", + "install", + "--upgrade", + "delocate", + "build[virtualenv, uv]", + *constraint_flags(dependency_constraint), + env=env, + ) + else: + assert_never(build_frontend) + + return base_python, env + + +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) + + if not python_configurations: + return + + try: + before_all_options_identifier = python_configurations[0].identifier + before_all_options = options.build_options(before_all_options_identifier) + + if before_all_options.before_all: + log.step("Running before_all...") + env = before_all_options.environment.as_dictionary(prev_environment=os.environ) + env.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.9") + before_all_prepared = prepare_command( + before_all_options.before_all, project=".", package=before_all_options.package_dir + ) + shell(before_all_prepared, env=env) + + built_wheels: list[Path] = [] + + for config in python_configurations: + build_options = options.build_options(config.identifier) + build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + use_uv = build_frontend.name == "build[uv]" + uv_path = find_uv() + if use_uv and uv_path is None: + msg = "uv not found" + raise AssertionError(msg) + pip = ["pip"] if not use_uv else [str(uv_path), "pip"] + log.build_start(config.identifier) + + identifier_tmp_dir = tmp_path / config.identifier + identifier_tmp_dir.mkdir() + built_wheel_dir = identifier_tmp_dir / "built_wheel" + repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" + + config_is_arm64 = config.identifier.endswith("arm64") + config_is_universal2 = config.identifier.endswith("universal2") + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=config.version, tmp_dir=identifier_tmp_dir + ) + + base_python, env = setup_python( + identifier_tmp_dir / "build", + config, + constraints_path, + build_options.environment, + build_frontend.name, + ) + pip_version = None if use_uv else get_pip_version(env) + + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + ) + repaired_wheel = compatible_wheel + else: + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, project=".", package=build_options.package_dir + ) + shell(before_build_prepared, env=env) + + log.step("Building wheel...") + built_wheel_dir.mkdir() + + extra_flags = get_build_frontend_extra_flags( + build_frontend, build_options.build_verbosity, build_options.config_settings + ) + + build_env = env.copy() + if pip_version is not None: + build_env["VIRTUALENV_PIP"] = pip_version + if constraints_path: + combine_constraints( + build_env, constraints_path, identifier_tmp_dir if use_uv else None + ) + + if build_frontend.name == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + build_options.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *extra_flags, + env=build_env, + ) + elif build_frontend.name == "build" or build_frontend.name == "build[uv]": + if use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: + extra_flags.append("--installer=uv") + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + *extra_flags, + env=build_env, + ) + else: + assert_never(build_frontend) + + built_wheel = next(built_wheel_dir.glob("*.whl")) + + repaired_wheel_dir.mkdir() + + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + + if build_options.repair_command: + log.step("Repairing wheel...") + + if config_is_universal2: + delocate_archs = "x86_64,arm64" + elif config_is_arm64: + delocate_archs = "arm64" + else: + delocate_archs = "x86_64" + + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, + delocate_archs=delocate_archs, + ) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) + + try: + repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + except StopIteration: + raise errors.RepairStepProducedNoWheelError() from None + + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise errors.AlreadyBuiltWheelError(repaired_wheel.name) + + log.step_end() + + if build_options.test_command and build_options.test_selector(config.identifier): + machine_arch = platform.machine() + python_arch = call( + "python", + "-sSc", + "import platform; print(platform.machine())", + env=env, + capture_stdout=True, + ).strip() + testing_archs: list[Literal["x86_64", "arm64"]] + + if config_is_arm64: + testing_archs = ["arm64"] + elif config_is_universal2: + testing_archs = ["x86_64", "arm64"] + else: + testing_archs = ["x86_64"] + + for testing_arch in testing_archs: + if config_is_universal2: + arch_specific_identifier = f"{config.identifier}:{testing_arch}" + if not build_options.test_selector(arch_specific_identifier): + continue + + if machine_arch == "x86_64" and testing_arch == "arm64": + if config_is_arm64: + log.warning( + unwrap( + """ + While arm64 wheels can be built on x86_64, they cannot be + tested. Consider building arm64 wheels natively, if your CI + provider offers this. To silence this warning, set + `CIBW_TEST_SKIP: "*-macosx_arm64"`. + """ + ) + ) + elif config_is_universal2: + log.warning( + unwrap( + """ + While universal2 wheels can be built on x86_64, the arm64 part + of the wheel cannot be tested on x86_64. Consider building + universal2 wheels on an arm64 runner, if your CI provider offers + this. Notably, an arm64 runner can also test the x86_64 part of + the wheel, through Rosetta emulation. To silence this warning, + set `CIBW_TEST_SKIP: "*-macosx_universal2:arm64"`. + """ + ) + ) + else: + msg = "unreachable" + raise RuntimeError(msg) + + # skip this test + continue + + is_cp38 = config.identifier.startswith("cp38-") + if testing_arch == "arm64" and is_cp38 and python_arch != "arm64": + log.warning( + unwrap( + """ + While cibuildwheel can build CPython 3.8 universal2/arm64 wheels, we + cannot test the arm64 part of them, even when running on an Apple + Silicon machine. This is because we use the x86_64 installer of + CPython 3.8. See the discussion in + https://github.com/pypa/cibuildwheel/pull/1169 for the details. To + silence this warning, set `CIBW_TEST_SKIP: "cp38-macosx_*:arm64"`. + """ + ) + ) + + # skip this test + continue + + log.step( + "Testing wheel..." + if testing_arch == machine_arch + else f"Testing wheel on {testing_arch}..." + ) + + arch_prefix = [] + uv_arch_args = [] + if testing_arch != machine_arch: + if machine_arch == "arm64" and testing_arch == "x86_64": + # rosetta2 will provide the emulation with just the arch prefix. + arch_prefix = ["arch", "-x86_64"] + uv_arch_args = ["--python-platform", "x86_64-apple-darwin"] + else: + msg = f"don't know how to emulate {testing_arch} on {machine_arch}" + raise RuntimeError(msg) + + # define a custom 'call' function that adds the arch prefix each time + call_with_arch = functools.partial(call, *arch_prefix) + shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c") + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}" + virtualenv_env = virtualenv( + config.version, + base_python, + venv_dir, + None, + use_uv=use_uv, + env=env, + pip_version=pip_version, + ) + if use_uv: + pip_install = functools.partial(call, *pip, "install", *uv_arch_args) + else: + pip_install = functools.partial(call_with_arch, *pip, "install") + + virtualenv_env["MACOSX_DEPLOYMENT_TARGET"] = get_test_macosx_deployment_target() + + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) + + # check that we are using the Python from the virtual environment + call_with_arch("which", "python", env=virtualenv_env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, + ) + shell_with_arch(before_test_prepared, env=virtualenv_env) + + # install the wheel + if is_cp38 and python_arch == "x86_64": + virtualenv_env_install_wheel = virtualenv_env.copy() + virtualenv_env_install_wheel["SYSTEM_VERSION_COMPAT"] = "0" + log.notice( + unwrap( + """ + Setting SYSTEM_VERSION_COMPAT=0 to ensure CPython 3.8 can get + correct macOS version and allow installation of wheels with + MACOSX_DEPLOYMENT_TARGET >= 11.0. + See https://github.com/pypa/cibuildwheel/issues/1767 for the + details. + """ + ) + ) + else: + virtualenv_env_install_wheel = virtualenv_env + + pip_install( + f"{repaired_wheel}{build_options.test_extras}", + env=virtualenv_env_install_wheel, + ) + + # test the wheel + if build_options.test_requires: + pip_install( + *build_options.test_requires, + env=virtualenv_env_install_wheel, + ) + + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + build_options.test_command, + project=Path.cwd(), + package=build_options.package_dir.resolve(), + wheel=repaired_wheel, + ) + + test_cwd = identifier_tmp_dir / "test_cwd" + + if build_options.test_sources: + # only create test_cwd if it doesn't already exist - it + # may have been created during a previous `testing_arch` + if not test_cwd.exists(): + test_cwd.mkdir() + copy_test_sources( + build_options.test_sources, + Path.cwd(), + test_cwd, + ) + else: + # Use the test_fail.py file to raise a nice error if the user + # tries to run tests in the cwd + test_cwd.mkdir(exist_ok=True) + (test_cwd / "test_fail.py").write_text( + resources.TEST_FAIL_CWD_FILE.read_text() + ) + + shell_with_arch(test_command_prepared, cwd=test_cwd, env=virtualenv_env) + + # we're all done here; move it to output (overwrite existing) + if compatible_wheel is None: + output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) + moved_wheel = move_file(repaired_wheel, output_wheel) + if moved_wheel != output_wheel.resolve(): + log.warning( + f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" + ) + built_wheels.append(output_wheel) + + # clean up + shutil.rmtree(identifier_tmp_dir) + + log.build_end() + except subprocess.CalledProcessError as error: + msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" + raise errors.FatalError(msg) from error diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py new file mode 100644 index 000000000..4e545260f --- /dev/null +++ b/cibuildwheel/platforms/pyodide.py @@ -0,0 +1,551 @@ +import dataclasses +import functools +import json +import os +import shutil +import sys +import tomllib +import typing +from collections.abc import Set +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Final, TypedDict + +from filelock import FileLock + +from .. import errors +from ..architecture import Architecture +from ..environment import ParsedEnvironment +from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags +from ..logger import log +from ..options import Options +from ..selector import BuildSelector +from ..util import resources +from ..util.cmd import call, shell +from ..util.file import ( + CIBW_CACHE_PATH, + copy_test_sources, + download, + extract_tar, + extract_zip, + move_file, +) +from ..util.helpers import prepare_command, unwrap, unwrap_preserving_paragraphs +from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version +from ..util.python_build_standalone import ( + PythonBuildStandaloneError, + create_python_build_standalone_environment, +) +from ..venv import constraint_flags, virtualenv + +IS_WIN: Final[bool] = sys.platform.startswith("win") + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class PythonConfiguration: + version: str + identifier: str + default_pyodide_version: str + node_version: str + + +class PyodideXBuildEnvInfoVersionRange(TypedDict): + min: str | None + max: str | None + + +class PyodideXBuildEnvInfo(TypedDict): + version: str + python: str + emscripten: str + pyodide_build: PyodideXBuildEnvInfoVersionRange + compatible: bool + + +@functools.cache +def ensure_node(major_version: str) -> Path: + with resources.NODEJS.open("rb") as f: + loaded_file = tomllib.load(f) + version = str(loaded_file[major_version]) + base_url = str(loaded_file["url"]) + ext = "zip" if IS_WIN else "tar.xz" + platform = "win" if IS_WIN else ("darwin" if sys.platform.startswith("darwin") else "linux") + linux_arch = Architecture.native_arch("linux") + assert linux_arch is not None + arch = {"x86_64": "x64", "i686": "x86", "aarch64": "arm64"}.get( + linux_arch.value, linux_arch.value + ) + name = f"node-{version}-{platform}-{arch}" + path = CIBW_CACHE_PATH / name + with FileLock(str(path) + ".lock"): + if not path.exists(): + url = f"{base_url}{version}/{name}.{ext}" + with TemporaryDirectory() as tmp_path: + archive = Path(tmp_path) / f"{name}.{ext}" + download(url, archive) + if ext == "zip": + extract_zip(archive, path.parent) + else: + extract_tar(archive, path.parent) + assert path.exists() + if not IS_WIN: + return path / "bin" + return path + + +def install_emscripten(tmp: Path, version: str) -> Path: + url = f"https://github.com/emscripten-core/emsdk/archive/refs/tags/%7Bversion%7D.zip" + installation_path = CIBW_CACHE_PATH / f"emsdk-{version}" + emsdk_path = installation_path / f"emsdk-{version}/emsdk" + emcc_path = installation_path / f"emsdk-{version}/upstream/emscripten/emcc" + with FileLock(f"{installation_path}.lock"): + if installation_path.exists(): + return emcc_path + emsdk_zip = tmp / "emsdk.zip" + download(url, emsdk_zip) + installation_path.mkdir() + extract_zip(emsdk_zip, installation_path) + call(emsdk_path, "install", version) + call(emsdk_path, "activate", version) + + return emcc_path + + +def get_all_xbuildenv_version_info(env: dict[str, str]) -> list[PyodideXBuildEnvInfo]: + xbuildenvs_info_str = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + env=env, + cwd=CIBW_CACHE_PATH, + capture_stdout=True, + ).strip() + + xbuildenvs_info = json.loads(xbuildenvs_info_str) + + if "environments" not in xbuildenvs_info: + msg = f"Invalid xbuildenvs info, got {xbuildenvs_info}" + raise ValueError(msg) + + return typing.cast(list[PyodideXBuildEnvInfo], xbuildenvs_info["environments"]) + + +def get_xbuildenv_version_info( + env: dict[str, str], version: str, pyodide_build_version: str +) -> PyodideXBuildEnvInfo: + xbuildenvs_info = get_all_xbuildenv_version_info(env) + for xbuildenv_info in xbuildenvs_info: + if xbuildenv_info["version"] == version: + return xbuildenv_info + + msg = unwrap(f""" + Could not find Pyodide cross-build environment version {version} in the available + versions as reported by pyodide-build v{pyodide_build_version}. + Available pyodide xbuildenv versions are: + {", ".join(e["version"] for e in xbuildenvs_info if e["compatible"])} + """) + raise errors.FatalError(msg) + + +# The default pyodide xbuildenv version that's specified in +# build-platforms.toml is compatible with the pyodide-build version that's +# pinned in the bundled constraints file. But if the user changes +# pyodide-version and/or dependency-constraints in the cibuildwheel config, we +# need to check if the xbuildenv version is compatible with the pyodide-build +# version. +def validate_pyodide_build_version( + xbuildenv_info: PyodideXBuildEnvInfo, pyodide_build_version: str +) -> None: + """ + Validate the Pyodide version is compatible with the installed + pyodide-build version. + """ + + pyodide_version = xbuildenv_info["version"] + + if not xbuildenv_info["compatible"]: + msg = unwrap_preserving_paragraphs(f""" + The Pyodide xbuildenv version {pyodide_version} is not compatible + with the pyodide-build version {pyodide_build_version}. Please use + the 'pyodide xbuildenv search --all' command to find a compatible + version. + + Set the pyodide-build version using the `dependency-constraints` + option, or set the Pyodide xbuildenv version using the + `pyodide-version` option. + """) + raise errors.FatalError(msg) + + +def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: + """Install a particular Pyodide xbuildenv version and set a path to the Pyodide root.""" + # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of + # pyodide-build are uncoupled from the versions of Pyodide. So, we specify + # both the pyodide-build version and the Pyodide version in the temp path. + xbuildenv_cache_path = CIBW_CACHE_PATH / f"pyodide-build-{pyodide_build_version}" + pyodide_root = xbuildenv_cache_path / pyodide_version / "xbuildenv" / "pyodide-root" + + with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): + if pyodide_root.exists(): + return str(pyodide_root) + + # We don't want to mutate env but we need to delete any existing + # PYODIDE_ROOT so copy it first. + env = dict(env) + env.pop("PYODIDE_ROOT", None) + + # Install the xbuildenv + call( + "pyodide", + "xbuildenv", + "install", + "--path", + str(xbuildenv_cache_path), + pyodide_version, + env=env, + cwd=CIBW_CACHE_PATH, + ) + assert pyodide_root.exists() + + return str(pyodide_root) + + +def get_base_python(tmp: Path, python_configuration: PythonConfiguration) -> Path: + try: + return create_python_build_standalone_environment( + python_version=python_configuration.version, + temp_dir=tmp, + cache_dir=CIBW_CACHE_PATH, + ) + except PythonBuildStandaloneError as e: + msg = unwrap(f""" + Failed to create a Python build environment: + {e} + """) + raise errors.FatalError(msg) from e + + +def setup_python( + tmp: Path, + python_configuration: PythonConfiguration, + constraints_path: Path | None, + environment: ParsedEnvironment, + user_pyodide_version: str | None, +) -> dict[str, str]: + log.step("Installing a base python environment...") + base_python = get_base_python(tmp / "base", python_configuration) + + log.step("Setting up build environment...") + pyodide_version = user_pyodide_version or python_configuration.default_pyodide_version + venv_path = tmp / "venv" + env = virtualenv(python_configuration.version, base_python, venv_path, None, use_uv=False) + venv_bin_path = venv_path / "bin" + assert venv_bin_path.exists() + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + # upgrade pip to the version matching our constraints + # if necessary, reinstall it to ensure that it's available on PATH as 'pip' + call( + "python", + "-m", + "pip", + "install", + "--upgrade", + "pip", + *constraint_flags(constraints_path), + env=env, + cwd=venv_path, + ) + + env = environment.as_dictionary(prev_environment=env) + + # check what Python version we're on + which_python = call("which", "python", env=env, capture_stdout=True).strip() + print(which_python) + if which_python != str(venv_bin_path / "python"): + msg = "python available on PATH doesn't match our venv instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." + raise errors.FatalError(msg) + call("python", "--version", env=env) + + # check what pip version we're on + assert (venv_bin_path / "pip").exists() + which_pip = call("which", "pip", env=env, capture_stdout=True).strip() + print(which_pip) + if which_pip != str(venv_bin_path / "pip"): + msg = "pip available on PATH doesn't match our venv instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." + raise errors.FatalError(msg) + call("pip", "--version", env=env) + + log.step("Installing build tools...") + call( + "pip", + "install", + "--upgrade", + "auditwheel-emscripten", + "build[virtualenv]", + "pyodide-build", + *constraint_flags(constraints_path), + env=env, + ) + + pyodide_build_version = call( + "python", + "-c", + "from importlib.metadata import version; print(version('pyodide-build'))", + env=env, + capture_stdout=True, + ).strip() + + xbuildenv_info = get_xbuildenv_version_info(env, pyodide_version, pyodide_build_version) + validate_pyodide_build_version( + xbuildenv_info=xbuildenv_info, + pyodide_build_version=pyodide_build_version, + ) + + emscripten_version = xbuildenv_info["emscripten"] + log.step(f"Installing Emscripten version: {emscripten_version} ...") + emcc_path = install_emscripten(tmp, emscripten_version) + + env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) + + log.step(f"Installing Pyodide xbuildenv version: {pyodide_version} ...") + env["PYODIDE_ROOT"] = install_xbuildenv(env, pyodide_build_version, pyodide_version) + + return env + + +def all_python_configurations() -> list[PythonConfiguration]: + full_python_configs = resources.read_python_configs("pyodide") + return [PythonConfiguration(**item) for item in full_python_configs] + + +def get_python_configurations( + build_selector: BuildSelector, + architectures: Set[Architecture], # noqa: ARG001 +) -> list[PythonConfiguration]: + return [c for c in all_python_configurations() if build_selector(c.identifier)] + + +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) + + if not python_configurations: + return + + try: + before_all_options_identifier = python_configurations[0].identifier + before_all_options = options.build_options(before_all_options_identifier) + + if before_all_options.before_all: + log.step("Running before_all...") + env = before_all_options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command( + before_all_options.before_all, project=".", package=before_all_options.package_dir + ) + shell(before_all_prepared, env=env) + + built_wheels: list[Path] = [] + + for config in python_configurations: + build_options = options.build_options(config.identifier) + build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + + if build_frontend.name == "pip": + msg = "The pyodide platform doesn't support pip frontend" + raise errors.FatalError(msg) + + log.build_start(config.identifier) + + identifier_tmp_dir = tmp_path / config.identifier + + built_wheel_dir = identifier_tmp_dir / "built_wheel" + repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" + identifier_tmp_dir.mkdir() + built_wheel_dir.mkdir() + repaired_wheel_dir.mkdir() + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir + ) + + env = setup_python( + tmp=identifier_tmp_dir / "build", + python_configuration=config, + constraints_path=constraints_path, + environment=build_options.environment, + user_pyodide_version=build_options.pyodide_version, + ) + pip_version = get_pip_version(env) + # The Pyodide command line runner mounts all directories in the host + # filesystem into the Pyodide file system, except for the custom + # file systems /dev, /lib, /proc, and /tmp. Mounting the mount + # points for alternate file systems causes some mysterious failure + # of the process (it just quits without any clear error). + # + # Because of this, by default Pyodide can't see anything under /tmp. + # This environment variable tells it also to mount our temp + # directory. + oldmounts = "" + extra_mounts = [str(identifier_tmp_dir)] + if Path.cwd().is_relative_to("/tmp"): + extra_mounts.append(str(Path.cwd())) + + if "_PYODIDE_EXTRA_MOUNTS" in env: + oldmounts = env["_PYODIDE_EXTRA_MOUNTS"] + ":" + env["_PYODIDE_EXTRA_MOUNTS"] = oldmounts + ":".join(extra_mounts) + + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + ) + built_wheel = compatible_wheel + else: + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, project=".", package=build_options.package_dir + ) + shell(before_build_prepared, env=env) + + log.step("Building wheel...") + + extra_flags = get_build_frontend_extra_flags( + build_frontend, build_options.build_verbosity, build_options.config_settings + ) + + build_env = env.copy() + if constraints_path: + combine_constraints(build_env, constraints_path, identifier_tmp_dir) + build_env["VIRTUALENV_PIP"] = pip_version + call( + "pyodide", + "build", + build_options.package_dir, + f"--outdir={built_wheel_dir}", + *extra_flags, + env=build_env, + ) + built_wheel = next(built_wheel_dir.glob("*.whl")) + + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + + if build_options.repair_command: + log.step("Repairing wheel...") + + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, + ) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) + + repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise errors.AlreadyBuiltWheelError(repaired_wheel.name) + + if build_options.test_command and build_options.test_selector(config.identifier): + log.step("Testing wheel...") + + venv_dir = identifier_tmp_dir / "venv-test" + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + + virtualenv_env = env.copy() + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(ensure_node(config.node_version)), + virtualenv_env["PATH"], + ] + ) + + # pyodide venv uses virtualenv under the hood + # use the pip embedded with virtualenv & disable network updates + virtualenv_create_env = virtualenv_env.copy() + virtualenv_create_env["VIRTUALENV_PIP"] = pip_version + virtualenv_create_env["VIRTUALENV_NO_PERIODIC_UPDATE"] = "1" + + call("pyodide", "venv", venv_dir, env=virtualenv_create_env) + + virtualenv_env["PATH"] = os.pathsep.join( + [ + str(venv_dir / "bin"), + virtualenv_env["PATH"], + ] + ) + virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) + + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) + + # check that we are using the Python from the virtual environment + call("which", "python", env=virtualenv_env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, + wheel=repaired_wheel, + ) + shell(before_test_prepared, env=virtualenv_env) + + # install the wheel + call( + "pip", + "install", + f"{repaired_wheel}{build_options.test_extras}", + env=virtualenv_env, + ) + + # test the wheel + if build_options.test_requires: + call("pip", "install", *build_options.test_requires, env=virtualenv_env) + + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + build_options.test_command, + project=Path.cwd(), + package=build_options.package_dir.resolve(), + ) + + test_cwd = identifier_tmp_dir / "test_cwd" + test_cwd.mkdir(exist_ok=True) + + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + Path.cwd(), + test_cwd, + ) + else: + # Use the test_fail.py file to raise a nice error if the user + # tries to run tests in the cwd + (test_cwd / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) + + shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) + + # we're all done here; move it to output (overwrite existing) + if compatible_wheel is None: + output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) + moved_wheel = move_file(repaired_wheel, output_wheel) + if moved_wheel != output_wheel.resolve(): + log.warning( + f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" + ) + built_wheels.append(output_wheel) + + finally: + pass diff --git a/cibuildwheel/platforms/windows.py b/cibuildwheel/platforms/windows.py new file mode 100644 index 000000000..c57a53de0 --- /dev/null +++ b/cibuildwheel/platforms/windows.py @@ -0,0 +1,627 @@ +import dataclasses +import json +import os +import platform as platform_module +import shutil +import subprocess +import textwrap +from collections.abc import MutableMapping, Set +from functools import cache +from pathlib import Path +from typing import assert_never + +from filelock import FileLock + +from .. import errors +from ..architecture import Architecture +from ..environment import ParsedEnvironment +from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_frontend_extra_flags +from ..logger import log +from ..options import Options +from ..selector import BuildSelector +from ..util import resources +from ..util.cmd import call, shell +from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, extract_zip, move_file +from ..util.helpers import prepare_command, unwrap +from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version +from ..venv import constraint_flags, find_uv, virtualenv + + +def get_nuget_args( + version: str, arch: str, free_threaded: bool, output_directory: Path +) -> list[str]: + package_name = { + "32": "pythonx86", + "64": "python", + "ARM64": "pythonarm64", + # Aliases for platform.machine() return values + "x86": "pythonx86", + "AMD64": "python", + }[arch] + if free_threaded: + package_name = f"{package_name}-freethreaded" + return [ + package_name, + "-Version", + version, + "-FallbackSource", + "https://api.nuget.org/v3/index.json", + "-OutputDirectory", + str(output_directory), + ] + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class PythonConfiguration: + version: str + arch: str + identifier: str + url: str | None = None + + +def all_python_configurations() -> list[PythonConfiguration]: + config_dicts = resources.read_python_configs("windows") + return [PythonConfiguration(**item) for item in config_dicts] + + +def get_python_configurations( + build_selector: BuildSelector, + architectures: Set[Architecture], +) -> list[PythonConfiguration]: + python_configurations = all_python_configurations() + + map_arch = {"32": Architecture.x86, "64": Architecture.AMD64, "ARM64": Architecture.ARM64} + + # skip builds as required + python_configurations = [ + c + for c in python_configurations + if build_selector(c.identifier) and map_arch[c.arch] in architectures + ] + + return python_configurations + + +@cache +def _ensure_nuget() -> Path: + nuget = CIBW_CACHE_PATH / "nuget.exe" + with FileLock(str(nuget) + ".lock"): + if not nuget.exists(): + download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget) + return nuget + + +def install_cpython(configuration: PythonConfiguration, arch: str | None = None) -> Path: + version = configuration.version + free_threaded = "t-" in configuration.identifier + if arch is None: + arch = configuration.arch + base_output_dir = CIBW_CACHE_PATH / "nuget-cpython" + nuget_args = get_nuget_args(version, arch, free_threaded, base_output_dir) + installation_path = base_output_dir / (nuget_args[0] + "." + version) / "tools" + free_threaded_str = "-freethreaded" if free_threaded else "" + with FileLock(str(base_output_dir) + f"-{version}{free_threaded_str}-{arch}.lock"): + if not installation_path.exists(): + nuget = _ensure_nuget() + call(nuget, "install", *nuget_args) + return installation_path / "python.exe" + + +def install_pypy(tmp: Path, arch: str, url: str) -> Path: + assert arch == "64" + assert "win64" in url + # Inside the PyPy zip file is a directory with the same name + zip_filename = url.rsplit("/", 1)[-1] + extension = ".zip" + assert zip_filename.endswith(extension) + installation_path = CIBW_CACHE_PATH / zip_filename[: -len(extension)] + with FileLock(str(installation_path) + ".lock"): + if not installation_path.exists(): + pypy_zip = tmp / zip_filename + download(url, pypy_zip) + # Extract to the parent directory because the zip file still contains a directory + extract_zip(pypy_zip, installation_path.parent) + return installation_path / "python.exe" + + +def install_graalpy(tmp: Path, url: str) -> Path: + zip_filename = url.rsplit("/", 1)[-1] + extension = ".zip" + assert zip_filename.endswith(extension) + installation_path = CIBW_CACHE_PATH / zip_filename[: -len(extension)] + with FileLock(str(installation_path) + ".lock"): + if not installation_path.exists(): + graalpy_zip = tmp / zip_filename + download(url, graalpy_zip) + # Extract to the parent directory because the zip file still contains a directory + extract_zip(graalpy_zip, installation_path.parent) + return installation_path / "bin" / "graalpy.exe" + + +def setup_setuptools_cross_compile( + tmp: Path, + python_configuration: PythonConfiguration, + python_libs_base: Path, + env: MutableMapping[str, str], +) -> None: + distutils_cfg = tmp / "extra-setup.cfg" + env["DIST_EXTRA_CONFIG"] = str(distutils_cfg) + log.notice(f"Setting DIST_EXTRA_CONFIG={distutils_cfg} for cross-compilation") + + # Ensure our additional import libraries are made available, and explicitly + # set the platform name + map_plat = {"32": "win32", "64": "win-amd64", "ARM64": "win-arm64"} + plat_name = map_plat[python_configuration.arch] + + # Set environment variable so that setuptools._distutils.get_platform() + # identifies the target, not the host + vscmd_arg_tgt_arch = {"32": "x86", "64": "x64", "ARM64": "arm64"} + current_tgt_arch = vscmd_arg_tgt_arch[python_configuration.arch] + if (env.get("VSCMD_ARG_TGT_ARCH") or current_tgt_arch) != current_tgt_arch: + msg = f"VSCMD_ARG_TGT_ARCH must be set to {current_tgt_arch!r}, got {env['VSCMD_ARG_TGT_ARCH']!r}. If you're setting up MSVC yourself (e.g. using vcvarsall.bat or msvc-dev-cmd), make sure to target the right architecture. Alternatively, run cibuildwheel without configuring MSVC, and let the build backend handle it." + raise errors.FatalError(msg) + env["VSCMD_ARG_TGT_ARCH"] = current_tgt_arch + + # (This file must be default/locale encoding, so we can't pass 'encoding') + distutils_cfg.write_text( + textwrap.dedent( + f"""\ + [build] + plat_name={plat_name} + [build_ext] + library_dirs={python_libs_base} + plat_name={plat_name} + [bdist_wheel] + plat_name={plat_name} + """ + ) + ) + + # setuptools builds require explicit override of PYD extension + # This is because it always gets the extension from the running + # interpreter, and has no logic to construct it. Currently, CPython's + # extensions follow our identifiers, but if they ever diverge in the + # future, we will need to store new data + log.notice( + f"Setting SETUPTOOLS_EXT_SUFFIX=.{python_configuration.identifier}.pyd for cross-compilation" + ) + env["SETUPTOOLS_EXT_SUFFIX"] = f".{python_configuration.identifier}.pyd" + + # Cross-compilation requires fixes that only exist in setuptools's copy of + # distutils, so ensure that it is activated + # Since not all projects can handle the newer distutils, display a warning + # to help them figure out what may have gone wrong if this breaks for them + log.notice("Setting SETUPTOOLS_USE_DISTUTILS=local as it is required for cross-compilation") + env["SETUPTOOLS_USE_DISTUTILS"] = "local" + + +# These cross-compile setup functions have the same signature by design +def setup_rust_cross_compile( + tmp: Path, # noqa: ARG001 + python_configuration: PythonConfiguration, + python_libs_base: Path, # noqa: ARG001 + env: MutableMapping[str, str], +) -> None: + # Assume that MSVC will be used, because we already know that we are + # cross-compiling. MinGW users can set CARGO_BUILD_TARGET themselves + # and we will respect the existing value. + cargo_target = { + "64": "x86_64-pc-windows-msvc", + "32": "i686-pc-windows-msvc", + "ARM64": "aarch64-pc-windows-msvc", + }.get(python_configuration.arch) + + # CARGO_BUILD_TARGET is the variable used by Cargo and setuptools_rust + if env.get("CARGO_BUILD_TARGET"): + if env["CARGO_BUILD_TARGET"] != cargo_target: + log.notice("Not overriding CARGO_BUILD_TARGET as it has already been set") + # No message if it was set to what we were planning to set it to + elif cargo_target: + log.notice(f"Setting CARGO_BUILD_TARGET={cargo_target} for cross-compilation") + env["CARGO_BUILD_TARGET"] = cargo_target + else: + log.warning( + f"Unable to configure Rust cross-compilation for architecture {python_configuration.arch}" + ) + + +def can_use_uv(python_configuration: PythonConfiguration) -> bool: + conditions = (not python_configuration.identifier.startswith("pp38-"),) + return all(conditions) + + +def setup_python( + tmp: Path, + python_configuration: PythonConfiguration, + dependency_constraint: Path | None, + environment: ParsedEnvironment, + build_frontend: BuildFrontendName, +) -> tuple[Path, dict[str, str]]: + tmp.mkdir() + implementation_id = python_configuration.identifier.split("-")[0] + python_libs_base = None + log.step(f"Installing Python {implementation_id}...") + if implementation_id.startswith("cp"): + native_arch = platform_module.machine() + base_python = install_cpython(python_configuration) + if python_configuration.arch == "ARM64" != native_arch: + # To cross-compile for ARM64, we need a native CPython to run the + # build, and a copy of the ARM64 import libraries ('.\libs\*.lib') + # for any extension modules. + python_libs_base = base_python.parent / "libs" + log.step(f"Installing native Python {native_arch} for cross-compilation...") + base_python = install_cpython(python_configuration, arch=native_arch) + elif implementation_id.startswith("pp"): + assert python_configuration.url is not None + base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url) + elif implementation_id.startswith("gp"): + base_python = install_graalpy(tmp, python_configuration.url or "") + else: + msg = "Unknown Python implementation" + raise ValueError(msg) + assert base_python.exists() + + if build_frontend == "build[uv]" and not can_use_uv(python_configuration): + build_frontend = "build" + + use_uv = build_frontend == "build[uv]" + uv_path = find_uv() + + log.step("Setting up build environment...") + venv_path = tmp / "venv" + env = virtualenv( + python_configuration.version, + base_python, + venv_path, + dependency_constraint, + use_uv=use_uv, + ) + + # set up environment variables for run_with_env + env["PYTHON_VERSION"] = python_configuration.version + env["PYTHON_ARCH"] = python_configuration.arch + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + + # update env with results from CIBW_ENVIRONMENT + env = environment.as_dictionary(prev_environment=env) + + # check what Python version we're on + where_python = call("where", "python", env=env, capture_stdout=True).splitlines()[0].strip() + print(where_python) + if where_python != str(venv_path / "Scripts" / "python.exe"): + msg = "python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." + raise errors.FatalError(msg) + call("python", "--version", env=env) + call("python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\"", env=env) + + # check what pip version we're on + if not use_uv: + assert (venv_path / "Scripts" / "pip.exe").exists() + where_pip = call("where", "pip", env=env, capture_stdout=True).splitlines()[0].strip() + print(where_pip) + if where_pip.strip() != str(venv_path / "Scripts" / "pip.exe"): + msg = "pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." + raise errors.FatalError(msg) + call("pip", "--version", env=env) + + log.step("Installing build tools...") + if build_frontend == "build": + call( + "pip", + "install", + "--upgrade", + "build[virtualenv]", + *constraint_flags(dependency_constraint), + env=env, + ) + elif build_frontend == "build[uv]": + assert uv_path is not None + call( + uv_path, + "pip", + "install", + "--upgrade", + "build[virtualenv]", + *constraint_flags(dependency_constraint), + env=env, + ) + + if python_libs_base: + # Set up the environment for various backends to enable cross-compilation + setup_setuptools_cross_compile(tmp, python_configuration, python_libs_base, env) + setup_rust_cross_compile(tmp, python_configuration, python_libs_base, env) + + if implementation_id.startswith("gp"): + # GraalPy fails to discover compilers, setup the relevant environment + # variables. Adapted from + # https://github.com/microsoft/vswhere/wiki/Start-Developer-Command-Prompt + # Remove when https://github.com/oracle/graalpython/issues/492 is fixed. + vcpath = call( + Path(os.environ["PROGRAMFILES(X86)"]) + / "Microsoft Visual Studio" + / "Installer" + / "vswhere.exe", + "-products", + "*", + "-latest", + "-property", + "installationPath", + capture_stdout=True, + ).strip() + log.notice(f"Discovering Visual Studio for GraalPy at {vcpath}") + vcvars_file = tmp / "vcvars.json" + call( + f"{vcpath}\\Common7\\Tools\\vsdevcmd.bat", + "-no_logo", + "-arch=amd64", + "-host_arch=amd64", + "&&", + "python", + "-c", + # this command needs to be one line for Windows reasons + "import sys, json, pathlib, os; pathlib.Path(sys.argv[1]).write_text(json.dumps(dict(os.environ)))", + vcvars_file, + env=env, + ) + with open(vcvars_file, encoding="utf-8") as f: + vcvars = json.load(f) + env.update(vcvars) + + return base_python, env + + +def build(options: Options, tmp_path: Path) -> None: + python_configurations = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) + + if not python_configurations: + return + + try: + before_all_options_identifier = python_configurations[0].identifier + before_all_options = options.build_options(before_all_options_identifier) + + if before_all_options.before_all: + log.step("Running before_all...") + env = before_all_options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command( + before_all_options.before_all, project=".", package=options.globals.package_dir + ) + shell(before_all_prepared, env=env) + + built_wheels: list[Path] = [] + + for config in python_configurations: + build_options = options.build_options(config.identifier) + build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + + use_uv = build_frontend.name == "build[uv]" and can_use_uv(config) + log.build_start(config.identifier) + + identifier_tmp_dir = tmp_path / config.identifier + identifier_tmp_dir.mkdir() + built_wheel_dir = identifier_tmp_dir / "built_wheel" + repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" + + constraints_path = build_options.dependency_constraints.get_for_python_version( + version=config.version, + tmp_dir=identifier_tmp_dir, + ) + + # install Python + base_python, env = setup_python( + identifier_tmp_dir / "build", + config, + constraints_path, + build_options.environment, + build_frontend.name, + ) + pip_version = None if use_uv else get_pip_version(env) + + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + log.step_end() + print( + f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." + ) + repaired_wheel = compatible_wheel + else: + # run the before_build command + if build_options.before_build: + log.step("Running before_build...") + before_build_prepared = prepare_command( + build_options.before_build, + project=".", + package=options.globals.package_dir, + ) + shell(before_build_prepared, env=env) + + log.step("Building wheel...") + built_wheel_dir.mkdir() + + extra_flags = get_build_frontend_extra_flags( + build_frontend, build_options.build_verbosity, build_options.config_settings + ) + + if ( + config.identifier.startswith("gp") + and build_frontend.name == "build" + and "--no-isolation" not in extra_flags + and "-n" not in extra_flags + ): + # GraalPy fails to discover its standard library when a venv is created + # from a virtualenv seeded executable. See + # https://github.com/oracle/graalpython/issues/491 and remove this once + # fixed upstream. + log.notice( + "Disabling build isolation to workaround GraalPy bug. If the build fails, consider using pip or build[uv] as build frontend." + ) + shell("graalpy -m pip install setuptools wheel", env=env) + extra_flags = [*extra_flags, "-n"] + + build_env = env.copy() + if pip_version is not None: + build_env["VIRTUALENV_PIP"] = pip_version + + if constraints_path: + combine_constraints(build_env, constraints_path, identifier_tmp_dir) + + if build_frontend.name == "pip": + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/pypa/cibuildwheel/pull/369 + call( + "python", + "-m", + "pip", + "wheel", + options.globals.package_dir.resolve(), + f"--wheel-dir={built_wheel_dir}", + "--no-deps", + *extra_flags, + env=build_env, + ) + elif build_frontend.name == "build" or build_frontend.name == "build[uv]": + if use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: + extra_flags.append("--installer=uv") + + call( + "python", + "-m", + "build", + build_options.package_dir, + "--wheel", + f"--outdir={built_wheel_dir}", + *extra_flags, + env=build_env, + ) + else: + assert_never(build_frontend) + + built_wheel = next(built_wheel_dir.glob("*.whl")) + + # repair the wheel + repaired_wheel_dir.mkdir() + + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + + if build_options.repair_command: + log.step("Repairing wheel...") + repair_command_prepared = prepare_command( + build_options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, + ) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) + + try: + repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) + except StopIteration: + raise errors.RepairStepProducedNoWheelError() from None + + if repaired_wheel.name in {wheel.name for wheel in built_wheels}: + raise errors.AlreadyBuiltWheelError(repaired_wheel.name) + + test_selected = options.globals.test_selector(config.identifier) + if test_selected and config.arch == "ARM64" != platform_module.machine(): + log.warning( + unwrap( + """ + While arm64 wheels can be built on other platforms, they cannot + be tested. An arm64 runner is required. To silence this warning, + set `CIBW_TEST_SKIP: "*-win_arm64"`. + """ + ) + ) + # skip this test + elif test_selected and build_options.test_command: + log.step("Testing wheel...") + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + venv_dir = identifier_tmp_dir / "venv-test" + virtualenv_env = virtualenv( + config.version, + base_python, + venv_dir, + None, + use_uv=use_uv, + env=env, + pip_version=pip_version, + ) + + virtualenv_env = build_options.test_environment.as_dictionary( + prev_environment=virtualenv_env + ) + + # check that we are using the Python from the virtual environment + call("where", "python", env=virtualenv_env) + + if build_options.before_test: + before_test_prepared = prepare_command( + build_options.before_test, + project=".", + package=build_options.package_dir, + ) + shell(before_test_prepared, env=virtualenv_env) + + pip = ["uv", "pip"] if use_uv else ["pip"] + + # install the wheel + call( + *pip, + "install", + str(repaired_wheel) + build_options.test_extras, + env=virtualenv_env, + ) + + # test the wheel + if build_options.test_requires: + call(*pip, "install", *build_options.test_requires, env=virtualenv_env) + + # run the tests from a temp dir, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_cwd = identifier_tmp_dir / "test_cwd" + test_cwd.mkdir() + + if build_options.test_sources: + copy_test_sources( + build_options.test_sources, + Path.cwd(), + test_cwd, + ) + else: + # Use the test_fail.py file to raise a nice error if the user + # tries to run tests in the cwd + (test_cwd / "test_fail.py").write_text(resources.TEST_FAIL_CWD_FILE.read_text()) + + test_command_prepared = prepare_command( + build_options.test_command, + project=Path.cwd(), + package=options.globals.package_dir.resolve(), + wheel=repaired_wheel, + ) + shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env) + + # we're all done here; move it to output (remove if already exists) + if compatible_wheel is None: + output_wheel = build_options.output_dir.joinpath(repaired_wheel.name) + moved_wheel = move_file(repaired_wheel, output_wheel) + if moved_wheel != output_wheel.resolve(): + log.warning( + f"{repaired_wheel} was moved to {moved_wheel} instead of {output_wheel}" + ) + built_wheels.append(output_wheel) + + # clean up + # (we ignore errors because occasionally Windows fails to unlink a file and we + # don't want to abort a build because of that) + shutil.rmtree(identifier_tmp_dir, ignore_errors=True) + + log.build_end() + except subprocess.CalledProcessError as error: + msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" + raise errors.FatalError(msg) from error diff --git a/cibuildwheel/projectfiles.py b/cibuildwheel/projectfiles.py index fece392f8..3fcebe535 100644 --- a/cibuildwheel/projectfiles.py +++ b/cibuildwheel/projectfiles.py @@ -1,78 +1,127 @@ import ast -import sys -from configparser import ConfigParser +import configparser +import contextlib from pathlib import Path -from typing import Any, Optional +from typing import Any -import tomli +import dependency_groups -if sys.version_info < (3, 8): - Constant = ast.Str - def get_constant(x: ast.Str) -> str: - return x.s +def get_parent(node: ast.AST | None, depth: int = 1) -> ast.AST | None: + for _ in range(depth): + node = getattr(node, "parent", None) + return node -else: - Constant = ast.Constant - def get_constant(x: ast.Constant) -> Any: - return x.value +def is_main(parent: ast.AST | None) -> bool: + if parent is None: + return False + + # This would be much nicer with 3.10's pattern matching! + if not isinstance(parent, ast.If): + return False + if not isinstance(parent.test, ast.Compare): + return False + + try: + (op,) = parent.test.ops + (comp,) = parent.test.comparators + except ValueError: + return False + + if not isinstance(op, ast.Eq): + return False + + values = {comp, parent.test.left} + + mains = {x for x in values if isinstance(x, ast.Constant) and x.value == "__main__"} + if len(mains) != 1: + return False + consts = {x for x in values if isinstance(x, ast.Name) and x.id == "__name__"} + + return len(consts) == 1 class Analyzer(ast.NodeVisitor): def __init__(self) -> None: - self.requires_python: Optional[str] = None + self.requires_python: str | None = None - def visit(self, content: ast.AST) -> None: - for node in ast.walk(content): - for child in ast.iter_child_nodes(node): - child.parent = node # type: ignore[attr-defined] - super().visit(content) + def visit(self, node: ast.AST) -> None: + for inner_node in ast.walk(node): + for child in ast.iter_child_nodes(inner_node): + child.parent = inner_node # type: ignore[attr-defined] + super().visit(node) def visit_keyword(self, node: ast.keyword) -> None: + # Must not be nested except for if __name__ == "__main__" + self.generic_visit(node) - if node.arg == "python_requires": - # Must not be nested in an if or other structure - # This will be Module -> Expr -> Call -> keyword - if not hasattr(node.parent.parent.parent, "parent") and isinstance( # type: ignore[attr-defined] - node.value, Constant - ): - self.requires_python = get_constant(node.value) + # This will be Module -> Expr -> Call -> keyword + parent = get_parent(node, 4) + unnested = parent is None + + # This will be Module -> If -> Expr -> Call -> keyword + name_main_unnested = ( + parent is not None and get_parent(parent) is None and is_main(get_parent(node, 3)) + ) + + if ( + node.arg == "python_requires" + and isinstance(node.value, ast.Constant) + and (unnested or name_main_unnested) + ): + self.requires_python = node.value.value -def setup_py_python_requires(content: str) -> Optional[str]: +def setup_py_python_requires(content: str) -> str | None: try: tree = ast.parse(content) analyzer = Analyzer() analyzer.visit(tree) return analyzer.requires_python or None - except Exception: + except Exception: # pylint: disable=broad-except return None -def get_requires_python_str(package_dir: Path) -> Optional[str]: +def get_requires_python_str(package_dir: Path, pyproject_toml: dict[str, Any] | None) -> str | None: """Return the python requires string from the most canonical source available, or None""" # Read in from pyproject.toml:project.requires-python - try: - with (package_dir / "pyproject.toml").open("rb") as f1: - info = tomli.load(f1) - return str(info["project"]["requires-python"]) - except (FileNotFoundError, KeyError, IndexError, TypeError): - pass + with contextlib.suppress(KeyError, IndexError, TypeError): + return str((pyproject_toml or {})["project"]["requires-python"]) # Read in from setup.cfg:options.python_requires - try: - config = ConfigParser() + config = configparser.ConfigParser() + with contextlib.suppress(FileNotFoundError): config.read(package_dir / "setup.cfg") - return str(config["options"]["python_requires"]) - except (FileNotFoundError, KeyError, IndexError, TypeError): - pass + with contextlib.suppress(KeyError, IndexError, TypeError): + return str(config["options"]["python_requires"]) - try: - with (package_dir / "setup.py").open(encoding="utf8") as f2: - return setup_py_python_requires(f2.read()) - except FileNotFoundError: - pass + setup_py = package_dir / "setup.py" + with contextlib.suppress(FileNotFoundError), setup_py.open(encoding="utf8") as f2: + return setup_py_python_requires(f2.read()) return None + + +def resolve_dependency_groups( + pyproject_toml: dict[str, Any] | None, *groups: str +) -> tuple[str, ...]: + """ + Get the packages in dependency-groups for a package. + """ + + if not groups: + return () + + if pyproject_toml is None: + msg = f"Didn't find a pyproject.toml, so can't read [dependency-groups] {groups!r} from it!" + raise FileNotFoundError(msg) + + try: + dependency_groups_toml = pyproject_toml["dependency-groups"] + except KeyError: + msg = f"Didn't find [dependency-groups] in pyproject.toml, which is needed to resolve {groups!r}." + raise KeyError(msg) from None + + return dependency_groups.resolve(dependency_groups_toml, *groups) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 00c633819..1b28c4dcd 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -1,94 +1,232 @@ [linux] python_configurations = [ - { identifier = "cp36-manylinux_x86_64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-manylinux_x86_64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, - { identifier = "cp36-manylinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-manylinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp311-manylinux_x86_64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-manylinux_x86_64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-manylinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-manylinux_x86_64", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-manylinux_x86_64", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, { identifier = "cp38-manylinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_i686", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-manylinux_i686", version = "3.10", path_str = "/opt/python/cp310-cp310" }, - { identifier = "pp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, + { identifier = "cp311-manylinux_i686", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-manylinux_i686", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-manylinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-manylinux_i686", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-manylinux_i686", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, { identifier = "pp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, - { identifier = "cp36-manylinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "pp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, + { identifier = "pp310-manylinux_x86_64", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" }, + { identifier = "pp311-manylinux_x86_64", version = "3.11", path_str = "/opt/python/pp311-pypy311_pp73" }, + { identifier = "gp311_242-manylinux_x86_64", version = "3.11", path_str = "/opt/python/graalpy311-graalpy242_311_native" }, { identifier = "cp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-manylinux_aarch64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, - { identifier = "cp36-manylinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-manylinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp311-manylinux_aarch64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-manylinux_aarch64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-manylinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-manylinux_aarch64", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-manylinux_aarch64", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, { identifier = "cp38-manylinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_ppc64le", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-manylinux_ppc64le", version = "3.10", path_str = "/opt/python/cp310-cp310" }, - { identifier = "cp36-manylinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-manylinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp311-manylinux_ppc64le", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-manylinux_ppc64le", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-manylinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-manylinux_ppc64le", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-manylinux_ppc64le", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, { identifier = "cp38-manylinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_s390x", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-manylinux_s390x", version = "3.10", path_str = "/opt/python/cp310-cp310" }, - { identifier = "pp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, + { identifier = "cp311-manylinux_s390x", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-manylinux_s390x", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-manylinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-manylinux_s390x", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-manylinux_s390x", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, + { identifier = "cp38-manylinux_armv7l", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-manylinux_armv7l", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-manylinux_armv7l", version = "3.10", path_str = "/opt/python/cp310-cp310" }, + { identifier = "cp311-manylinux_armv7l", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-manylinux_armv7l", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-manylinux_armv7l", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_armv7l", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-manylinux_armv7l", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-manylinux_armv7l", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, + { identifier = "cp38-manylinux_riscv64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-manylinux_riscv64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-manylinux_riscv64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, + { identifier = "cp311-manylinux_riscv64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-manylinux_riscv64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-manylinux_riscv64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_riscv64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-manylinux_riscv64", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-manylinux_riscv64", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, { identifier = "pp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, - { identifier = "pp37-manylinux_i686", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, + { identifier = "pp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, + { identifier = "pp310-manylinux_aarch64", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" }, + { identifier = "pp311-manylinux_aarch64", version = "3.11", path_str = "/opt/python/pp311-pypy311_pp73" }, + { identifier = "gp311_242-manylinux_aarch64", version = "3.11", path_str = "/opt/python/graalpy311-graalpy242_311_native" }, { identifier = "pp38-manylinux_i686", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, - { identifier = "cp36-musllinux_x86_64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-musllinux_x86_64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "pp39-manylinux_i686", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, + { identifier = "pp310-manylinux_i686", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" }, + { identifier = "pp311-manylinux_i686", version = "3.11", path_str = "/opt/python/pp311-pypy311_pp73" }, { identifier = "cp38-musllinux_x86_64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-musllinux_x86_64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-musllinux_x86_64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, - { identifier = "cp36-musllinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-musllinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp311-musllinux_x86_64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-musllinux_x86_64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-musllinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-musllinux_x86_64", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-musllinux_x86_64", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, { identifier = "cp38-musllinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-musllinux_i686", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-musllinux_i686", version = "3.10", path_str = "/opt/python/cp310-cp310" }, - { identifier = "cp36-musllinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-musllinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp311-musllinux_i686", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-musllinux_i686", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-musllinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-musllinux_i686", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-musllinux_i686", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, { identifier = "cp38-musllinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-musllinux_aarch64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-musllinux_aarch64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, - { identifier = "cp36-musllinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-musllinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp311-musllinux_aarch64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-musllinux_aarch64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-musllinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-musllinux_aarch64", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-musllinux_aarch64", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, { identifier = "cp38-musllinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-musllinux_ppc64le", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-musllinux_ppc64le", version = "3.10", path_str = "/opt/python/cp310-cp310" }, - { identifier = "cp36-musllinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, - { identifier = "cp37-musllinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, + { identifier = "cp311-musllinux_ppc64le", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-musllinux_ppc64le", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-musllinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-musllinux_ppc64le", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-musllinux_ppc64le", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, { identifier = "cp38-musllinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-musllinux_s390x", version = "3.9", path_str = "/opt/python/cp39-cp39" }, { identifier = "cp310-musllinux_s390x", version = "3.10", path_str = "/opt/python/cp310-cp310" }, + { identifier = "cp311-musllinux_s390x", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-musllinux_s390x", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-musllinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-musllinux_s390x", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-musllinux_s390x", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, + { identifier = "cp38-musllinux_armv7l", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-musllinux_armv7l", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-musllinux_armv7l", version = "3.10", path_str = "/opt/python/cp310-cp310" }, + { identifier = "cp311-musllinux_armv7l", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-musllinux_armv7l", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-musllinux_armv7l", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_armv7l", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-musllinux_armv7l", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-musllinux_armv7l", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, + { identifier = "cp38-musllinux_riscv64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, + { identifier = "cp39-musllinux_ricv64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-musllinux_riscv64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, + { identifier = "cp311-musllinux_riscv64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, + { identifier = "cp312-musllinux_riscv64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, + { identifier = "cp313-musllinux_riscv64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_riscv64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, + { identifier = "cp314-musllinux_riscv64", version = "3.14", path_str = "/opt/python/cp314-cp314" }, + { identifier = "cp314t-musllinux_riscv64", version = "3.14", path_str = "/opt/python/cp314-cp314t" }, ] [macos] python_configurations = [ - { identifier = "cp36-macosx_x86_64", version = "3.6", url = "https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg" }, - { identifier = "cp37-macosx_x86_64", version = "3.7", url = "https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg" }, { identifier = "cp38-macosx_x86_64", version = "3.8", url = "https://www.python.org/ftp/python/3.8.10/python-3.8.10-macosx10.9.pkg" }, { identifier = "cp38-macosx_arm64", version = "3.8", url = "https://www.python.org/ftp/python/3.8.10/python-3.8.10-macosx10.9.pkg" }, { identifier = "cp38-macosx_universal2", version = "3.8", url = "https://www.python.org/ftp/python/3.8.10/python-3.8.10-macosx10.9.pkg" }, - { identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.10/python-3.9.10-macos11.pkg" }, - { identifier = "cp39-macosx_arm64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.10/python-3.9.10-macos11.pkg" }, - { identifier = "cp39-macosx_universal2", version = "3.9", url = "https://www.python.org/ftp/python/3.9.10/python-3.9.10-macos11.pkg" }, - { identifier = "cp310-macosx_x86_64", version = "3.10", url = "https://www.python.org/ftp/python/3.10.2/python-3.10.2-macos11.pkg" }, - { identifier = "cp310-macosx_arm64", version = "3.10", url = "https://www.python.org/ftp/python/3.10.2/python-3.10.2-macos11.pkg" }, - { identifier = "cp310-macosx_universal2", version = "3.10", url = "https://www.python.org/ftp/python/3.10.2/python-3.10.2-macos11.pkg" }, - { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.7-osx64.tar.bz2" }, - { identifier = "pp38-macosx_x86_64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.7-osx64.tar.bz2" }, + { identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.13/python-3.9.13-macos11.pkg" }, + { identifier = "cp39-macosx_arm64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.13/python-3.9.13-macos11.pkg" }, + { identifier = "cp39-macosx_universal2", version = "3.9", url = "https://www.python.org/ftp/python/3.9.13/python-3.9.13-macos11.pkg" }, + { identifier = "cp310-macosx_x86_64", version = "3.10", url = "https://www.python.org/ftp/python/3.10.11/python-3.10.11-macos11.pkg" }, + { identifier = "cp310-macosx_arm64", version = "3.10", url = "https://www.python.org/ftp/python/3.10.11/python-3.10.11-macos11.pkg" }, + { identifier = "cp310-macosx_universal2", version = "3.10", url = "https://www.python.org/ftp/python/3.10.11/python-3.10.11-macos11.pkg" }, + { identifier = "cp311-macosx_x86_64", version = "3.11", url = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-macos11.pkg" }, + { identifier = "cp311-macosx_arm64", version = "3.11", url = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-macos11.pkg" }, + { identifier = "cp311-macosx_universal2", version = "3.11", url = "https://www.python.org/ftp/python/3.11.9/python-3.11.9-macos11.pkg" }, + { identifier = "cp312-macosx_x86_64", version = "3.12", url = "https://www.python.org/ftp/python/3.12.10/python-3.12.10-macos11.pkg" }, + { identifier = "cp312-macosx_arm64", version = "3.12", url = "https://www.python.org/ftp/python/3.12.10/python-3.12.10-macos11.pkg" }, + { identifier = "cp312-macosx_universal2", version = "3.12", url = "https://www.python.org/ftp/python/3.12.10/python-3.12.10-macos11.pkg" }, + { identifier = "cp313-macosx_x86_64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.4/python-3.13.4-macos11.pkg" }, + { identifier = "cp313-macosx_arm64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.4/python-3.13.4-macos11.pkg" }, + { identifier = "cp313-macosx_universal2", version = "3.13", url = "https://www.python.org/ftp/python/3.13.4/python-3.13.4-macos11.pkg" }, + { identifier = "cp313t-macosx_x86_64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.4/python-3.13.4-macos11.pkg" }, + { identifier = "cp313t-macosx_arm64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.4/python-3.13.4-macos11.pkg" }, + { identifier = "cp313t-macosx_universal2", version = "3.13", url = "https://www.python.org/ftp/python/3.13.4/python-3.13.4-macos11.pkg" }, + { identifier = "cp314-macosx_x86_64", version = "3.14", url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0b2-macos11.pkg" }, + { identifier = "cp314-macosx_arm64", version = "3.14", url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0b2-macos11.pkg" }, + { identifier = "cp314-macosx_universal2", version = "3.14", url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0b2-macos11.pkg" }, + { identifier = "cp314t-macosx_x86_64", version = "3.14", url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0b2-macos11.pkg" }, + { identifier = "cp314t-macosx_arm64", version = "3.14", url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0b2-macos11.pkg" }, + { identifier = "cp314t-macosx_universal2", version = "3.14", url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0b2-macos11.pkg" }, + { identifier = "pp38-macosx_x86_64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_x86_64.tar.bz2" }, + { identifier = "pp38-macosx_arm64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_arm64.tar.bz2" }, + { identifier = "pp39-macosx_x86_64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-macos_x86_64.tar.bz2" }, + { identifier = "pp39-macosx_arm64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-macos_arm64.tar.bz2" }, + { identifier = "pp310-macosx_x86_64", version = "3.10", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.19-macos_x86_64.tar.bz2" }, + { identifier = "pp310-macosx_arm64", version = "3.10", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.19-macos_arm64.tar.bz2" }, + { identifier = "pp311-macosx_x86_64", version = "3.11", url = "https://downloads.python.org/pypy/pypy3.11-v7.3.19-macos_x86_64.tar.bz2" }, + { identifier = "pp311-macosx_arm64", version = "3.11", url = "https://downloads.python.org/pypy/pypy3.11-v7.3.19-macos_arm64.tar.bz2" }, + { identifier = "gp311_242-macosx_x86_64", version = "3.11", url = "https://github.com/oracle/graalpython/releases/download/graal-24.2.1/graalpy-24.2.1-macos-amd64.tar.gz" }, + { identifier = "gp311_242-macosx_arm64", version = "3.11", url = "https://github.com/oracle/graalpython/releases/download/graal-24.2.1/graalpy-24.2.1-macos-aarch64.tar.gz" }, ] [windows] python_configurations = [ - { identifier = "cp36-win32", version = "3.6.8", arch = "32" }, - { identifier = "cp36-win_amd64", version = "3.6.8", arch = "64" }, - { identifier = "cp37-win32", version = "3.7.9", arch = "32" }, - { identifier = "cp37-win_amd64", version = "3.7.9", arch = "64" }, { identifier = "cp38-win32", version = "3.8.10", arch = "32" }, { identifier = "cp38-win_amd64", version = "3.8.10", arch = "64" }, - { identifier = "cp39-win32", version = "3.9.10", arch = "32" }, - { identifier = "cp39-win_amd64", version = "3.9.10", arch = "64" }, - { identifier = "cp310-win32", version = "3.10.2", arch = "32" }, - { identifier = "cp310-win_amd64", version = "3.10.2", arch = "64" }, + { identifier = "cp39-win32", version = "3.9.13", arch = "32" }, + { identifier = "cp39-win_amd64", version = "3.9.13", arch = "64" }, + { identifier = "cp310-win32", version = "3.10.11", arch = "32" }, + { identifier = "cp310-win_amd64", version = "3.10.11", arch = "64" }, + { identifier = "cp311-win32", version = "3.11.9", arch = "32" }, + { identifier = "cp311-win_amd64", version = "3.11.9", arch = "64" }, + { identifier = "cp312-win32", version = "3.12.10", arch = "32" }, + { identifier = "cp312-win_amd64", version = "3.12.10", arch = "64" }, + { identifier = "cp313-win32", version = "3.13.4", arch = "32" }, + { identifier = "cp313t-win32", version = "3.13.4", arch = "32" }, + { identifier = "cp313-win_amd64", version = "3.13.4", arch = "64" }, + { identifier = "cp313t-win_amd64", version = "3.13.4", arch = "64" }, + { identifier = "cp314-win32", version = "3.14.0-b2", arch = "32" }, + { identifier = "cp314t-win32", version = "3.14.0-b2", arch = "32" }, + { identifier = "cp314-win_amd64", version = "3.14.0-b2", arch = "64" }, + { identifier = "cp314t-win_amd64", version = "3.14.0-b2", arch = "64" }, { identifier = "cp39-win_arm64", version = "3.9.10", arch = "ARM64" }, - { identifier = "cp310-win_arm64", version = "3.10.2", arch = "ARM64" }, - { identifier = "pp37-win_amd64", version = "3.7", arch = "64", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.7-win64.zip" }, - { identifier = "pp38-win_amd64", version = "3.8", arch = "64", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.7-win64.zip" }, + { identifier = "cp310-win_arm64", version = "3.10.11", arch = "ARM64" }, + { identifier = "cp311-win_arm64", version = "3.11.9", arch = "ARM64" }, + { identifier = "cp312-win_arm64", version = "3.12.10", arch = "ARM64" }, + { identifier = "cp313-win_arm64", version = "3.13.4", arch = "ARM64" }, + { identifier = "cp313t-win_arm64", version = "3.13.4", arch = "ARM64" }, + { identifier = "cp314-win_arm64", version = "3.14.0-b2", arch = "ARM64" }, + { identifier = "cp314t-win_arm64", version = "3.14.0-b2", arch = "ARM64" }, + { identifier = "pp38-win_amd64", version = "3.8", arch = "64", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-win64.zip" }, + { identifier = "pp39-win_amd64", version = "3.9", arch = "64", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-win64.zip" }, + { identifier = "pp310-win_amd64", version = "3.10", arch = "64", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.19-win64.zip" }, + { identifier = "pp311-win_amd64", version = "3.11", arch = "64", url = "https://downloads.python.org/pypy/pypy3.11-v7.3.19-win64.zip" }, + { identifier = "gp311_242-win_amd64", version = "3.11", arch = "64", url = "https://github.com/oracle/graalpython/releases/download/graal-24.2.1/graalpy-24.2.1-windows-amd64.zip" }, +] + +[pyodide] +python_configurations = [ + { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.7", node_version = "v22" }, + { identifier = "cp313-pyodide_wasm32", version = "3.13", default_pyodide_version = "0.28.0a3", node_version = "v22" }, +] + +[ios] +python_configurations = [ + { identifier = "cp313-ios_arm64_iphoneos", version = "3.13", url = "https://github.com/beeware/Python-Apple-support/releases/download/3.13-b9/Python-3.13-iOS-support.b9.tar.gz" }, + { identifier = "cp313-ios_x86_64_iphonesimulator", version = "3.13", url = "https://github.com/beeware/Python-Apple-support/releases/download/3.13-b9/Python-3.13-iOS-support.b9.tar.gz" }, + { identifier = "cp313-ios_arm64_iphonesimulator", version = "3.13", url = "https://github.com/beeware/Python-Apple-support/releases/download/3.13-b9/Python-3.13-iOS-support.b9.tar.gz" }, ] diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json new file mode 100644 index 000000000..7d6aaf10d --- /dev/null +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -0,0 +1,1136 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/resources/cibuildwheel.schema.json", + "$defs": { + "inherit": { + "enum": [ + "none", + "prepend", + "append" + ], + "default": "none", + "description": "How to inherit the parent's value." + }, + "enable": { + "enum": [ + "cpython-experimental-riscv64", + "cpython-freethreading", + "cpython-prerelease", + "graalpy", + "pyodide-prerelease", + "pypy", + "pypy-eol" + ] + }, + "description": "A Python version or flavor to enable." + }, + "additionalProperties": false, + "description": "cibuildwheel's settings.", + "type": "object", + "properties": { + "archs": { + "description": "Change the architectures built on your machine by default.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_ARCHS" + }, + "before-all": { + "description": "Execute a shell command on the build system before any wheels are built.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_BEFORE_ALL" + }, + "before-build": { + "description": "Execute a shell command preparing each wheel's build.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_BEFORE_BUILD" + }, + "before-test": { + "description": "Execute a shell command before testing each wheel.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_BEFORE_TEST" + }, + "build": { + "default": [ + "*" + ], + "description": "Choose the Python versions to build.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_BUILD" + }, + "build-frontend": { + "default": "default", + "description": "Set the tool to use to build, either \"pip\" (default for now), \"build\", or \"build[uv]\"", + "oneOf": [ + { + "enum": [ + "pip", + "build", + "build[uv]", + "default" + ] + }, + { + "type": "string", + "pattern": "^pip; ?args:" + }, + { + "type": "string", + "pattern": "^build; ?args:" + }, + { + "type": "string", + "pattern": "^build\\[uv\\]; ?args:" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "enum": [ + "pip", + "build", + "build[uv]" + ] + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + ], + "title": "CIBW_BUILD_FRONTEND" + }, + "build-verbosity": { + "type": "integer", + "minimum": -3, + "maximum": 3, + "default": 0, + "description": "Increase/decrease the output of pip wheel.", + "title": "CIBW_BUILD_VERBOSITY" + }, + "config-settings": { + "description": "Specify config-settings for the build backend.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".+": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + } + } + ], + "title": "CIBW_CONFIG_SETTINGS" + }, + "container-engine": { + "oneOf": [ + { + "enum": [ + "docker", + "podman" + ] + }, + { + "type": "string", + "pattern": "^docker; ?(create_args|disable_host_mount):" + }, + { + "type": "string", + "pattern": "^podman; ?(create_args|disable_host_mount):" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "name" + ], + "properties": { + "name": { + "enum": [ + "docker", + "podman" + ] + }, + "create-args": { + "type": "array", + "items": { + "type": "string" + } + }, + "disable-host-mount": { + "type": "boolean" + } + } + } + ], + "title": "CIBW_CONTAINER_ENGINE" + }, + "dependency-versions": { + "default": "pinned", + "description": "Specify how cibuildwheel controls the versions of the tools it uses", + "oneOf": [ + { + "enum": [ + "pinned", + "latest" + ] + }, + { + "type": "string", + "description": "Path to a file containing dependency versions, or inline package specifications, starting with \"packages:\"", + "not": { + "enum": [ + "pinned", + "latest" + ] + } + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "file": { + "type": "string" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "packages": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + ], + "title": "CIBW_DEPENDENCY_VERSIONS" + }, + "enable": { + "description": "Enable or disable certain builds.", + "oneOf": [ + { + "$ref": "#/$defs/enable" + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/enable" + } + } + ], + "title": "CIBW_ENABLE" + }, + "environment": { + "description": "Set environment variables needed during the build.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".+": { + "type": "string" + } + } + } + ], + "title": "CIBW_ENVIRONMENT" + }, + "environment-pass": { + "description": "Set environment variables on the host to pass-through to the container during the build.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_ENVIRONMENT_PASS" + }, + "manylinux-aarch64-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_AARCH64_IMAGE" + }, + "manylinux-armv7l-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_ARMV7L_IMAGE" + }, + "manylinux-i686-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_I686_IMAGE" + }, + "manylinux-ppc64le-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_PPC64LE_IMAGE" + }, + "manylinux-pypy_aarch64-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_PYPY_AARCH64_IMAGE" + }, + "manylinux-pypy_i686-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_PYPY_I686_IMAGE" + }, + "manylinux-pypy_x86_64-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_PYPY_X86_64_IMAGE" + }, + "manylinux-riscv64-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_RISCV64_IMAGE" + }, + "manylinux-s390x-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_S390X_IMAGE" + }, + "manylinux-x86_64-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MANYLINUX_X86_64_IMAGE" + }, + "musllinux-aarch64-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MUSLLINUX_AARCH64_IMAGE" + }, + "musllinux-armv7l-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MUSLLINUX_ARMV7L_IMAGE" + }, + "musllinux-i686-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MUSLLINUX_I686_IMAGE" + }, + "musllinux-ppc64le-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MUSLLINUX_PPC64LE_IMAGE" + }, + "musllinux-riscv64-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MUSLLINUX_RISCV64_IMAGE" + }, + "musllinux-s390x-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MUSLLINUX_S390X_IMAGE" + }, + "musllinux-x86_64-image": { + "type": "string", + "description": "Specify alternative manylinux / musllinux container images", + "title": "CIBW_MUSLLINUX_X86_64_IMAGE" + }, + "xbuild-tools": { + "description": "Binaries on the path that should be included in an isolated cross-build environment", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_XBUILD_TOOLS" + }, + "pyodide-version": { + "type": "string", + "description": "Specify the version of Pyodide to use", + "title": "CIBW_PYODIDE_VERSION" + }, + "repair-wheel-command": { + "description": "Execute a shell command to repair each built wheel.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_REPAIR_WHEEL_COMMAND" + }, + "skip": { + "description": "Choose the Python versions to skip.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_SKIP" + }, + "test-command": { + "description": "Execute a shell command to test each built wheel.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_TEST_COMMAND" + }, + "test-extras": { + "description": "Install your wheel for testing using `extras_require`", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_TEST_EXTRAS" + }, + "test-sources": { + "description": "Test files that are required by the test environment", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_TEST_SOURCES" + }, + "test-groups": { + "description": "Install extra groups when testing", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_TEST_GROUPS" + }, + "test-requires": { + "description": "Install Python dependencies before running the tests", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_TEST_REQUIRES" + }, + "test-skip": { + "description": "Skip running tests on some builds.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_TEST_SKIP" + }, + "test-environment": { + "description": "Set environment variables for the test environment", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".+": { + "type": "string" + } + } + } + ], + "title": "CIBW_TEST_ENVIRONMENT" + }, + "overrides": { + "type": "array", + "description": "An overrides array", + "items": { + "type": "object", + "required": [ + "select" + ], + "minProperties": 2, + "additionalProperties": false, + "properties": { + "select": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "inherit": { + "type": "object", + "additionalProperties": false, + "properties": { + "before-all": { + "$ref": "#/$defs/inherit" + }, + "before-build": { + "$ref": "#/$defs/inherit" + }, + "xbuild-tools": { + "$ref": "#/$defs/inherit" + }, + "before-test": { + "$ref": "#/$defs/inherit" + }, + "config-settings": { + "$ref": "#/$defs/inherit" + }, + "container-engine": { + "$ref": "#/$defs/inherit" + }, + "environment": { + "$ref": "#/$defs/inherit" + }, + "environment-pass": { + "$ref": "#/$defs/inherit" + }, + "repair-wheel-command": { + "$ref": "#/$defs/inherit" + }, + "test-command": { + "$ref": "#/$defs/inherit" + }, + "test-extras": { + "$ref": "#/$defs/inherit" + }, + "test-sources": { + "$ref": "#/$defs/inherit" + }, + "test-requires": { + "$ref": "#/$defs/inherit" + }, + "test-environment": { + "$ref": "#/$defs/inherit" + } + } + }, + "before-all": { + "$ref": "#/properties/before-all" + }, + "before-build": { + "$ref": "#/properties/before-build" + }, + "before-test": { + "$ref": "#/properties/before-test" + }, + "build-frontend": { + "$ref": "#/properties/build-frontend" + }, + "build-verbosity": { + "$ref": "#/properties/build-verbosity" + }, + "config-settings": { + "$ref": "#/properties/config-settings" + }, + "container-engine": { + "$ref": "#/properties/container-engine" + }, + "dependency-versions": { + "$ref": "#/properties/dependency-versions" + }, + "environment": { + "$ref": "#/properties/environment" + }, + "environment-pass": { + "$ref": "#/properties/environment-pass" + }, + "manylinux-aarch64-image": { + "$ref": "#/properties/manylinux-aarch64-image" + }, + "manylinux-armv7l-image": { + "$ref": "#/properties/manylinux-armv7l-image" + }, + "manylinux-i686-image": { + "$ref": "#/properties/manylinux-i686-image" + }, + "manylinux-ppc64le-image": { + "$ref": "#/properties/manylinux-ppc64le-image" + }, + "manylinux-pypy_aarch64-image": { + "$ref": "#/properties/manylinux-pypy_aarch64-image" + }, + "manylinux-pypy_i686-image": { + "$ref": "#/properties/manylinux-pypy_i686-image" + }, + "manylinux-pypy_x86_64-image": { + "$ref": "#/properties/manylinux-pypy_x86_64-image" + }, + "manylinux-riscv64-image": { + "$ref": "#/properties/manylinux-riscv64-image" + }, + "manylinux-s390x-image": { + "$ref": "#/properties/manylinux-s390x-image" + }, + "manylinux-x86_64-image": { + "$ref": "#/properties/manylinux-x86_64-image" + }, + "musllinux-aarch64-image": { + "$ref": "#/properties/musllinux-aarch64-image" + }, + "musllinux-armv7l-image": { + "$ref": "#/properties/musllinux-armv7l-image" + }, + "musllinux-i686-image": { + "$ref": "#/properties/musllinux-i686-image" + }, + "musllinux-ppc64le-image": { + "$ref": "#/properties/musllinux-ppc64le-image" + }, + "musllinux-riscv64-image": { + "$ref": "#/properties/musllinux-riscv64-image" + }, + "musllinux-s390x-image": { + "$ref": "#/properties/musllinux-s390x-image" + }, + "musllinux-x86_64-image": { + "$ref": "#/properties/musllinux-x86_64-image" + }, + "xbuild-tools": { + "$ref": "#/properties/xbuild-tools" + }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, + "repair-wheel-command": { + "$ref": "#/properties/repair-wheel-command" + }, + "test-command": { + "$ref": "#/properties/test-command" + }, + "test-extras": { + "$ref": "#/properties/test-extras" + }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, + "test-requires": { + "$ref": "#/properties/test-requires" + }, + "test-environment": { + "$ref": "#/properties/test-environment" + } + } + } + }, + "linux": { + "type": "object", + "additionalProperties": false, + "properties": { + "archs": { + "$ref": "#/properties/archs" + }, + "before-all": { + "$ref": "#/properties/before-all" + }, + "before-build": { + "$ref": "#/properties/before-build" + }, + "before-test": { + "$ref": "#/properties/before-test" + }, + "build-frontend": { + "$ref": "#/properties/build-frontend" + }, + "build-verbosity": { + "$ref": "#/properties/build-verbosity" + }, + "config-settings": { + "$ref": "#/properties/config-settings" + }, + "container-engine": { + "$ref": "#/properties/container-engine" + }, + "environment": { + "$ref": "#/properties/environment" + }, + "environment-pass": { + "$ref": "#/properties/environment-pass" + }, + "manylinux-aarch64-image": { + "$ref": "#/properties/manylinux-aarch64-image" + }, + "manylinux-armv7l-image": { + "$ref": "#/properties/manylinux-armv7l-image" + }, + "manylinux-i686-image": { + "$ref": "#/properties/manylinux-i686-image" + }, + "manylinux-ppc64le-image": { + "$ref": "#/properties/manylinux-ppc64le-image" + }, + "manylinux-pypy_aarch64-image": { + "$ref": "#/properties/manylinux-pypy_aarch64-image" + }, + "manylinux-pypy_i686-image": { + "$ref": "#/properties/manylinux-pypy_i686-image" + }, + "manylinux-pypy_x86_64-image": { + "$ref": "#/properties/manylinux-pypy_x86_64-image" + }, + "manylinux-riscv64-image": { + "$ref": "#/properties/manylinux-riscv64-image" + }, + "manylinux-s390x-image": { + "$ref": "#/properties/manylinux-s390x-image" + }, + "manylinux-x86_64-image": { + "$ref": "#/properties/manylinux-x86_64-image" + }, + "musllinux-aarch64-image": { + "$ref": "#/properties/musllinux-aarch64-image" + }, + "musllinux-armv7l-image": { + "$ref": "#/properties/musllinux-armv7l-image" + }, + "musllinux-i686-image": { + "$ref": "#/properties/musllinux-i686-image" + }, + "musllinux-ppc64le-image": { + "$ref": "#/properties/musllinux-ppc64le-image" + }, + "musllinux-riscv64-image": { + "$ref": "#/properties/musllinux-riscv64-image" + }, + "musllinux-s390x-image": { + "$ref": "#/properties/musllinux-s390x-image" + }, + "musllinux-x86_64-image": { + "$ref": "#/properties/musllinux-x86_64-image" + }, + "xbuild-tools": { + "$ref": "#/properties/xbuild-tools" + }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, + "repair-wheel-command": { + "description": "Execute a shell command to repair each built wheel.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_REPAIR_WHEEL_COMMAND", + "default": "auditwheel repair -w {dest_dir} {wheel}" + }, + "test-command": { + "$ref": "#/properties/test-command" + }, + "test-extras": { + "$ref": "#/properties/test-extras" + }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, + "test-requires": { + "$ref": "#/properties/test-requires" + }, + "test-environment": { + "$ref": "#/properties/test-environment" + } + } + }, + "windows": { + "type": "object", + "additionalProperties": false, + "properties": { + "archs": { + "$ref": "#/properties/archs" + }, + "before-all": { + "$ref": "#/properties/before-all" + }, + "before-build": { + "$ref": "#/properties/before-build" + }, + "before-test": { + "$ref": "#/properties/before-test" + }, + "build-frontend": { + "$ref": "#/properties/build-frontend" + }, + "build-verbosity": { + "$ref": "#/properties/build-verbosity" + }, + "config-settings": { + "$ref": "#/properties/config-settings" + }, + "dependency-versions": { + "$ref": "#/properties/dependency-versions" + }, + "environment": { + "$ref": "#/properties/environment" + }, + "xbuild-tools": { + "$ref": "#/properties/xbuild-tools" + }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, + "repair-wheel-command": { + "$ref": "#/properties/repair-wheel-command" + }, + "test-command": { + "$ref": "#/properties/test-command" + }, + "test-extras": { + "$ref": "#/properties/test-extras" + }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, + "test-requires": { + "$ref": "#/properties/test-requires" + }, + "test-environment": { + "$ref": "#/properties/test-environment" + } + } + }, + "macos": { + "type": "object", + "additionalProperties": false, + "properties": { + "archs": { + "$ref": "#/properties/archs" + }, + "before-all": { + "$ref": "#/properties/before-all" + }, + "before-build": { + "$ref": "#/properties/before-build" + }, + "before-test": { + "$ref": "#/properties/before-test" + }, + "build-frontend": { + "$ref": "#/properties/build-frontend" + }, + "build-verbosity": { + "$ref": "#/properties/build-verbosity" + }, + "config-settings": { + "$ref": "#/properties/config-settings" + }, + "dependency-versions": { + "$ref": "#/properties/dependency-versions" + }, + "environment": { + "$ref": "#/properties/environment" + }, + "xbuild-tools": { + "$ref": "#/properties/xbuild-tools" + }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, + "repair-wheel-command": { + "description": "Execute a shell command to repair each built wheel.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_REPAIR_WHEEL_COMMAND", + "default": "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" + }, + "test-command": { + "$ref": "#/properties/test-command" + }, + "test-extras": { + "$ref": "#/properties/test-extras" + }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, + "test-requires": { + "$ref": "#/properties/test-requires" + }, + "test-environment": { + "$ref": "#/properties/test-environment" + } + } + }, + "pyodide": { + "type": "object", + "additionalProperties": false, + "properties": { + "archs": { + "$ref": "#/properties/archs" + }, + "before-all": { + "$ref": "#/properties/before-all" + }, + "before-build": { + "$ref": "#/properties/before-build" + }, + "before-test": { + "$ref": "#/properties/before-test" + }, + "build-frontend": { + "$ref": "#/properties/build-frontend" + }, + "build-verbosity": { + "$ref": "#/properties/build-verbosity" + }, + "config-settings": { + "$ref": "#/properties/config-settings" + }, + "dependency-versions": { + "$ref": "#/properties/dependency-versions" + }, + "environment": { + "$ref": "#/properties/environment" + }, + "xbuild-tools": { + "$ref": "#/properties/xbuild-tools" + }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, + "repair-wheel-command": { + "$ref": "#/properties/repair-wheel-command" + }, + "test-command": { + "$ref": "#/properties/test-command" + }, + "test-extras": { + "$ref": "#/properties/test-extras" + }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, + "test-requires": { + "$ref": "#/properties/test-requires" + }, + "test-environment": { + "$ref": "#/properties/test-environment" + } + } + }, + "ios": { + "type": "object", + "additionalProperties": false, + "properties": { + "archs": { + "$ref": "#/properties/archs" + }, + "before-all": { + "$ref": "#/properties/before-all" + }, + "before-build": { + "$ref": "#/properties/before-build" + }, + "before-test": { + "$ref": "#/properties/before-test" + }, + "build-frontend": { + "$ref": "#/properties/build-frontend" + }, + "build-verbosity": { + "$ref": "#/properties/build-verbosity" + }, + "config-settings": { + "$ref": "#/properties/config-settings" + }, + "dependency-versions": { + "$ref": "#/properties/dependency-versions" + }, + "environment": { + "$ref": "#/properties/environment" + }, + "xbuild-tools": { + "$ref": "#/properties/xbuild-tools" + }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, + "repair-wheel-command": { + "$ref": "#/properties/repair-wheel-command" + }, + "test-command": { + "$ref": "#/properties/test-command" + }, + "test-extras": { + "$ref": "#/properties/test-extras" + }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, + "test-requires": { + "$ref": "#/properties/test-requires" + }, + "test-environment": { + "$ref": "#/properties/test-environment" + } + } + } + } +} diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt new file mode 100644 index 000000000..b41080b7b --- /dev/null +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -0,0 +1,117 @@ +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via httpx +auditwheel-emscripten==0.1.0 + # via pyodide-build +build==1.2.2.post1 + # via + # -r .nox/update_constraints/tmp/constraints-pyodide.in + # pyodide-build +certifi==2025.4.26 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.2 + # via requests +click==8.1.8 + # via + # -r .nox/update_constraints/tmp/constraints-pyodide.in + # typer +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via unearth +idna==3.10 + # via + # anyio + # httpx + # requests +leb128==1.0.8 + # via auditwheel-emscripten +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +packaging==25.0 + # via + # auditwheel-emscripten + # build + # pyodide-build + # unearth +pip==25.1.1 + # via -r .nox/update_constraints/tmp/constraints-pyodide.in +platformdirs==4.3.8 + # via + # pyodide-build + # virtualenv +pydantic==2.11.5 + # via + # pyodide-build + # pyodide-lock +pydantic-core==2.33.2 + # via pydantic +pygments==2.19.1 + # via rich +pyodide-build==0.30.5 + # via -r .nox/update_constraints/tmp/constraints-pyodide.in +pyodide-cli==0.3.0 + # via + # auditwheel-emscripten + # pyodide-build +pyodide-lock==0.1.0a7 + # via pyodide-build +pyproject-hooks==1.2.0 + # via build +requests==2.32.4 + # via pyodide-build +resolvelib==1.1.0 + # via pyodide-build +rich==14.0.0 + # via + # pyodide-build + # pyodide-cli + # typer +ruamel-yaml==0.18.14 + # via pyodide-build +ruamel-yaml-clib==0.2.12 + # via ruamel-yaml +shellingham==1.5.4 + # via typer +sniffio==1.3.1 + # via anyio +typer==0.16.0 + # via + # auditwheel-emscripten + # pyodide-build + # pyodide-cli +typing-extensions==4.14.0 + # via + # anyio + # pydantic + # pydantic-core + # typer + # typing-inspection +typing-inspection==0.4.1 + # via pydantic +unearth==0.17.5 + # via pyodide-build +urllib3==2.4.0 + # via requests +virtualenv==20.31.2 + # via + # build + # pyodide-build +wheel==0.45.1 + # via + # auditwheel-emscripten + # pyodide-build diff --git a/cibuildwheel/resources/constraints-pyodide313.txt b/cibuildwheel/resources/constraints-pyodide313.txt new file mode 100644 index 000000000..b3a5ffd68 --- /dev/null +++ b/cibuildwheel/resources/constraints-pyodide313.txt @@ -0,0 +1,116 @@ +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via httpx +auditwheel-emscripten==0.1.0 + # via pyodide-build +build==1.2.2.post1 + # via + # -r .nox/update_constraints/tmp/constraints-pyodide.in + # pyodide-build +certifi==2025.4.26 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.2 + # via requests +click==8.1.8 + # via + # -r .nox/update_constraints/tmp/constraints-pyodide.in + # typer +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via unearth +idna==3.10 + # via + # anyio + # httpx + # requests +leb128==1.0.8 + # via auditwheel-emscripten +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +packaging==25.0 + # via + # auditwheel-emscripten + # build + # pyodide-build + # unearth +pip==25.1.1 + # via -r .nox/update_constraints/tmp/constraints-pyodide.in +platformdirs==4.3.8 + # via + # pyodide-build + # virtualenv +pydantic==2.11.5 + # via + # pyodide-build + # pyodide-lock +pydantic-core==2.33.2 + # via pydantic +pygments==2.19.1 + # via rich +pyodide-build==0.30.5 + # via -r .nox/update_constraints/tmp/constraints-pyodide.in +pyodide-cli==0.3.0 + # via + # auditwheel-emscripten + # pyodide-build +pyodide-lock==0.1.0a7 + # via pyodide-build +pyproject-hooks==1.2.0 + # via build +requests==2.32.4 + # via pyodide-build +resolvelib==1.1.0 + # via pyodide-build +rich==14.0.0 + # via + # pyodide-build + # pyodide-cli + # typer +ruamel-yaml==0.18.14 + # via pyodide-build +ruamel-yaml-clib==0.2.12 + # via ruamel-yaml +shellingham==1.5.4 + # via typer +sniffio==1.3.1 + # via anyio +typer==0.16.0 + # via + # auditwheel-emscripten + # pyodide-build + # pyodide-cli +typing-extensions==4.14.0 + # via + # pydantic + # pydantic-core + # typer + # typing-inspection +typing-inspection==0.4.1 + # via pydantic +unearth==0.17.5 + # via pyodide-build +urllib3==2.4.0 + # via requests +virtualenv==20.31.2 + # via + # build + # pyodide-build +wheel==0.45.1 + # via + # auditwheel-emscripten + # pyodide-build diff --git a/cibuildwheel/resources/constraints-python310.txt b/cibuildwheel/resources/constraints-python310.txt new file mode 100644 index 000000000..ee7e66578 --- /dev/null +++ b/cibuildwheel/resources/constraints-python310.txt @@ -0,0 +1,34 @@ +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +altgraph==0.17.4 + # via macholib +build==1.2.2.post1 + # via -r cibuildwheel/resources/constraints.in +delocate==0.13.0 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +importlib-metadata==8.7.0 + # via build +macholib==1.16.3 + # via delocate +packaging==25.0 + # via + # build + # delocate +pip==25.1.1 + # via -r cibuildwheel/resources/constraints.in +platformdirs==4.3.8 + # via virtualenv +pyproject-hooks==1.2.0 + # via build +tomli==2.2.1 + # via build +typing-extensions==4.14.0 + # via delocate +virtualenv==20.31.2 + # via -r cibuildwheel/resources/constraints.in +zipp==3.23.0 + # via importlib-metadata diff --git a/cibuildwheel/resources/constraints-python311.txt b/cibuildwheel/resources/constraints-python311.txt new file mode 100644 index 000000000..4f0b01819 --- /dev/null +++ b/cibuildwheel/resources/constraints-python311.txt @@ -0,0 +1,28 @@ +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +altgraph==0.17.4 + # via macholib +build==1.2.2.post1 + # via -r cibuildwheel/resources/constraints.in +delocate==0.13.0 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +macholib==1.16.3 + # via delocate +packaging==25.0 + # via + # build + # delocate +pip==25.1.1 + # via -r cibuildwheel/resources/constraints.in +platformdirs==4.3.8 + # via virtualenv +pyproject-hooks==1.2.0 + # via build +typing-extensions==4.14.0 + # via delocate +virtualenv==20.31.2 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python312.txt b/cibuildwheel/resources/constraints-python312.txt new file mode 100644 index 000000000..4f0b01819 --- /dev/null +++ b/cibuildwheel/resources/constraints-python312.txt @@ -0,0 +1,28 @@ +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +altgraph==0.17.4 + # via macholib +build==1.2.2.post1 + # via -r cibuildwheel/resources/constraints.in +delocate==0.13.0 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +macholib==1.16.3 + # via delocate +packaging==25.0 + # via + # build + # delocate +pip==25.1.1 + # via -r cibuildwheel/resources/constraints.in +platformdirs==4.3.8 + # via virtualenv +pyproject-hooks==1.2.0 + # via build +typing-extensions==4.14.0 + # via delocate +virtualenv==20.31.2 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt new file mode 100644 index 000000000..4f0b01819 --- /dev/null +++ b/cibuildwheel/resources/constraints-python313.txt @@ -0,0 +1,28 @@ +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +altgraph==0.17.4 + # via macholib +build==1.2.2.post1 + # via -r cibuildwheel/resources/constraints.in +delocate==0.13.0 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +macholib==1.16.3 + # via delocate +packaging==25.0 + # via + # build + # delocate +pip==25.1.1 + # via -r cibuildwheel/resources/constraints.in +platformdirs==4.3.8 + # via virtualenv +pyproject-hooks==1.2.0 + # via build +typing-extensions==4.14.0 + # via delocate +virtualenv==20.31.2 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python314.txt b/cibuildwheel/resources/constraints-python314.txt new file mode 100644 index 000000000..4f0b01819 --- /dev/null +++ b/cibuildwheel/resources/constraints-python314.txt @@ -0,0 +1,28 @@ +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +altgraph==0.17.4 + # via macholib +build==1.2.2.post1 + # via -r cibuildwheel/resources/constraints.in +delocate==0.13.0 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +macholib==1.16.3 + # via delocate +packaging==25.0 + # via + # build + # delocate +pip==25.1.1 + # via -r cibuildwheel/resources/constraints.in +platformdirs==4.3.8 + # via virtualenv +pyproject-hooks==1.2.0 + # via build +typing-extensions==4.14.0 + # via delocate +virtualenv==20.31.2 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python36.txt b/cibuildwheel/resources/constraints-python36.txt deleted file mode 100644 index 83fb265f5..000000000 --- a/cibuildwheel/resources/constraints-python36.txt +++ /dev/null @@ -1,40 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.6 -# To update, run: -# -# bin/update_dependencies.py -# -delocate==0.10.2 - # via -r cibuildwheel/resources/constraints.in -distlib==0.3.4 - # via virtualenv -filelock==3.4.1 - # via virtualenv -importlib-metadata==4.8.3 - # via virtualenv -importlib-resources==5.4.0 - # via virtualenv -platformdirs==2.4.0 - # via virtualenv -six==1.16.0 - # via virtualenv -typing-extensions==4.0.1 - # via - # delocate - # importlib-metadata -virtualenv==20.13.0 - # via -r cibuildwheel/resources/constraints.in -wheel==0.37.1 - # via - # -r cibuildwheel/resources/constraints.in - # delocate -zipp==3.6.0 - # via - # importlib-metadata - # importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -pip==21.3.1 - # via -r cibuildwheel/resources/constraints.in -setuptools==59.6.0 - # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python37.txt b/cibuildwheel/resources/constraints-python37.txt deleted file mode 100644 index f1138c100..000000000 --- a/cibuildwheel/resources/constraints-python37.txt +++ /dev/null @@ -1,36 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: -# -# bin/update_dependencies.py -# -delocate==0.10.2 - # via -r cibuildwheel/resources/constraints.in -distlib==0.3.4 - # via virtualenv -filelock==3.4.2 - # via virtualenv -importlib-metadata==4.10.1 - # via virtualenv -platformdirs==2.4.1 - # via virtualenv -six==1.16.0 - # via virtualenv -typing-extensions==4.0.1 - # via - # delocate - # importlib-metadata -virtualenv==20.13.0 - # via -r cibuildwheel/resources/constraints.in -wheel==0.37.1 - # via - # -r cibuildwheel/resources/constraints.in - # delocate -zipp==3.7.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -pip==21.3.1 - # via -r cibuildwheel/resources/constraints.in -setuptools==60.5.0 - # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index 16424d18a..2c40131f4 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -1,30 +1,34 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# bin/update_dependencies.py -# -delocate==0.10.2 +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +altgraph==0.17.4 + # via macholib +build==1.2.2.post1 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.4 - # via virtualenv -filelock==3.4.2 - # via virtualenv -platformdirs==2.4.1 +delocate==0.12.0 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.9 # via virtualenv -six==1.16.0 +filelock==3.16.1 # via virtualenv -typing-extensions==4.0.1 +importlib-metadata==8.5.0 + # via build +macholib==1.16.3 # via delocate -virtualenv==20.13.0 - # via -r cibuildwheel/resources/constraints.in -wheel==0.37.1 +packaging==25.0 # via - # -r cibuildwheel/resources/constraints.in + # build # delocate - -# The following packages are considered to be unsafe in a requirements file: -pip==21.3.1 +pip==25.0.1 # via -r cibuildwheel/resources/constraints.in -setuptools==60.5.0 +platformdirs==4.3.6 + # via virtualenv +pyproject-hooks==1.2.0 + # via build +tomli==2.2.1 + # via build +typing-extensions==4.13.2 + # via delocate +virtualenv==20.31.2 # via -r cibuildwheel/resources/constraints.in +zipp==3.20.2 + # via importlib-metadata diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index 70434786b..ee7e66578 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -1,30 +1,34 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# bin/update_dependencies.py -# -delocate==0.10.2 +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +altgraph==0.17.4 + # via macholib +build==1.2.2.post1 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.4 - # via virtualenv -filelock==3.4.2 - # via virtualenv -platformdirs==2.4.1 +delocate==0.13.0 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.9 # via virtualenv -six==1.16.0 +filelock==3.18.0 # via virtualenv -typing-extensions==4.0.1 +importlib-metadata==8.7.0 + # via build +macholib==1.16.3 # via delocate -virtualenv==20.13.0 - # via -r cibuildwheel/resources/constraints.in -wheel==0.37.1 +packaging==25.0 # via - # -r cibuildwheel/resources/constraints.in + # build # delocate - -# The following packages are considered to be unsafe in a requirements file: -pip==21.3.1 +pip==25.1.1 # via -r cibuildwheel/resources/constraints.in -setuptools==60.5.0 +platformdirs==4.3.8 + # via virtualenv +pyproject-hooks==1.2.0 + # via build +tomli==2.2.1 + # via build +typing-extensions==4.14.0 + # via delocate +virtualenv==20.31.2 # via -r cibuildwheel/resources/constraints.in +zipp==3.23.0 + # via importlib-metadata diff --git a/cibuildwheel/resources/constraints.in b/cibuildwheel/resources/constraints.in index 9753106ac..50bfabb6e 100644 --- a/cibuildwheel/resources/constraints.in +++ b/cibuildwheel/resources/constraints.in @@ -1,5 +1,4 @@ pip -setuptools -wheel +build delocate virtualenv diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index 70434786b..4f0b01819 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -1,30 +1,28 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# bin/update_dependencies.py -# -delocate==0.10.2 +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +altgraph==0.17.4 + # via macholib +build==1.2.2.post1 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.4 - # via virtualenv -filelock==3.4.2 - # via virtualenv -platformdirs==2.4.1 +delocate==0.13.0 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.9 # via virtualenv -six==1.16.0 +filelock==3.18.0 # via virtualenv -typing-extensions==4.0.1 +macholib==1.16.3 # via delocate -virtualenv==20.13.0 - # via -r cibuildwheel/resources/constraints.in -wheel==0.37.1 +packaging==25.0 # via - # -r cibuildwheel/resources/constraints.in + # build # delocate - -# The following packages are considered to be unsafe in a requirements file: -pip==21.3.1 +pip==25.1.1 # via -r cibuildwheel/resources/constraints.in -setuptools==60.5.0 +platformdirs==4.3.8 + # via virtualenv +pyproject-hooks==1.2.0 + # via build +typing-extensions==4.14.0 + # via delocate +virtualenv==20.31.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 5890c8edf..396c1d3cc 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -2,46 +2,62 @@ build = "*" skip = "" test-skip = "" +enable = [] archs = ["auto"] -build-frontend = "pip" +build-frontend = "default" +config-settings = {} dependency-versions = "pinned" environment = {} environment-pass = [] -build-verbosity = "" +build-verbosity = 0 before-all = "" before-build = "" +# TOML doesn't support explicit NULLs; use ["\u0000"] as a sentinel value. +xbuild-tools = ["\u0000"] repair-wheel-command = "" test-command = "" before-test = "" +test-sources = [] test-requires = [] test-extras = [] +test-groups = [] +test-environment = {} -manylinux-x86_64-image = "manylinux2014" +container-engine = "docker" + +pyodide-version = "" + +manylinux-x86_64-image = "manylinux_2_28" manylinux-i686-image = "manylinux2014" -manylinux-aarch64-image = "manylinux2014" -manylinux-ppc64le-image = "manylinux2014" -manylinux-s390x-image = "manylinux2014" -manylinux-pypy_x86_64-image = "manylinux2014" +manylinux-aarch64-image = "manylinux_2_28" +manylinux-ppc64le-image = "manylinux_2_28" +manylinux-s390x-image = "manylinux_2_28" +manylinux-armv7l-image = "manylinux_2_31" +manylinux-riscv64-image = "ghcr.io/pypa/cibuildwheel/no_default_image:please_use_override" +manylinux-pypy_x86_64-image = "manylinux_2_28" manylinux-pypy_i686-image = "manylinux2014" -manylinux-pypy_aarch64-image = "manylinux2014" +manylinux-pypy_aarch64-image = "manylinux_2_28" -musllinux-x86_64-image = "musllinux_1_1" -musllinux-i686-image = "musllinux_1_1" -musllinux-aarch64-image = "musllinux_1_1" -musllinux-ppc64le-image = "musllinux_1_1" -musllinux-s390x-image = "musllinux_1_1" +musllinux-x86_64-image = "musllinux_1_2" +musllinux-i686-image = "musllinux_1_2" +musllinux-aarch64-image = "musllinux_1_2" +musllinux-ppc64le-image = "musllinux_1_2" +musllinux-s390x-image = "musllinux_1_2" +musllinux-armv7l-image = "musllinux_1_2" +musllinux-riscv64-image = "ghcr.io/pypa/cibuildwheel/no_default_image:please_use_override" [tool.cibuildwheel.linux] repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" [tool.cibuildwheel.macos] -repair-wheel-command = [ - "delocate-listdeps {wheel}", - "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}", -] +repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" [tool.cibuildwheel.windows] + +[tool.cibuildwheel.ios] + +[tool.cibuildwheel.pyodide] diff --git a/cibuildwheel/resources/free-threaded-enable-313.xml b/cibuildwheel/resources/free-threaded-enable-313.xml new file mode 100644 index 000000000..0c1c6c78c --- /dev/null +++ b/cibuildwheel/resources/free-threaded-enable-313.xml @@ -0,0 +1,14 @@ + + + + + + attributeSetting + 1 + choiceAttribute + selected + choiceIdentifier + org.python.Python.PythonTFramework-3.13 + + + diff --git a/cibuildwheel/resources/free-threaded-enable-314.xml b/cibuildwheel/resources/free-threaded-enable-314.xml new file mode 100644 index 000000000..88fa8dd98 --- /dev/null +++ b/cibuildwheel/resources/free-threaded-enable-314.xml @@ -0,0 +1,14 @@ + + + + + + attributeSetting + 1 + choiceAttribute + selected + choiceIdentifier + org.python.Python.PythonTFramework-3.14 + + + diff --git a/cibuildwheel/resources/install_certifi.py b/cibuildwheel/resources/install_certifi.py index 89bf82f04..860521750 100644 --- a/cibuildwheel/resources/install_certifi.py +++ b/cibuildwheel/resources/install_certifi.py @@ -6,6 +6,7 @@ # for the ssl module. Uses the certificates provided by the certifi package: # https://pypi.org/project/certifi/ +import contextlib import os import os.path import ssl @@ -25,7 +26,7 @@ ) -def main(): +def main() -> None: openssl_dir, openssl_cafile = os.path.split(ssl.get_default_verify_paths().openssl_cafile) print(" -- pip install --upgrade certifi") subprocess.check_call( @@ -39,10 +40,8 @@ def main(): relpath_to_certifi_cafile = os.path.relpath(certifi.where()) print(" -- removing any existing file or link") - try: + with contextlib.suppress(FileNotFoundError): os.remove(openssl_cafile) - except FileNotFoundError: - pass print(" -- creating symlink to certifi certificate bundle") os.symlink(relpath_to_certifi_cafile, openssl_cafile) diff --git a/cibuildwheel/resources/nodejs.toml b/cibuildwheel/resources/nodejs.toml new file mode 100644 index 000000000..2897eac28 --- /dev/null +++ b/cibuildwheel/resources/nodejs.toml @@ -0,0 +1,3 @@ +url = "https://nodejs.org/dist/" +v22 = "v22.16.0" +v20 = "v20.19.2" diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index 5970ea43e..479ffca70 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,43 +1,47 @@ [x86_64] -manylinux1 = quay.io/pypa/manylinux1_x86_64:2022-01-22-4072568 -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-01-22-72ab18b -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2022-01-22-72ab18b -manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-01-22-72ab18b -musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2022-01-22-72ab18b +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2025.06.08-1 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2025.06.08-1 +manylinux_2_34 = quay.io/pypa/manylinux_2_34_x86_64:2025.06.08-1 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64:2025.06.08-1 [i686] -manylinux1 = quay.io/pypa/manylinux1_i686:2022-01-22-4072568 -manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-01-22-72ab18b -manylinux2014 = quay.io/pypa/manylinux2014_i686:2022-01-22-72ab18b -manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-01-22-72ab18b -musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2022-01-22-72ab18b - -[pypy_x86_64] -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-01-22-72ab18b -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2022-01-22-72ab18b -manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-01-22-72ab18b - -[pypy_i686] -manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-01-22-72ab18b -manylinux2014 = quay.io/pypa/manylinux2014_i686:2022-01-22-72ab18b -manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-01-22-72ab18b +manylinux2014 = quay.io/pypa/manylinux2014_i686:2025.06.08-1 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686:2025.06.08-1 [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2022-01-22-72ab18b -manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-01-22-72ab18b -musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2022-01-22-72ab18b +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2025.06.08-1 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2025.06.08-1 +manylinux_2_34 = quay.io/pypa/manylinux_2_34_aarch64:2025.06.08-1 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_aarch64:2025.06.08-1 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2022-01-22-72ab18b -manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2022-01-22-72ab18b -musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2022-01-22-72ab18b +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2025.06.08-1 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2025.06.08-1 +manylinux_2_34 = quay.io/pypa/manylinux_2_34_ppc64le:2025.06.08-1 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_ppc64le:2025.06.08-1 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2022-01-22-72ab18b -manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2022-01-22-72ab18b -musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2022-01-22-72ab18b +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2025.06.08-1 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x:2025.06.08-1 +manylinux_2_34 = quay.io/pypa/manylinux_2_34_s390x:2025.06.08-1 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_s390x:2025.06.08-1 + +[pypy_x86_64] +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2025.06.08-1 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2025.06.08-1 +manylinux_2_34 = quay.io/pypa/manylinux_2_34_x86_64:2025.06.08-1 + +[pypy_i686] +manylinux2014 = quay.io/pypa/manylinux2014_i686:2025.06.08-1 [pypy_aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2022-01-22-72ab18b -manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-01-22-72ab18b +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2025.06.08-1 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2025.06.08-1 +manylinux_2_34 = quay.io/pypa/manylinux_2_34_aarch64:2025.06.08-1 + +[armv7l] +manylinux_2_31 = quay.io/pypa/manylinux_2_31_armv7l:2025.06.08-1 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_armv7l:2025.06.08-1 + +[riscv64] diff --git a/cibuildwheel/resources/python-build-standalone-releases.json b/cibuildwheel/resources/python-build-standalone-releases.json new file mode 100644 index 000000000..2597c29ba --- /dev/null +++ b/cibuildwheel/resources/python-build-standalone-releases.json @@ -0,0 +1,441 @@ +{ + "releases": [ + { + "tag": "20250604", + "assets": [ + { + "name": "cpython-3.10.18+20250604-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.18+20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.10.18%2B20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.13+20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.11.13%2B20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.11+20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.12.11%2B20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.4+20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.13.4%2B20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0b1+20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.14.0b1%2B20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.23+20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250604/cpython-3.9.23%2B20250604-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + } + ] + } + ] +} diff --git a/cibuildwheel/resources/testing_temp_dir_file.py b/cibuildwheel/resources/testing_temp_dir_file.py new file mode 100644 index 000000000..094f2f6fb --- /dev/null +++ b/cibuildwheel/resources/testing_temp_dir_file.py @@ -0,0 +1,36 @@ +# this file is copied to the testing cwd, to raise the below error message if +# pytest/unittest is run from there. + +import sys +import unittest +from typing import NoReturn + + +class TestStringMethods(unittest.TestCase): + def test_fail(self) -> NoReturn: + if sys.platform == "ios": + msg = ( + "You tried to run tests from the testbed app's working " + "directory, without specifying `test-sources`. " + "On iOS, you must copy your test files to the testbed app by " + "setting the `test-sources` option in your cibuildwheel " + "configuration." + ) + else: + msg = ( + "cibuildwheel executes tests from a different working directory to " + "your project. This ensures only your wheel is imported, preventing " + "Python from accessing files that haven't been packaged into the " + "wheel. " + "\n\n" + "Please specify a path to your tests when invoking pytest " + "using the {project} placeholder, e.g. `pytest {project}` or " + "`pytest {project}/tests`. cibuildwheel will replace {project} with " + "the path to your project. " + "\n\n" + "Alternatively, you can specify your test files using the " + "`test-sources` option, and cibuildwheel will copy them to the " + "working directory for testing." + ) + + self.fail(msg) diff --git a/cibuildwheel/resources/virtualenv.toml b/cibuildwheel/resources/virtualenv.toml index e6c1df91c..9e5d08c24 100644 --- a/cibuildwheel/resources/virtualenv.toml +++ b/cibuildwheel/resources/virtualenv.toml @@ -1,2 +1 @@ -version = "20.13.0" -url = "https://github.com/pypa/get-virtualenv/blob/20.13.0/public/virtualenv.pyz?raw=true" +default = { version = "20.31.2", url = "https://github.com/pypa/get-virtualenv/blob/20.31.2/public/virtualenv.pyz?raw=true" } diff --git a/cibuildwheel/schema.py b/cibuildwheel/schema.py new file mode 100644 index 000000000..ef8f603b2 --- /dev/null +++ b/cibuildwheel/schema.py @@ -0,0 +1,12 @@ +import json +from typing import Any + +from .util import resources + + +def get_schema(tool_name: str = "cibuildwheel") -> dict[str, Any]: + "Get the stored complete schema for cibuildwheel settings." + assert tool_name == "cibuildwheel", "Only cibuildwheel is supported." + + with resources.CIBUILDWHEEL_SCHEMA.open(encoding="utf-8") as f: + return json.load(f) # type: ignore[no-any-return] diff --git a/cibuildwheel/selector.py b/cibuildwheel/selector.py new file mode 100644 index 000000000..256e4fb04 --- /dev/null +++ b/cibuildwheel/selector.py @@ -0,0 +1,134 @@ +import dataclasses +import itertools +from enum import StrEnum +from fnmatch import fnmatch +from typing import Any + +import bracex +from packaging.specifiers import SpecifierSet +from packaging.version import Version + + +def selector_matches(patterns: str, string: str) -> bool: + """ + Returns True if `string` is matched by any of the wildcard patterns in + `patterns`. + + Matching is according to fnmatch, but with shell-like curly brace + expansion. For example, 'cp{36,37}-*' would match either of 'cp36-*' or + 'cp37-*'. + """ + + patterns_list = patterns.split() + expanded_patterns = itertools.chain.from_iterable(bracex.expand(p) for p in patterns_list) + return any(fnmatch(string, pat) for pat in expanded_patterns) + + +class EnableGroup(StrEnum): + """ + Groups of build selectors that are not enabled by default. + """ + + CPythonExperimentalRiscV64 = "cpython-experimental-riscv64" + CPythonFreeThreading = "cpython-freethreading" + CPythonPrerelease = "cpython-prerelease" + GraalPy = "graalpy" + PyPy = "pypy" + PyPyEoL = "pypy-eol" + PyodidePrerelease = "pyodide-prerelease" + + @classmethod + def all_groups(cls) -> frozenset["EnableGroup"]: + return frozenset(cls) + + @classmethod + def parse_option_value(cls, value: str) -> frozenset["EnableGroup"]: + """ + Parses a string of space-separated values into a set of EnableGroup + members. The string may contain group names or "all". + """ + result = set() + for group in value.strip().split(): + if group == "all": + return cls.all_groups() + try: + result.add(cls(group)) + except ValueError: + msg = f"Unknown enable group: {group}" + raise ValueError(msg) from None + return frozenset(result) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class BuildSelector: + """ + This class holds a set of build/skip patterns. You call an instance with a + build identifier, and it returns True if that identifier should be + included. Only call this on valid identifiers, ones that have at least 2 + numeric digits before the first dash. + """ + + build_config: str + skip_config: str + requires_python: SpecifierSet | None = None + enable: frozenset[EnableGroup] = frozenset() + + def __call__(self, build_id: str) -> bool: + # Filter build selectors by python_requires if set + if self.requires_python is not None: + py_ver_str = build_id.split("-")[0].split("_")[0] + py_ver_str = py_ver_str.removesuffix("t") + major = int(py_ver_str[2]) + minor = int(py_ver_str[3:]) + version = Version(f"{major}.{minor}.99") + if not self.requires_python.contains(version): + return False + + # filter out groups that are not enabled + if EnableGroup.CPythonFreeThreading not in self.enable and fnmatch(build_id, "cp3??t-*"): + return False + if EnableGroup.CPythonPrerelease not in self.enable and fnmatch(build_id, "cp314*"): + return False + if EnableGroup.PyPy not in self.enable and fnmatch(build_id, "pp31*"): + return False + if EnableGroup.PyPyEoL not in self.enable and fnmatch(build_id, "pp3?-*"): + return False + if EnableGroup.CPythonExperimentalRiscV64 not in self.enable and fnmatch( + build_id, "*_riscv64" + ): + return False + if EnableGroup.GraalPy not in self.enable and fnmatch(build_id, "gp*"): + return False + if EnableGroup.PyodidePrerelease not in self.enable and fnmatch( + build_id, "cp313-pyodide_*" + ): + return False + + should_build = selector_matches(self.build_config, build_id) + should_skip = selector_matches(self.skip_config, build_id) + + return should_build and not should_skip + + def options_summary(self) -> Any: + return { + "build_config": self.build_config, + "skip_config": self.skip_config, + "requires_python": str(self.requires_python), + "enable": sorted(group.value for group in self.enable), + } + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class TestSelector: + """ + A build selector that can only skip tests according to a skip pattern. + """ + + skip_config: str + + def __call__(self, build_id: str) -> bool: + should_skip = selector_matches(self.skip_config, build_id) + return not should_skip + + def options_summary(self) -> Any: + return {"skip_config": self.skip_config} diff --git a/cibuildwheel/typing.py b/cibuildwheel/typing.py index 1e6e57873..420ef05f6 100644 --- a/cibuildwheel/typing.py +++ b/cibuildwheel/typing.py @@ -1,43 +1,21 @@ import os -import subprocess -import sys -from typing import TYPE_CHECKING, NoReturn, Set, Union - -if sys.version_info < (3, 8): - from typing_extensions import Final, Literal, OrderedDict, Protocol, TypedDict -else: - from typing import Final, Literal, OrderedDict, Protocol, TypedDict - +import typing +from typing import Final, Literal, Protocol __all__ = ( - "Final", - "Literal", "PLATFORMS", "PathOrStr", "PlatformName", - "Protocol", - "PLATFORMS", - "PopenBytes", - "Protocol", - "Set", - "TypedDict", - "OrderedDict", - "Union", - "assert_never", ) -if TYPE_CHECKING: - PopenBytes = subprocess.Popen[bytes] - PathOrStr = Union[str, os.PathLike[str]] -else: - PopenBytes = subprocess.Popen - PathOrStr = Union[str, "os.PathLike[str]"] +PathOrStr = str | os.PathLike[str] -PlatformName = Literal["linux", "macos", "windows"] -PLATFORMS: Final[Set[PlatformName]] = {"linux", "macos", "windows"} +PlatformName = Literal["linux", "macos", "windows", "pyodide", "ios"] +PLATFORMS: Final[frozenset[PlatformName]] = frozenset(typing.get_args(PlatformName)) -def assert_never(value: NoReturn) -> NoReturn: - assert False, f"Unhandled value: {value} ({type(value).__name__})" # noqa: B011 +class GenericPythonConfiguration(Protocol): + @property + def identifier(self) -> str: ... diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py deleted file mode 100644 index 516804d27..000000000 --- a/cibuildwheel/util.py +++ /dev/null @@ -1,562 +0,0 @@ -import contextlib -import fnmatch -import itertools -import os -import re -import shlex -import ssl -import subprocess -import sys -import textwrap -import time -import urllib.request -from enum import Enum -from functools import lru_cache -from pathlib import Path -from time import sleep -from typing import ( - Any, - Dict, - Iterable, - Iterator, - List, - NamedTuple, - Optional, - Sequence, - TextIO, - cast, - overload, -) - -import bracex -import certifi -import tomli -from filelock import FileLock -from packaging.requirements import InvalidRequirement, Requirement -from packaging.specifiers import SpecifierSet -from packaging.version import Version -from platformdirs import user_cache_path - -from cibuildwheel.typing import Literal, PathOrStr, PlatformName - -resources_dir = Path(__file__).parent / "resources" - -install_certifi_script = resources_dir / "install_certifi.py" - -BuildFrontend = Literal["pip", "build"] - -MANYLINUX_ARCHS = ( - "x86_64", - "i686", - "pypy_x86_64", - "aarch64", - "ppc64le", - "s390x", - "pypy_aarch64", - "pypy_i686", -) - -MUSLLINUX_ARCHS = ( - "x86_64", - "i686", - "aarch64", - "ppc64le", - "s390x", -) - -DEFAULT_CIBW_CACHE_PATH = user_cache_path(appname="cibuildwheel", appauthor="pypa") -CIBW_CACHE_PATH = Path(os.environ.get("CIBW_CACHE_PATH", DEFAULT_CIBW_CACHE_PATH)).resolve() - -IS_WIN = sys.platform.startswith("win") - - -@overload -def call( - *args: PathOrStr, - env: Optional[Dict[str, str]] = None, - cwd: Optional[PathOrStr] = None, - capture_stdout: Literal[False] = ..., -) -> None: - ... - - -@overload -def call( - *args: PathOrStr, - env: Optional[Dict[str, str]] = None, - cwd: Optional[PathOrStr] = None, - capture_stdout: Literal[True], -) -> str: - ... - - -def call( - *args: PathOrStr, - env: Optional[Dict[str, str]] = None, - cwd: Optional[PathOrStr] = None, - capture_stdout: bool = False, -) -> Optional[str]: - """ - Run subprocess.run, but print the commands first. Takes the commands as - *args. Uses shell=True on Windows due to a bug. Also converts to - Paths to strings, due to Windows behavior at least on older Pythons. - https://bugs.python.org/issue8557 - """ - args_ = [str(arg) for arg in args] - # print the command executing for the logs - print("+ " + " ".join(shlex.quote(a) for a in args_)) - kwargs: Dict[str, Any] = {} - if capture_stdout: - kwargs["universal_newlines"] = True - kwargs["stdout"] = subprocess.PIPE - result = subprocess.run(args_, check=True, shell=IS_WIN, env=env, cwd=cwd, **kwargs) - if not capture_stdout: - return None - return cast(str, result.stdout) - - -def shell( - command: str, env: Optional[Dict[str, str]] = None, cwd: Optional[PathOrStr] = None -) -> None: - print(f"+ {command}") - subprocess.run(command, env=env, cwd=cwd, shell=True, check=True) - - -def format_safe(template: str, **kwargs: Any) -> str: - """ - Works similarly to `template.format(**kwargs)`, except that unmatched - fields in `template` are passed through untouched. - - >>> format_safe('{a} {b}', a='123') - '123 {b}' - >>> format_safe('{a} {b[4]:3f}', a='123') - '123 {b[4]:3f}' - - To avoid variable expansion, precede with a single backslash e.g. - >>> format_safe('\\{a} {b}', a='123') - '{a} {b}' - """ - - result = template - - for key, value in kwargs.items(): - find_pattern = re.compile( - fr""" - (? str: - """ - Preprocesses a command by expanding variables like {python}. - - For example, used in the test_command option to specify the path to the - project's root. Unmatched syntax will mostly be allowed through. - """ - return format_safe(command, python="python", pip="pip", **kwargs) - - -def get_build_verbosity_extra_flags(level: int) -> List[str]: - if level > 0: - return ["-" + level * "v"] - elif level < 0: - return ["-" + -level * "q"] - else: - return [] - - -def read_python_configs(config: PlatformName) -> List[Dict[str, str]]: - input_file = resources_dir / "build-platforms.toml" - with input_file.open("rb") as f: - loaded_file = tomli.load(f) - results: List[Dict[str, str]] = list(loaded_file[config]["python_configurations"]) - return results - - -def selector_matches(patterns: str, string: str) -> bool: - """ - Returns True if `string` is matched by any of the wildcard patterns in - `patterns`. - - Matching is according to fnmatch, but with shell-like curly brace - expansion. For example, 'cp{36,37}-*' would match either of 'cp36-*' or - 'cp37-*'. - """ - patterns_list = patterns.split() - expanded_patterns = itertools.chain.from_iterable(bracex.expand(p) for p in patterns_list) - return any(fnmatch.fnmatch(string, pat) for pat in expanded_patterns) - - -class IdentifierSelector: - """ - This class holds a set of build/skip patterns. You call an instance with a - build identifier, and it returns True if that identifier should be - included. Only call this on valid identifiers, ones that have at least 2 - numeric digits before the first dash. If a pre-release version X.Y is present, - you can filter it with prerelease="XY". - """ - - # a pattern that skips prerelease versions, when include_prereleases is False. - PRERELEASE_SKIP = "" - - def __init__( - self, - *, - build_config: str, - skip_config: str, - requires_python: Optional[SpecifierSet] = None, - prerelease_pythons: bool = False, - ): - self.build_config = build_config - self.skip_config = skip_config - self.requires_python = requires_python - self.prerelease_pythons = prerelease_pythons - - def __call__(self, build_id: str) -> bool: - # Filter build selectors by python_requires if set - if self.requires_python is not None: - py_ver_str = build_id.split("-")[0] - major = int(py_ver_str[2]) - minor = int(py_ver_str[3:]) - version = Version(f"{major}.{minor}.99") - if not self.requires_python.contains(version): - return False - - # filter out the prerelease pythons if self.prerelease_pythons is False - if not self.prerelease_pythons and selector_matches( - BuildSelector.PRERELEASE_SKIP, build_id - ): - return False - - should_build = selector_matches(self.build_config, build_id) - should_skip = selector_matches(self.skip_config, build_id) - - return should_build and not should_skip - - def __repr__(self) -> str: - result = f"{self.__class__.__name__}(build_config={self.build_config!r}" - - if self.skip_config: - result += f", skip_config={self.skip_config!r}" - if self.prerelease_pythons: - result += ", prerelease_pythons=True" - - result += ")" - - return result - - -class BuildSelector(IdentifierSelector): - pass - - -# Note that requires-python is not needed for TestSelector, as you can't test -# what you can't build. -class TestSelector(IdentifierSelector): - def __init__(self, *, skip_config: str): - super().__init__(build_config="*", skip_config=skip_config) - - -# Taken from https://stackoverflow.com/a/107717 -class Unbuffered: - def __init__(self, stream: TextIO) -> None: - self.stream = stream - - def write(self, data: str) -> None: - self.stream.write(data) - self.stream.flush() - - def writelines(self, data: Iterable[str]) -> None: - self.stream.writelines(data) - self.stream.flush() - - def __getattr__(self, attr: str) -> Any: - return getattr(self.stream, attr) - - -def download(url: str, dest: Path) -> None: - print(f"+ Download {url} to {dest}") - dest_dir = dest.parent - if not dest_dir.exists(): - dest_dir.mkdir(parents=True) - - # we've had issues when relying on the host OS' CA certificates on Windows, - # so we use certifi (this sounds odd but requests also does this by default) - cafile = os.environ.get("SSL_CERT_FILE", certifi.where()) - context = ssl.create_default_context(cafile=cafile) - repeat_num = 3 - for i in range(repeat_num): - try: - response = urllib.request.urlopen(url, context=context) - except Exception: - if i == repeat_num - 1: - raise - sleep(3) - continue - break - - try: - dest.write_bytes(response.read()) - finally: - response.close() - - -class DependencyConstraints: - def __init__(self, base_file_path: Path): - assert base_file_path.exists() - self.base_file_path = base_file_path.resolve() - - @staticmethod - def with_defaults() -> "DependencyConstraints": - return DependencyConstraints(base_file_path=resources_dir / "constraints.txt") - - def get_for_python_version(self, version: str) -> Path: - version_parts = version.split(".") - - # try to find a version-specific dependency file e.g. if - # ./constraints.txt is the base, look for ./constraints-python36.txt - specific_stem = self.base_file_path.stem + f"-python{version_parts[0]}{version_parts[1]}" - specific_name = specific_stem + self.base_file_path.suffix - specific_file_path = self.base_file_path.with_name(specific_name) - if specific_file_path.exists(): - return specific_file_path - else: - return self.base_file_path - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.base_file_path!r})" - - def __eq__(self, o: object) -> bool: - if not isinstance(o, DependencyConstraints): - return False - - return self.base_file_path == o.base_file_path - - -class NonPlatformWheelError(Exception): - def __init__(self) -> None: - message = textwrap.dedent( - """ - cibuildwheel: Build failed because a pure Python wheel was generated. - - If you intend to build a pure-Python wheel, you don't need cibuildwheel - use - `pip wheel -w DEST_DIR .` instead. - - If you expected a platform wheel, check your project configuration, or run - cibuildwheel with CIBW_BUILD_VERBOSITY=1 to view build logs. - """ - ) - - super().__init__(message) - - -def strtobool(val: str) -> bool: - return val.lower() in {"y", "yes", "t", "true", "on", "1"} - - -class CIProvider(Enum): - travis_ci = "travis" - appveyor = "appveyor" - circle_ci = "circle_ci" - azure_pipelines = "azure_pipelines" - github_actions = "github_actions" - gitlab = "gitlab" - other = "other" - - -def detect_ci_provider() -> Optional[CIProvider]: - if "TRAVIS" in os.environ: - return CIProvider.travis_ci - elif "APPVEYOR" in os.environ: - return CIProvider.appveyor - elif "CIRCLECI" in os.environ: - return CIProvider.circle_ci - elif "AZURE_HTTP_USER_AGENT" in os.environ: - return CIProvider.azure_pipelines - elif "GITHUB_ACTIONS" in os.environ: - return CIProvider.github_actions - elif "GITLAB_CI" in os.environ: - return CIProvider.gitlab - elif strtobool(os.environ.get("CI", "false")): - return CIProvider.other - else: - return None - - -def unwrap(text: str) -> str: - """ - Unwraps multi-line text to a single line - """ - # remove initial line indent - text = textwrap.dedent(text) - # remove leading/trailing whitespace - text = text.strip() - # remove consecutive whitespace - return re.sub(r"\s+", " ", text) - - -@contextlib.contextmanager -def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]: - """ - Prints the new items in a directory upon exiting. The message to display - can include {n} for number of wheels, {s} for total number of seconds, - and/or {m} for total number of minutes. Does not print anything if this - exits via exception. - """ - - start_time = time.time() - existing_contents = set(output_dir.iterdir()) - yield - final_contents = set(output_dir.iterdir()) - - class FileReport(NamedTuple): - name: str - size: str - - new_contents = [ - FileReport(wheel.name, f"{(wheel.stat().st_size + 1023) // 1024:,d}") - for wheel in final_contents - existing_contents - ] - max_name_len = max(len(f.name) for f in new_contents) - max_size_len = max(len(f.size) for f in new_contents) - n = len(new_contents) - s = time.time() - start_time - m = s / 60 - print( - msg.format(n=n, s=s, m=m), - *sorted( - f" {f.name:<{max_name_len}s} {f.size:>{max_size_len}s} kB" for f in new_contents - ), - sep="\n", - ) - - -def get_pip_version(env: Dict[str, str]) -> str: - versions_output_text = call( - "python", "-m", "pip", "freeze", "--all", capture_stdout=True, env=env - ) - (pip_version,) = ( - version[5:] - for version in versions_output_text.strip().splitlines() - if version.startswith("pip==") - ) - return pip_version - - -@lru_cache(maxsize=None) -def _ensure_virtualenv() -> Path: - input_file = resources_dir / "virtualenv.toml" - with input_file.open("rb") as f: - loaded_file = tomli.load(f) - version = str(loaded_file["version"]) - url = str(loaded_file["url"]) - path = CIBW_CACHE_PATH / f"virtualenv-{version}.pyz" - with FileLock(str(path) + ".lock"): - if not path.exists(): - download(url, path) - return path - - -def _parse_constraints_for_virtualenv( - dependency_constraint_flags: Sequence[PathOrStr], -) -> Dict[str, str]: - """ - Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where - the key is the package name, and the value is the constraint version. - If a package version cannot be found, its value is "embed" meaning that virtualenv will install - its bundled version, already available locally. - The function does not try to be too smart and just handles basic constraints. - If it can't get an exact version, the real constraint will be handled by the - {macos|windows}.setup_python function. - """ - assert len(dependency_constraint_flags) in {0, 2} - packages = ["pip", "setuptools", "wheel"] - constraints_dict = {package: "embed" for package in packages} - if len(dependency_constraint_flags) == 2: - assert dependency_constraint_flags[0] == "-c" - constraint_path = Path(dependency_constraint_flags[1]) - assert constraint_path.exists() - with constraint_path.open() as constraint_file: - for line in constraint_file: - line = line.strip() - if len(line) == 0: - continue - if line.startswith("#"): - continue - try: - requirement = Requirement(line) - package = requirement.name - if ( - package not in packages - or requirement.url is not None - or requirement.marker is not None - or len(requirement.extras) != 0 - or len(requirement.specifier) != 1 - ): - continue - specifier = next(iter(requirement.specifier)) - if specifier.operator != "==": - continue - constraints_dict[package] = specifier.version - except InvalidRequirement: - continue - return constraints_dict - - -def virtualenv( - python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr] -) -> Dict[str, str]: - assert python.exists() - virtualenv_app = _ensure_virtualenv() - constraints = _parse_constraints_for_virtualenv(dependency_constraint_flags) - additional_flags = [f"--{package}={version}" for package, version in constraints.items()] - - # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual - # environment. The initial cost is a bit higher but reusing is much faster. - # Windows does not always allow symlinks so just disabling for now. - # Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the - # version of pip that will end-up installed. - # c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder - if ( - not IS_WIN - and constraints["pip"] != "embed" - and Version(constraints["pip"]) >= Version("19.3") - ): - additional_flags.append("--symlink-app-data") - - call( - sys.executable, - "-sS", # just the stdlib, https://github.com/pypa/virtualenv/issues/2133#issuecomment-1003710125 - virtualenv_app, - "--activators=", - "--no-periodic-update", - *additional_flags, - "--python", - python, - venv_path, - ) - if IS_WIN: - paths = [str(venv_path), str(venv_path / "Scripts")] - else: - paths = [str(venv_path / "bin")] - env = os.environ.copy() - env["PATH"] = os.pathsep.join(paths + [env["PATH"]]) - return env diff --git a/cibuildwheel/util/__init__.py b/cibuildwheel/util/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cibuildwheel/util/cmd.py b/cibuildwheel/util/cmd.py new file mode 100644 index 000000000..a528480e7 --- /dev/null +++ b/cibuildwheel/util/cmd.py @@ -0,0 +1,98 @@ +import os +import shlex +import shutil +import subprocess +import sys +import typing +from collections.abc import Iterator, Mapping +from typing import Final, Literal + +from ..errors import FatalError +from ..typing import PathOrStr + +_IS_WIN: Final[bool] = sys.platform.startswith("win") + + +@typing.overload +def call( + *args: PathOrStr, + env: Mapping[str, str] | None = None, + cwd: PathOrStr | None = None, + capture_stdout: Literal[False] = ..., +) -> None: ... + + +@typing.overload +def call( + *args: PathOrStr, + env: Mapping[str, str] | None = None, + cwd: PathOrStr | None = None, + capture_stdout: Literal[True], +) -> str: ... + + +def call( + *args: PathOrStr, + env: Mapping[str, str] | None = None, + cwd: PathOrStr | None = None, + capture_stdout: bool = False, +) -> str | None: + """ + Run subprocess.run, but print the commands first. Takes the commands as + *args. Uses shell=True on Windows due to a bug. Also converts to + Paths to strings, due to Windows behavior at least on older Pythons. + https://bugs.python.org/issue8557 + """ + args_ = [str(arg) for arg in args] + # print the command executing for the logs + print("+ " + " ".join(shlex.quote(a) for a in args_)) + # workaround platform behaviour differences outlined + # in https://github.com/python/cpython/issues/52803 + path_env = env if env is not None else os.environ + path = path_env.get("PATH", None) + executable = shutil.which(args_[0], path=path) + if executable is None: + msg = f"Couldn't find {args_[0]!r} in PATH {path!r}" + raise FatalError(msg) + args_[0] = executable + try: + result = subprocess.run( + args_, + check=True, + shell=_IS_WIN, + env=env, + cwd=cwd, + capture_output=capture_stdout, + text=capture_stdout, + ) + except subprocess.CalledProcessError as e: + if capture_stdout: + sys.stderr.write(e.stderr) + raise + if not capture_stdout: + return None + sys.stderr.write(result.stderr) + return typing.cast(str, result.stdout) + + +def shell( + *commands: str, env: Mapping[str, str] | None = None, cwd: PathOrStr | None = None +) -> None: + command = " ".join(commands) + print(f"+ {command}") + subprocess.run(command, env=env, cwd=cwd, shell=True, check=True) + + +def split_command(lst: list[str]) -> Iterator[list[str]]: + """ + Split a shell-style command, as returned by shlex.split, into a sequence + of commands, separated by '&&'. + """ + items = list[str]() + for item in lst: + if item == "&&": + yield items + items = [] + else: + items.append(item) + yield items diff --git a/cibuildwheel/util/file.py b/cibuildwheel/util/file.py new file mode 100644 index 000000000..c0fdc2c19 --- /dev/null +++ b/cibuildwheel/util/file.py @@ -0,0 +1,134 @@ +import os +import shutil +import ssl +import tarfile +import time +import urllib.request +from collections.abc import Callable +from pathlib import Path, PurePath +from typing import Final +from zipfile import ZipFile + +import certifi +from platformdirs import user_cache_path + +from ..errors import FatalError + +DEFAULT_CIBW_CACHE_PATH: Final[Path] = user_cache_path(appname="cibuildwheel", appauthor="pypa") +CIBW_CACHE_PATH: Final[Path] = Path( + os.environ.get("CIBW_CACHE_PATH", DEFAULT_CIBW_CACHE_PATH) +).resolve() + + +def download(url: str, dest: Path) -> None: + print(f"+ Download {url} to {dest}") + dest_dir = dest.parent + dest_dir.mkdir(parents=True, exist_ok=True) + + # we've had issues when relying on the host OS' CA certificates on Windows, + # so we use certifi (this sounds odd but requests also does this by default) + cafile = os.environ.get("SSL_CERT_FILE", certifi.where()) + context = ssl.create_default_context(cafile=cafile) + repeat_num = 3 + for i in range(repeat_num): + try: + with urllib.request.urlopen(url, context=context) as response: + dest.write_bytes(response.read()) + return + + except OSError: + if i == repeat_num - 1: + raise + time.sleep(3) + + +def extract_zip(zip_src: Path, dest: Path) -> None: + """Extracts a zip and correctly sets permissions on extracted files. + + Notes: + - sets permissions to the same values as they were set in the archive + - files with no clear permissions in `external_attr` will be extracted with default values + """ + with ZipFile(zip_src) as zip_: + for zinfo in zip_.filelist: + zip_.extract(zinfo, dest) + + # Set permissions to the same values as they were set in the archive + # We have to do this manually due to https://github.com/python/cpython/issues/59999 + permissions = (zinfo.external_attr >> 16) & 0o777 + if permissions != 0: + dest.joinpath(zinfo.filename).chmod(permissions) + + +def extract_tar(tar_src: Path, dest: Path) -> None: + """Extracts a tar file using the stdlib 'tar' filter. + + See: https://docs.python.org/3/library/tarfile.html#tarfile.tar_filter for filter details + """ + with tarfile.open(tar_src) as tar_: + tar_.extraction_filter = getattr(tarfile, "tar_filter", (lambda member, _: member)) + tar_.extractall(dest) + + +def move_file(src_file: Path, dst_file: Path) -> Path: + """Moves a file safely while avoiding potential semantic confusion: + + 1. `dst_file` must point to the target filename, not a directory + 2. `dst_file` will be overwritten if it already exists + 3. any missing parent directories will be created + + Returns the fully resolved Path of the resulting file. + + Raises: + NotADirectoryError: If any part of the intermediate path to `dst_file` is an existing file + IsADirectoryError: If `dst_file` points directly to an existing directory + """ + src_file = src_file.resolve(strict=True) + dst_file = dst_file.resolve() + + if dst_file.is_dir(): + msg = "dst_file must be a valid target filename, not an existing directory." + raise IsADirectoryError(msg) + dst_file.unlink(missing_ok=True) + dst_file.parent.mkdir(parents=True, exist_ok=True) + + # using shutil.move() as Path.rename() is not guaranteed to work across filesystem boundaries + # explicit str() needed for Python 3.8 + resulting_file = shutil.move(str(src_file), str(dst_file)) + return Path(resulting_file).resolve(strict=True) + + +def copy_into_local(src: Path, dst: PurePath) -> None: + """Copy a path from src to dst, regardless of whether it's a file or a directory.""" + # Ensure the target folder location exists + Path(dst.parent).mkdir(exist_ok=True, parents=True) + + if src.is_dir(): + shutil.copytree(src, dst) + else: + shutil.copy(src, dst) + + +def copy_test_sources( + test_sources: list[str], + project_dir: Path, + test_dir: PurePath, + copy_into: Callable[[Path, PurePath], None] = copy_into_local, +) -> None: + """Copy the list of test sources from the package to the test directory. + + :param test_sources: A list of test paths, relative to the project_dir. + :param project_dir: The root of the project. + :param test_dir: The folder where test sources should be placed. + :param copy_into: The copy function to use. By default, does a local + filesystem copy; but an OCIContainer.copy_info method (or equivalent) + can be provided. + """ + for test_path in test_sources: + source = project_dir.resolve() / test_path + + if not source.exists(): + msg = f"Test source {test_path} does not exist." + raise FatalError(msg) + + copy_into(source, test_dir / test_path) diff --git a/cibuildwheel/util/helpers.py b/cibuildwheel/util/helpers.py new file mode 100644 index 000000000..363fd3345 --- /dev/null +++ b/cibuildwheel/util/helpers.py @@ -0,0 +1,189 @@ +import itertools +import os +import re +import shlex +import textwrap +from collections import defaultdict +from collections.abc import Sequence +from functools import total_ordering + +from ..typing import PathOrStr + + +def format_safe(template: str, **kwargs: str | os.PathLike[str]) -> str: + """ + Works similarly to `template.format(**kwargs)`, except that unmatched + fields in `template` are passed through untouched. + + >>> format_safe('{a} {b}', a='123') + '123 {b}' + >>> format_safe('{a} {b[4]:3f}', a='123') + '123 {b[4]:3f}' + + To avoid variable expansion, precede with a single backslash e.g. + >>> format_safe('\\{a} {b}', a='123') + '{a} {b}' + """ + + result = template + + for key, value in kwargs.items(): + find_pattern = re.compile( + rf""" + (? str: + """ + Preprocesses a command by expanding variables like {project}. + + For example, used in the test_command option to specify the path to the + project's root. Unmatched syntax will mostly be allowed through. + """ + return format_safe(command, **kwargs) + + +def strtobool(val: str) -> bool: + return val.lower() in {"y", "yes", "t", "true", "on", "1"} + + +def unwrap(text: str) -> str: + """ + Unwraps multi-line text to a single line + """ + # remove initial line indent + text = textwrap.dedent(text) + # remove leading/trailing whitespace + text = text.strip() + # remove consecutive whitespace + return re.sub(r"\s+", " ", text) + + +def unwrap_preserving_paragraphs(text: str) -> str: + """ + Unwraps multi-line text to a single line, but preserves paragraphs + """ + # remove initial line indent + text = textwrap.dedent(text) + # remove leading/trailing whitespace + text = text.strip() + + paragraphs = text.split("\n\n") + # remove consecutive whitespace + paragraphs = [re.sub(r"\s+", " ", paragraph) for paragraph in paragraphs] + return "\n\n".join(paragraphs) + + +def parse_key_value_string( + key_value_string: str, + positional_arg_names: Sequence[str] | None = None, + kw_arg_names: Sequence[str] | None = None, +) -> dict[str, list[str]]: + """ + Parses a string like "docker; create_args: --some-option=value another-option" + """ + if positional_arg_names is None: + positional_arg_names = [] + if kw_arg_names is None: + kw_arg_names = [] + + all_field_names = [*positional_arg_names, *kw_arg_names] + + shlexer = shlex.shlex(key_value_string, posix=True, punctuation_chars=";") + shlexer.commenters = "" + shlexer.whitespace_split = True + parts = list(shlexer) + # parts now looks like + # ['docker', ';', 'create_args:', '--some-option=value', 'another-option'] + + # split by semicolon + fields = [list(group) for k, group in itertools.groupby(parts, lambda x: x == ";") if not k] + + result: defaultdict[str, list[str]] = defaultdict(list) + for field_i, field in enumerate(fields): + # check to see if the option name is specified + field_name, sep, first_value = field[0].partition(":") + if sep: + if field_name not in all_field_names: + msg = f"Failed to parse {key_value_string!r}. Unknown field name {field_name!r}" + raise ValueError(msg) + + values = ([first_value] if first_value else []) + field[1:] + else: + try: + field_name = positional_arg_names[field_i] + except IndexError: + msg = f"Failed to parse {key_value_string!r}. Too many positional arguments - expected a maximum of {len(positional_arg_names)}" + raise ValueError(msg) from None + + values = field + + result[field_name] += values + + return dict(result) + + +@total_ordering +class FlexibleVersion: + version_str: str + version_parts: tuple[int, ...] + suffix: str + + def __init__(self, version_str: str) -> None: + self.version_str = version_str + + # Split into numeric parts and the optional suffix + match = re.match(r"^[v]?(\d+(\.\d+)*)(.*)$", version_str) + if not match: + msg = f"Invalid version string: {version_str}" + raise ValueError(msg) + + version_part, _, suffix = match.groups() + + # Convert numeric version part into a tuple of integers + self.version_parts = tuple(map(int, version_part.split("."))) + self.suffix = suffix.strip() if suffix else "" + + # Normalize by removing trailing zeros + self.version_parts = self._remove_trailing_zeros(self.version_parts) + + @staticmethod + def _remove_trailing_zeros(parts: tuple[int, ...]) -> tuple[int, ...]: + # Remove trailing zeros for accurate comparisons + # without this, "3.0" would be considered greater than "3" + while parts and parts[-1] == 0: + parts = parts[:-1] + return parts + + def __eq__(self, other: object) -> bool: + if not isinstance(other, FlexibleVersion): + raise NotImplementedError() + return (self.version_parts, self.suffix) == (other.version_parts, other.suffix) + + def __lt__(self, other: object) -> bool: + if not isinstance(other, FlexibleVersion): + raise NotImplementedError() + return (self.version_parts, self.suffix) < (other.version_parts, other.suffix) + + def __repr__(self) -> str: + return f"FlexibleVersion('{self.version_str}')" + + def __str__(self) -> str: + return self.version_str diff --git a/cibuildwheel/util/packaging.py b/cibuildwheel/util/packaging.py new file mode 100644 index 000000000..295bc091c --- /dev/null +++ b/cibuildwheel/util/packaging.py @@ -0,0 +1,207 @@ +import shlex +from collections.abc import Mapping, MutableMapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path, PurePath +from typing import Any, Literal, Self, TypeVar + +from packaging.utils import parse_wheel_filename + +from . import resources +from .cmd import call +from .helpers import parse_key_value_string, unwrap + + +@dataclass(kw_only=True) +class DependencyConstraints: + base_file_path: Path | None = None + packages: list[str] = field(default_factory=list) + + def __post_init__(self) -> None: + if self.packages and self.base_file_path is not None: + msg = "Cannot specify both a file and packages in the dependency constraints" + raise ValueError(msg) + + if self.base_file_path is not None: + if not self.base_file_path.exists(): + msg = f"Dependency constraints file not found: {self.base_file_path}" + raise FileNotFoundError(msg) + self.base_file_path = self.base_file_path.resolve() + + @classmethod + def pinned(cls) -> Self: + return cls(base_file_path=resources.CONSTRAINTS) + + @classmethod + def latest(cls) -> Self: + return cls() + + @classmethod + def from_config_string(cls, config_string: str) -> Self: + if config_string == "pinned": + return cls.pinned() + + if config_string == "latest" or not config_string: + return cls.latest() + + if config_string.startswith(("file:", "packages:")): + # we only do the table-style parsing if it looks like a table, + # because this option used to be only a file path. We don't want + # to break existing configurations, whose file paths might include + # special characters like ':' or ' ', which would require quoting + # if they were to be passed as a parse_key_value_string positional + # argument. + return cls.from_table_style_config_string(config_string) + + return cls(base_file_path=Path(config_string)) + + @classmethod + def from_table_style_config_string(cls, config_string: str) -> Self: + config_dict = parse_key_value_string(config_string, kw_arg_names=["file", "packages"]) + files = config_dict.get("file") + packages = config_dict.get("packages") or [] + + if files and packages: + msg = "Cannot specify both a file and packages in dependency-versions" + raise ValueError(msg) + + if files: + if len(files) > 1: + msg = unwrap(""" + Only one file can be specified in dependency-versions. + If you intended to pass only one, perhaps you need to quote the path? + """) + raise ValueError(msg) + + return cls(base_file_path=Path(files[0])) + + return cls(packages=packages) + + def get_for_python_version( + self, *, version: str, variant: Literal["python", "pyodide"] = "python", tmp_dir: Path + ) -> Path | None: + if self.packages: + constraint_file = tmp_dir / "constraints.txt" + constraint_file.write_text("\n".join(self.packages)) + return constraint_file + + if self.base_file_path is not None: + version_parts = version.split(".") + + # try to find a version-specific dependency file e.g. if + # ./constraints.txt is the base, look for ./constraints-python36.txt + specific_stem = ( + self.base_file_path.stem + f"-{variant}{version_parts[0]}{version_parts[1]}" + ) + specific_name = specific_stem + self.base_file_path.suffix + specific_file_path = self.base_file_path.with_name(specific_name) + + if specific_file_path.exists(): + return specific_file_path + else: + return self.base_file_path + + return None + + def options_summary(self) -> Any: + if self == DependencyConstraints.pinned(): + return "pinned" + elif self.packages: + return {"packages": " ".join(shlex.quote(p) for p in self.packages)} + elif self.base_file_path is not None: + return self.base_file_path.name + else: + return "latest" + + +def get_pip_version(env: Mapping[str, str]) -> str: + versions_output_text = call( + "python", "-m", "pip", "freeze", "--all", capture_stdout=True, env=env + ) + (pip_version,) = ( + version[5:] + for version in versions_output_text.strip().splitlines() + if version.startswith("pip==") + ) + return pip_version + + +T = TypeVar("T", bound=PurePath) + + +def find_compatible_wheel(wheels: Sequence[T], identifier: str) -> T | None: + """ + Finds a wheel with an abi3 or a none ABI tag in `wheels` compatible with the Python interpreter + specified by `identifier` that is previously built. + """ + + interpreter, platform = identifier.split("-", 1) + interpreter = interpreter.split("_")[0] + free_threaded = interpreter.endswith("t") + if free_threaded: + interpreter = interpreter[:-1] + for wheel in wheels: + _, _, _, tags = parse_wheel_filename(wheel.name) + for tag in tags: + if tag.abi == "abi3" and not free_threaded: + # ABI3 wheels must start with cp3 for impl and tag + if not (interpreter.startswith("cp3") and tag.interpreter.startswith("cp3")): + continue + elif tag.abi == "none": + # CPythonless wheels must include py3 tag + if tag.interpreter[:3] != "py3": + continue + else: + # Other types of wheels are not detected, this is looking for previously built wheels. + continue + + if tag.interpreter != "py3" and int(tag.interpreter[3:]) > int(interpreter[3:]): + # If a minor version number is given, it has to be lower than the current one. + continue + + if platform.startswith(("manylinux", "musllinux", "macosx", "ios")): + # Linux, macOS, and iOS require the beginning and ending match + # (macos/manylinux/iOS version number doesn't need to match) + os_, arch = platform.split("_", 1) + if not tag.platform.startswith(os_): + continue + if not tag.platform.endswith(f"_{arch}"): + continue + elif platform.startswith("pyodide"): + # each Pyodide version has its own platform tag + continue + else: + # Windows should exactly match + if tag.platform != platform: + continue + + # If all the filters above pass, then the wheel is a previously built compatible wheel. + return wheel + + return None + + +def combine_constraints( + env: MutableMapping[str, str], /, constraints_path: Path, tmp_dir: Path | None +) -> None: + """ + This will workaround a bug in pip<=21.1.1 or uv<=0.2.0 if a tmp_dir is given. + If set to None, this will use the modern URI method. + """ + + if tmp_dir: + if " " in str(constraints_path): + assert " " not in str(tmp_dir) + tmp_file = tmp_dir / "constraints.txt" + tmp_file.write_bytes(constraints_path.read_bytes()) + constraints_path = tmp_file + our_constraints = str(constraints_path) + else: + our_constraints = ( + constraints_path.as_uri() if " " in str(constraints_path) else str(constraints_path) + ) + + user_constraints = env.get("PIP_CONSTRAINT") + + env["UV_CONSTRAINT"] = env["PIP_CONSTRAINT"] = " ".join( + c for c in [our_constraints, user_constraints] if c + ) diff --git a/cibuildwheel/util/python_build_standalone.py b/cibuildwheel/util/python_build_standalone.py new file mode 100644 index 000000000..95721ee54 --- /dev/null +++ b/cibuildwheel/util/python_build_standalone.py @@ -0,0 +1,181 @@ +import fnmatch +import functools +import json +import platform +import typing +from pathlib import Path + +from filelock import FileLock + +from cibuildwheel.util.file import download, extract_tar +from cibuildwheel.util.resources import PYTHON_BUILD_STANDALONE_RELEASES + + +class PythonBuildStandaloneAsset(typing.TypedDict): + name: str + url: str + + +class PythonBuildStandaloneRelease(typing.TypedDict): + tag: str + assets: list[PythonBuildStandaloneAsset] + + +class PythonBuildStandaloneReleaseData(typing.TypedDict): + releases: list[PythonBuildStandaloneRelease] + + +@functools.cache +def get_python_build_standalone_release_data() -> PythonBuildStandaloneReleaseData: + with open(PYTHON_BUILD_STANDALONE_RELEASES, "rb") as f: + return typing.cast(PythonBuildStandaloneReleaseData, json.load(f)) + + +class PythonBuildStandaloneError(Exception): + """Errors related to python-build-standalone.""" + + +def _get_platform_identifiers() -> tuple[str, str, str | None]: + """ + Detects the current platform and returns architecture, platform, and libc + identifiers. + """ + system = platform.system() + machine = platform.machine() + machine_lower = machine.lower() + + arch_identifier: str + platform_identifier: str + libc_identifier: str | None = None + + # Map Architecture + if machine_lower in ["x86_64", "amd64"]: + arch_identifier = "x86_64" + elif machine_lower in ["aarch64", "arm64"]: + arch_identifier = "aarch64" + else: + msg = f"Unsupported architecture: {system} {machine}. Cannot download appropriate Python build." + raise PythonBuildStandaloneError(msg) + + # Map OS + Libc + if system == "Linux": + platform_identifier = "unknown-linux" + libc_identifier = "musl" if "musl" in (platform.libc_ver() or ("", "")) else "gnu" + elif system == "Darwin": + platform_identifier = "apple-darwin" + elif system == "Windows": + platform_identifier = "pc-windows-msvc" + else: + msg = f"Unsupported operating system: {system}. Cannot download appropriate Python build." + raise PythonBuildStandaloneError(msg) + + print( + f"Detected platform: arch='{arch_identifier}', platform='{platform_identifier}', libc='{libc_identifier}'" + ) + return arch_identifier, platform_identifier, libc_identifier + + +def _get_pbs_asset( + *, + python_version: str, + arch_identifier: str, + platform_identifier: str, + libc_identifier: str | None, +) -> tuple[str, str, str]: + """Finds the asset, returning (tag, filename, url).""" + release_data = get_python_build_standalone_release_data() + + expected_suffix = f"{arch_identifier}-{platform_identifier}" + if libc_identifier: + expected_suffix += f"-{libc_identifier}" + expected_suffix += "-install_only.tar.gz" + + asset_pattern = f"cpython-{python_version}.*-{expected_suffix}" + print(f"Looking for file with pattern {asset_pattern}") + + for release in release_data["releases"]: + for asset in release["assets"]: + asset_name = asset["name"] + if not fnmatch.fnmatch(asset_name, asset_pattern): + continue + + asset_url = asset["url"] + return release["tag"], asset_url, asset_name + + # If loop completes without finding a match + msg = f"Could not find python-build-standalone release asset matching {asset_pattern!r}." + raise PythonBuildStandaloneError(msg) + + +def _download_or_get_from_cache(asset_url: str, asset_filename: str, cache_dir: Path) -> Path: + with FileLock(cache_dir / (asset_filename + ".lock")): + asset_cache_path = cache_dir / asset_filename + if asset_cache_path.is_file(): + print(f"Using cached python_build_standalone: {asset_cache_path}") + return asset_cache_path + + print(f"Downloading python_build_standalone: {asset_url} to {asset_cache_path}") + download(asset_url, asset_cache_path) + return asset_cache_path + + +def _find_python_executable(extracted_dir: Path) -> Path: + """Finds the python executable within the extracted directory structure.""" + # Structure is typically 'python/bin/python' or 'python/python.exe' + base_install_dir = extracted_dir / "python" + + if platform.system() == "Windows": + executable_path = base_install_dir / "python.exe" + else: + executable_path = base_install_dir / "bin" / "python" + + if not executable_path.is_file(): + msg = f"Could not locate python executable at expected path {executable_path} within {extracted_dir}." + raise PythonBuildStandaloneError(msg) + + print(f"Found python executable: {executable_path}") + return executable_path.resolve() # Return absolute path + + +def create_python_build_standalone_environment( + python_version: str, temp_dir: Path, cache_dir: Path +) -> Path: + """ + Returns a Python environment from python-build-standalone, downloading it + if necessary using a cache, and expanding it into a fresh base path. + + Args: + python_version: The Python version string (e.g., "3.12"). + temp_dir: A directory where the Python environment will be created. + cache_dir: A directory to store/retrieve downloaded archives. + + Returns: + The absolute path to the python executable within the created environment (in temp_dir). + + Raises: + PythonBuildStandaloneError: If the platform is unsupported, the build cannot be found, + download/extraction fails, or configuration is invalid. + """ + + print(f"Creating python-build-standalone environment: version={python_version}") + + arch_id, platform_id, libc_id = _get_platform_identifiers() + + pbs_tag, asset_url, asset_filename = _get_pbs_asset( + python_version=python_version, + arch_identifier=arch_id, + platform_identifier=platform_id, + libc_identifier=libc_id, + ) + + print(f"Using python-build-standalone release: {pbs_tag}") + + archive_path = _download_or_get_from_cache( + asset_url=asset_url, asset_filename=asset_filename, cache_dir=cache_dir + ) + + python_base_dir = temp_dir / "pbs" + assert not python_base_dir.exists() + extract_tar(archive_path, python_base_dir) + + return _find_python_executable(python_base_dir) diff --git a/cibuildwheel/util/resources.py b/cibuildwheel/util/resources.py new file mode 100644 index 000000000..0cb09a26d --- /dev/null +++ b/cibuildwheel/util/resources.py @@ -0,0 +1,29 @@ +import functools +import tomllib +from pathlib import Path +from typing import Final + +from ..typing import PlatformName + +PATH: Final[Path] = Path(__file__).parent.parent / "resources" +INSTALL_CERTIFI_SCRIPT: Final[Path] = PATH / "install_certifi.py" +FREE_THREAD_ENABLE_313: Final[Path] = PATH / "free-threaded-enable-313.xml" +FREE_THREAD_ENABLE_314: Final[Path] = PATH / "free-threaded-enable-314.xml" +NODEJS: Final[Path] = PATH / "nodejs.toml" +DEFAULTS: Final[Path] = PATH / "defaults.toml" +PINNED_DOCKER_IMAGES: Final[Path] = PATH / "pinned_docker_images.cfg" +BUILD_PLATFORMS: Final[Path] = PATH / "build-platforms.toml" +CONSTRAINTS: Final[Path] = PATH / "constraints.txt" +VIRTUALENV: Final[Path] = PATH / "virtualenv.toml" +CIBUILDWHEEL_SCHEMA: Final[Path] = PATH / "cibuildwheel.schema.json" +PYTHON_BUILD_STANDALONE_RELEASES: Final[Path] = PATH / "python-build-standalone-releases.json" +TEST_FAIL_CWD_FILE: Final[Path] = PATH / "testing_temp_dir_file.py" + + +# this value is cached because it's used a lot in unit tests +@functools.cache +def read_python_configs(config: PlatformName) -> list[dict[str, str]]: + with BUILD_PLATFORMS.open("rb") as f: + loaded_file = tomllib.load(f) + results: list[dict[str, str]] = list(loaded_file[config]["python_configurations"]) + return results diff --git a/cibuildwheel/venv.py b/cibuildwheel/venv.py new file mode 100644 index 000000000..567ba4001 --- /dev/null +++ b/cibuildwheel/venv.py @@ -0,0 +1,166 @@ +import contextlib +import functools +import os +import shutil +import sys +import tomllib +from collections.abc import Sequence +from pathlib import Path +from typing import Final + +from filelock import FileLock +from packaging.requirements import InvalidRequirement, Requirement +from packaging.version import Version + +from .util import resources +from .util.cmd import call +from .util.file import CIBW_CACHE_PATH, download + +_IS_WIN: Final[bool] = sys.platform.startswith("win") + + +@functools.cache +def _ensure_virtualenv(version: str) -> tuple[Path, Version]: + version_parts = version.split(".") + key = f"py{version_parts[0]}{version_parts[1]}" + with resources.VIRTUALENV.open("rb") as f: + loaded_file = tomllib.load(f) + configuration = loaded_file.get(key, loaded_file["default"]) + version = str(configuration["version"]) + url = str(configuration["url"]) + path = CIBW_CACHE_PATH / f"virtualenv-{version}.pyz" + with FileLock(str(path) + ".lock"): + if not path.exists(): + download(url, path) + return (path, Version(version)) + + +def constraint_flags( + dependency_constraint: Path | None, +) -> Sequence[str]: + """ + Returns the flags to pass to pip for the given dependency constraint. + """ + + return ["-c", dependency_constraint.as_uri()] if dependency_constraint else [] + + +def _parse_pip_constraint_for_virtualenv( + constraint_path: Path | None, +) -> str: + """ + Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where + the key is the package name, and the value is the constraint version. + If a package version cannot be found, its value is "embed" meaning that virtualenv will install + its bundled version, already available locally. + The function does not try to be too smart and just handles basic constraints. + If it can't get an exact version, the real constraint will be handled by the + {macos|windows}.setup_python function. + """ + if constraint_path: + assert constraint_path.exists() + with constraint_path.open(encoding="utf-8") as constraint_file: + for line_ in constraint_file: + line = line_.strip() + if not line: + continue + if line.startswith("#"): + continue + try: + requirement = Requirement(line) + package = requirement.name + if ( + package != "pip" + or requirement.url is not None + or requirement.marker is not None + or len(requirement.extras) != 0 + or len(requirement.specifier) != 1 + ): + continue + specifier = next(iter(requirement.specifier)) + if specifier.operator != "==": + continue + return specifier.version + except InvalidRequirement: + continue + return "embed" + + +def virtualenv( + version: str, + python: Path, + venv_path: Path, + dependency_constraint: Path | None, + *, + use_uv: bool, + env: dict[str, str] | None = None, + pip_version: str | None = None, +) -> dict[str, str]: + """ + Create a virtual environment. If `use_uv` is True, + dependency_constraint_flags are ignored since nothing is installed in the + venv. Otherwise, pip is installed. + """ + + # virtualenv may fail if this is a symlink. + python = python.resolve() + + assert python.exists() + + if use_uv: + call("uv", "venv", venv_path, "--python", python) + else: + virtualenv_app, virtualenv_version = _ensure_virtualenv(version) + if pip_version is None: + pip_version = _parse_pip_constraint_for_virtualenv(dependency_constraint) + additional_flags = [f"--pip={pip_version}", "--no-setuptools"] + if virtualenv_version < Version("20.31") or Version(version) < Version("3.9"): + additional_flags.append("--no-wheel") + + # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual + # environment. The initial cost is a bit higher but reusing is much faster. + # Windows does not always allow symlinks so just disabling for now. + # Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the + # version of pip that will end-up installed. + # c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder + if not _IS_WIN and pip_version != "embed" and Version(pip_version) >= Version("19.3"): + additional_flags.append("--symlink-app-data") + + call( + sys.executable, + "-sS", # just the stdlib, https://github.com/pypa/virtualenv/issues/2133#issuecomment-1003710125 + virtualenv_app, + "--activators=", + "--no-periodic-update", + *additional_flags, + "--python", + python, + venv_path, + ) + paths = [str(venv_path), str(venv_path / "Scripts")] if _IS_WIN else [str(venv_path / "bin")] + venv_env = os.environ.copy() if env is None else env.copy() + venv_env["PATH"] = os.pathsep.join([*paths, venv_env["PATH"]]) + venv_env["VIRTUAL_ENV"] = str(venv_path) + if not use_uv and pip_version == "embed": + call( + "pip", + "install", + "--upgrade", + "pip", + *constraint_flags(dependency_constraint), + env=venv_env, + cwd=venv_path, + ) + return venv_env + + +def find_uv() -> Path | None: + # Prefer uv in our environment + with contextlib.suppress(ImportError, FileNotFoundError): + # pylint: disable-next=import-outside-toplevel + from uv import find_uv_bin + + return Path(find_uv_bin()) + + uv_on_path = shutil.which("uv") + return Path(uv_on_path) if uv_on_path else None diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py deleted file mode 100644 index 86d01fa82..000000000 --- a/cibuildwheel/windows.py +++ /dev/null @@ -1,417 +0,0 @@ -import os -import shutil -import subprocess -import sys -from functools import lru_cache -from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence, Set -from zipfile import ZipFile - -from filelock import FileLock -from packaging.version import Version - -from .architecture import Architecture -from .environment import ParsedEnvironment -from .logger import log -from .options import Options -from .typing import PathOrStr, assert_never -from .util import ( - CIBW_CACHE_PATH, - BuildFrontend, - BuildSelector, - NonPlatformWheelError, - call, - download, - get_build_verbosity_extra_flags, - get_pip_version, - prepare_command, - read_python_configs, - shell, - virtualenv, -) - - -def get_nuget_args(version: str, arch: str, output_directory: Path) -> List[str]: - platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"} - python_name = "python" + platform_suffix[arch] - return [ - python_name, - "-Version", - version, - "-FallbackSource", - "https://api.nuget.org/v3/index.json", - "-OutputDirectory", - str(output_directory), - ] - - -class PythonConfiguration(NamedTuple): - version: str - arch: str - identifier: str - url: Optional[str] = None - - -def get_python_configurations( - build_selector: BuildSelector, - architectures: Set[Architecture], -) -> List[PythonConfiguration]: - - full_python_configs = read_python_configs("windows") - - python_configurations = [PythonConfiguration(**item) for item in full_python_configs] - - map_arch = {"32": Architecture.x86, "64": Architecture.AMD64, "ARM64": Architecture.ARM64} - - # skip builds as required - python_configurations = [ - c - for c in python_configurations - if build_selector(c.identifier) and map_arch[c.arch] in architectures - ] - - return python_configurations - - -def extract_zip(zip_src: Path, dest: Path) -> None: - with ZipFile(zip_src) as zip_: - zip_.extractall(dest) - - -@lru_cache(maxsize=None) -def _ensure_nuget() -> Path: - nuget = CIBW_CACHE_PATH / "nuget.exe" - with FileLock(str(nuget) + ".lock"): - if not nuget.exists(): - download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget) - return nuget - - -def install_cpython(version: str, arch: str) -> Path: - base_output_dir = CIBW_CACHE_PATH / "nuget-cpython" - nuget_args = get_nuget_args(version, arch, base_output_dir) - installation_path = base_output_dir / (nuget_args[0] + "." + version) / "tools" - with FileLock(str(base_output_dir) + f"-{version}-{arch}.lock"): - if not installation_path.exists(): - nuget = _ensure_nuget() - call(nuget, "install", *nuget_args) - return installation_path / "python.exe" - - -def install_pypy(tmp: Path, arch: str, url: str) -> Path: - assert arch == "64" and "win64" in url - # Inside the PyPy zip file is a directory with the same name - zip_filename = url.rsplit("/", 1)[-1] - extension = ".zip" - assert zip_filename.endswith(extension) - installation_path = CIBW_CACHE_PATH / zip_filename[: -len(extension)] - with FileLock(str(installation_path) + ".lock"): - if not installation_path.exists(): - pypy_zip = tmp / zip_filename - download(url, pypy_zip) - # Extract to the parent directory because the zip file still contains a directory - extract_zip(pypy_zip, installation_path.parent) - return installation_path / "python.exe" - - -def setup_python( - tmp: Path, - python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[PathOrStr], - environment: ParsedEnvironment, - build_frontend: BuildFrontend, -) -> Dict[str, str]: - tmp.mkdir() - implementation_id = python_configuration.identifier.split("-")[0] - log.step(f"Installing Python {implementation_id}...") - if implementation_id.startswith("cp"): - base_python = install_cpython(python_configuration.version, python_configuration.arch) - elif implementation_id.startswith("pp"): - assert python_configuration.url is not None - base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url) - else: - raise ValueError("Unknown Python implementation") - assert base_python.exists() - - log.step("Setting up build environment...") - venv_path = tmp / "venv" - env = virtualenv(base_python, venv_path, dependency_constraint_flags) - - # set up environment variables for run_with_env - env["PYTHON_VERSION"] = python_configuration.version - env["PYTHON_ARCH"] = python_configuration.arch - env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - - # pip older than 21.3 builds executables such as pip.exe for x64 platform. - # The first re-install of pip updates pip module but builds pip.exe using - # the old pip which still generates x64 executable. But the second - # re-install uses updated pip and correctly builds pip.exe for the target. - # This can be removed once ARM64 Pythons (currently 3.9 and 3.10) bundle - # pip versions newer than 21.3. - if python_configuration.arch == "ARM64" and Version(get_pip_version(env)) < Version("21.3"): - call( - "python", - "-m", - "pip", - "install", - "--force-reinstall", - "--upgrade", - "pip", - *dependency_constraint_flags, - env=env, - cwd=venv_path, - ) - - # upgrade pip to the version matching our constraints - # if necessary, reinstall it to ensure that it's available on PATH as 'pip.exe' - call( - "python", - "-m", - "pip", - "install", - "--upgrade", - "pip", - *dependency_constraint_flags, - env=env, - cwd=venv_path, - ) - - # update env with results from CIBW_ENVIRONMENT - env = environment.as_dictionary(prev_environment=env) - - # check what Python version we're on - call("where", "python", env=env) - call("python", "--version", env=env) - call("python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\"", env=env) - where_python = call("where", "python", env=env, capture_stdout=True).splitlines()[0].strip() - if where_python != str(venv_path / "Scripts" / "python.exe"): - print( - "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", - file=sys.stderr, - ) - sys.exit(1) - - # check what pip version we're on - assert (venv_path / "Scripts" / "pip.exe").exists() - where_pip = call("where", "pip", env=env, capture_stdout=True).splitlines()[0].strip() - if where_pip.strip() != str(venv_path / "Scripts" / "pip.exe"): - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) - - call("pip", "--version", env=env) - - log.step("Installing build tools...") - if build_frontend == "pip": - call( - "pip", - "install", - "--upgrade", - "setuptools", - "wheel", - *dependency_constraint_flags, - env=env, - ) - elif build_frontend == "build": - call( - "pip", - "install", - "--upgrade", - "build[virtualenv]", - *dependency_constraint_flags, - env=env, - ) - else: - assert_never(build_frontend) - - return env - - -def build(options: Options, tmp_path: Path) -> None: - python_configurations = get_python_configurations( - options.globals.build_selector, options.globals.architectures - ) - - try: - before_all_options_identifier = python_configurations[0].identifier - before_all_options = options.build_options(before_all_options_identifier) - - if before_all_options.before_all: - log.step("Running before_all...") - env = before_all_options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command( - before_all_options.before_all, project=".", package=options.globals.package_dir - ) - shell(before_all_prepared, env=env) - - for config in python_configurations: - build_options = options.build_options(config.identifier) - log.build_start(config.identifier) - - identifier_tmp_dir = tmp_path / config.identifier - identifier_tmp_dir.mkdir() - built_wheel_dir = identifier_tmp_dir / "built_wheel" - repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" - - dependency_constraint_flags: Sequence[PathOrStr] = [] - if build_options.dependency_constraints: - dependency_constraint_flags = [ - "-c", - build_options.dependency_constraints.get_for_python_version(config.version), - ] - - # install Python - env = setup_python( - identifier_tmp_dir / "build", - config, - dependency_constraint_flags, - build_options.environment, - build_options.build_frontend, - ) - - # run the before_build command - if build_options.before_build: - log.step("Running before_build...") - before_build_prepared = prepare_command( - build_options.before_build, project=".", package=options.globals.package_dir - ) - shell(before_build_prepared, env=env) - - log.step("Building wheel...") - built_wheel_dir.mkdir() - - verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) - - if build_options.build_frontend == "pip": - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/pypa/cibuildwheel/pull/369 - call( - "python", - "-m", - "pip", - "wheel", - options.globals.package_dir.resolve(), - f"--wheel-dir={built_wheel_dir}", - "--no-deps", - *get_build_verbosity_extra_flags(build_options.build_verbosity), - env=env, - ) - elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) - build_env = env.copy() - if build_options.dependency_constraints: - constraints_path = build_options.dependency_constraints.get_for_python_version( - config.version - ) - # Bug in pip <= 21.1.3 - we can't have a space in the - # constraints file, and pip doesn't support drive letters - # in uhi. After probably pip 21.2, we can use uri. For - # now, use a temporary file. - if " " in str(constraints_path): - assert " " not in str(identifier_tmp_dir) - tmp_file = identifier_tmp_dir / "constraints.txt" - tmp_file.write_bytes(constraints_path.read_bytes()) - constraints_path = tmp_file - - build_env["PIP_CONSTRAINT"] = str(constraints_path) - build_env["VIRTUALENV_PIP"] = get_pip_version(env) - call( - "python", - "-m", - "build", - build_options.package_dir, - "--wheel", - f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", - env=build_env, - ) - else: - assert_never(build_options.build_frontend) - - built_wheel = next(built_wheel_dir.glob("*.whl")) - - # repair the wheel - repaired_wheel_dir.mkdir() - - if built_wheel.name.endswith("none-any.whl"): - raise NonPlatformWheelError() - - if build_options.repair_command: - log.step("Repairing wheel...") - repair_command_prepared = prepare_command( - build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir - ) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) - - repaired_wheel = next(repaired_wheel_dir.glob("*.whl")) - - if build_options.test_command and options.globals.test_selector(config.identifier): - log.step("Testing wheel...") - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) - venv_dir = identifier_tmp_dir / "venv-test" - - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call("python", "-m", "virtualenv", "--no-download", venv_dir, env=env) - - virtualenv_env = env.copy() - virtualenv_env["PATH"] = os.pathsep.join( - [ - str(venv_dir / "Scripts"), - virtualenv_env["PATH"], - ] - ) - - # check that we are using the Python from the virtual environment - call("where", "python", env=virtualenv_env) - - if build_options.before_test: - before_test_prepared = prepare_command( - build_options.before_test, - project=".", - package=build_options.package_dir, - ) - shell(before_test_prepared, env=virtualenv_env) - - # install the wheel - call( - "pip", - "install", - str(repaired_wheel) + build_options.test_extras, - env=virtualenv_env, - ) - - # test the wheel - if build_options.test_requires: - call("pip", "install", *build_options.test_requires, env=virtualenv_env) - - # run the tests from c:\, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - build_options.test_command, - project=Path(".").resolve(), - package=options.globals.package_dir.resolve(), - ) - shell(test_command_prepared, cwd="c:\\", env=virtualenv_env) - - # we're all done here; move it to output (remove if already exists) - shutil.move(str(repaired_wheel), build_options.output_dir) - - # clean up - # (we ignore errors because occasionally Windows fails to unlink a file and we - # don't want to abort a build because of that) - shutil.rmtree(identifier_tmp_dir, ignore_errors=True) - - log.build_end() - except subprocess.CalledProcessError as error: - log.step_end_with_error( - f"Command {error.cmd} failed with code {error.returncode}. {error.stdout}" - ) - sys.exit(1) diff --git a/docs/changelog.md b/docs/changelog.md index 2ad1d4fe2..06356a4f6 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,508 @@ title: Changelog --- +# Changelog + +### v3.0.0 + +_11 June 2025_ + +See @henryiii's [release post](https://iscinumpy.dev/post/cibuildwheel-3-0-0/) for more info on new features! + +- 🌟 Adds the ability to [build wheels for iOS](https://cibuildwheel.pypa.io/en/stable/platforms/#ios)! Set the [`platform` option](https://cibuildwheel.pypa.io/en/stable/options/#platform) to `ios` on a Mac with the iOS toolchain to try it out! (#2286, #2363, #2432) +- 🌟 Adds support for the GraalPy interpreter! Enable for your project using the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable). (#1538, #2411, #2414) +- ✨ Adds CPython 3.14 support, under the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable) `cpython-prerelease`. This version of cibuildwheel uses 3.14.0b2. (#2390) + + _While CPython is in beta, the ABI can change, so your wheels might not be compatible with the final release. For this reason, we don't recommend distributing wheels until RC1, at which point 3.14 will be available in cibuildwheel without the flag._ (#2390) +- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), and changes the working directory for tests. (#2062, #2284, #2437) + + - If this option is set, cibuildwheel will copy the files and folders specified in `test-sources` into the temporary directory we run from. This is required for iOS builds, but also useful for other platforms, as it allows you to avoid placeholders. + - If this option is not set, behaviour matches v2.x - cibuildwheel will run the tests from a temporary directory, and you can use the `{project}` placeholder in the `test-command` to refer to the project directory. (#2420) + +- ✨ Adds [`dependency-versions`](https://cibuildwheel.pypa.io/en/stable/options/#dependency-versions) inline syntax (#2122) +- ✨ Improves support for Pyodide builds and adds the experimental [`pyodide-version`](https://cibuildwheel.pypa.io/en/stable/options/#pyodide-version) option, which allows you to specify the version of Pyodide to use for builds. (#2002) +- ✨ Add `pyodide-prerelease` [enable](https://cibuildwheel.pypa.io/en/stable/options/#enable) option, with an early build of 0.28 (Python 3.13). (#2431) +- ✨ Adds the [`test-environment`](https://cibuildwheel.pypa.io/en/stable/options/#test-environment) option, which allows you to set environment variables for the test command. (#2388) +- ✨ Adds the [`xbuild-tools`](https://cibuildwheel.pypa.io/en/stable/options/#xbuild-tools) option, which allows you to specify tools safe for cross-compilation. Currently only used on iOS; will be useful for Android in the future. (#2317) +- 🛠 The default [manylinux image](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) has changed from `manylinux2014` to `manylinux_2_28`. (#2330) +- 🛠 EOL images `manylinux1`, `manylinux2010`, `manylinux_2_24` and `musllinux_1_1` can no longer be specified by their shortname. The full OCI name can still be used for these images, if you wish. (#2316) +- 🛠 Invokes `build` rather than `pip wheel` to build wheels by default. You can control this via the [`build-frontend`](https://cibuildwheel.pypa.io/en/stable/options/#build-frontend) option. You might notice that you can see your build log output now! (#2321) +- 🛠 Build verbosity settings have been reworked to have consistent meanings between build backends when non-zero. (#2339) +- 🛠 Removed the `CIBW_PRERELEASE_PYTHONS` and `CIBW_FREE_THREADED_SUPPORT` options - these have been folded into the [`enable`](https://cibuildwheel.pypa.io/en/stable/options/#enable) option instead. (#2095) +- 🛠 Build environments no longer have setuptools and wheel preinstalled. (#2329) +- 🛠 Use the standard Schema line for the integrated JSONSchema. (#2433) +- ⚠️ Dropped support for building Python 3.6 and 3.7 wheels. If you need to build wheels for these versions, use cibuildwheel v2.23.3 or earlier. (#2282) +- ⚠️ The minimum Python version required to run cibuildwheel is now Python 3.11. You can still build wheels for Python 3.8 and newer. (#1912) +- ⚠️ 32-bit Linux wheels no longer built by default - the [arch](https://cibuildwheel.pypa.io/en/stable/options/#archs) was removed from `"auto"`. It now requires explicit `"auto32"`. Note that modern manylinux images (like the new default, `manylinux_2_28`) do not have 32-bit versions. (#2458) +- ⚠️ PyPy wheels no longer built by default, due to a change to our options system. To continue building PyPy wheels, you'll now need to set the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable) to `pypy` or `pypy-eol`. (#2095) +- ⚠️ Dropped official support for Appveyor. If it was working for you before, it will probably continue to do so, but we can't be sure, because our CI doesn't run there anymore. (#2386) +- 📚 A reorganisation of the docs, and numerous updates. (#2280) +- 📚 Use Python 3.14 color output in docs CLI output. (#2407) +- 📚 Docs now primarily use the pyproject.toml name of options, rather than the environment variable name. (#2389) +- 📚 README table now matches docs and auto-updates. (#2427, #2428) + +### v2.23.3 + +_26 April 2025_ + +- 🛠 Dependency updates, including Python 3.13.3 (#2371) + +### v2.23.2 + +_24 March 2025_ + +- 🐛 Workaround an issue with pyodide builds when running cibuildwheel with a Python that was installed via UV (#2328 via #2331) +- 🛠 Dependency updates, including a manylinux update that fixes an ['undefined symbol' error](https://github.com/pypa/manylinux/issues/1760) in gcc-toolset (#2334) + +### v2.23.1 + +_15 March 2025_ + +- ⚠️ Added warnings when the shorthand values `manylinux1`, `manylinux2010`, `manylinux_2_24`, and `musllinux_1_1` are used to specify the images in linux builds. The shorthand to these (unmaintainted) images will be removed in v3.0. If you want to keep using these images, explicitly opt-in using the full image URL, which can be found in [this file](https://github.com/pypa/cibuildwheel/blob/v2.23.1/cibuildwheel/resources/pinned_docker_images.cfg). (#2312) +- 🛠 Dependency updates, including a manylinux update which fixes an [issue with rustup](https://github.com/pypa/cibuildwheel/issues/2303). (#2315) + +### v2.23.0 + +_1 March 2025_ + +- ✨ Adds official support for the new GitHub Actions Arm runners. In fact these worked out-of-the-box, now we include them in our tests and example configs. (#2135 via #2281) +- ✨ Adds support for building PyPy 3.11 wheels (#2268 via #2281) +- 🛠 Adopts the beta pypa/manylinux image for armv7l builds (#2269 via #2281) +- 🛠 Dependency updates, including Pyodide 0.27 (#2117 and #2281) + +### v2.22.0 + +_23 November 2024_ + +- 🌟 Added a new `CIBW_ENABLE`/`enable` feature that replaces `CIBW_FREETHREADED_SUPPORT`/`free-threaded-support` and `CIBW_PRERELEASE_PYTHONS` with a system that supports both. In cibuildwheel 3, this will also include a PyPy setting and the deprecated options will be removed. (#2048) +- 🌟 [Dependency groups](https://peps.python.org/pep-0735/) are now supported for tests. Use `CIBW_TEST_GROUPS`/`test-groups` to specify groups in `[dependency-groups]` for testing. (#2063) +- 🌟 Support for the experimental Ubuntu-based ARMv7l manylinux image (#2052) +- ✨ Show a warning when cibuildwheel is run from Python 3.10 or older; cibuildwheel 3.0 will require Python 3.11 or newer as host (#2050) +- 🐛 Fix issue with stderr interfering with checking the docker version (#2074) +- 🛠 Python 3.9 is now used in `CIBW_BEFORE_ALL`/`before-all` on linux, replacing 3.8, which is now EoL (#2043) +- 🛠 Error messages for producing a pure-Python wheel are slightly more informative (#2044) +- 🛠 Better error when `uname -m` fails on ARM (#2049) +- 🛠 Better error when repair fails and docs for abi3audit on Windows (#2058) +- 🛠 Better error when `manylinux-interpreters ensure` fails (#2066) +- 🛠 Update Pyodide to 0.26.4, and adapt to the unbundled pyodide-build (now 0.29) (#2090) +- 🛠 Now cibuildwheel uses dependency-groups for development dependencies (#2064, #2085) +- 📚 Docs updates and tidy ups (#2061, #2067, #2072) + + +### v2.21.3 + +_9 October 2024_ + +- 🛠 Update CPython 3.13 to 3.13.0 final release (#2032) +- 📚 Docs updates and tidy ups (#2035) + +### v2.21.2 + +_2 October 2024_ + +- ✨ Adds support for building 32-bit armv7l wheels on musllinux. On a Linux system with emulation set up, set [CIBW_ARCHS](https://cibuildwheel.pypa.io/en/stable/options/#archs) to `armv7l` on Linux to try it out if you're interested! (#2017) +- 🐛 Fix Linux Podman builds on some systems (#2016) +- ✨ Adds official support for running on Python 3.13 (#2026) +- 🛠 Update CPython 3.13 to 3.13.0rc3 (#2029) + +Note: the default [manylinux image](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) is **scheduled to change** from `manylinux2014` to `manylinux_2_28` in a cibuildwheel release on or after **6th May 2025** - you can set the value now to avoid getting upgraded if you want. (#1992) + +### v2.21.1 + +_16 September 2024_ + +- 🐛 Fix a bug in the Linux build, where files copied to the container would have invalid ownership permissions (#2007) +- 🐛 Fix a bug on Windows where cibuildwheel would call upon `uv` to install dependencies for versions of CPython that it does not support (#2005) +- 🐛 Fix a bug where `uv 0.4.10` would not use the right Python when testing on Linux. (#2008) +- 🛠 Bump our documentation pins, fixes an issue with a missing package (#2011) + +### v2.21.0 + +_13 September 2024_ + +- ⚠️ Update CPython 3.12 to 3.12.6, which changes the macOS minimum deployment target on CPython 3.12 from macOS 10.9 to macOS 10.13 (#1998) +- 🛠 Changes the behaviour when inheriting `config-settings` in TOML overrides - rather than extending each key, which is rarely useful, individual keys will override previously set values. (#1803) +- 🛠 Update CPython 3.13 to 3.13.0rc2 (#1998) +- ✨ Adds support for multiarch OCI images (#1961) +- 🐛 Fixes some bugs building Linux wheels on macOS. (#1961) +- ⚠️ Changes the minimum version of Docker/Podman to Docker API version 1.43, Podman API version 3. The only mainstream runner this should affect is Travis Graviton2 runners - if so you can [upgrade your version of Docker](https://github.com/pypa/cibuildwheel/pull/1961#issuecomment-2304060019). (#1961) + + +### v2.20.0 + +_4 August 2024_ + +- 🌟 CPython 3.13 wheels are now built by default - without the `CIBW_PRERELEASE_PYTHONS` flag. It's time to build and upload these wheels to PyPI! This release includes CPython 3.13.0rc1, which is guaranteed to be ABI compatible with the final release. Free-threading is still behind a flag/config option. (#1950) +- ✨ Provide a `CIBW_ALLOW_EMPTY` environment variable as an alternative to the command line flag. (#1937) +- 🐛 Don't use uv on PyPy3.8 on Windows, it stopped working starting in 0.2.25. Note that PyPy 3.8 is EoL. (#1868) +- 🛠 Set the `VSCMD_ARG_TGT_ARCH` variable based on target arch. (#1876) +- 🛠 Undo cleaner output on pytest 8-8.2 now that 8.3 is out. (#1943) +- 📚 Update examples to use Python 3.12 on host (cibuildwheel will require Python 3.11+ on the host machine starting in October 2024) (#1919) + + +### v2.19.2 + +_2 July 2024_ + +- 🐛 Update manylinux2014 pins to versions that support past-EoL CentOS 7 mirrors. (#1917) +- 🐛 Support `--no-isolation` with `build[uv]` build-frontend. (#1889) +- 🛠 Provide attestations for releases at . (#1916) +- 🛠 Provide CPython 3.13.0b3. (#1913) +- 🛠 Remove some workarounds now that pip 21.1 is available. (#1891, #1892) +- 📚 Remove nosetest from our docs. (#1821) +- 📚 Document the macOS ARM workaround for 3.8 on GHA. (#1871) +- 📚 GitLab CI + macOS is now a supported platform with an example. (#1911) + + +### v2.19.1 + +_13 June 2024_ + +- 🐛 Don't require setup-python on GHA for Pyodide (#1868) +- 🐛 Specify full python path for uv (fixes issue in 0.2.10 & 0.2.11) (#1881) +- 🛠 Update for pip 24.1b2 on CPython 3.13. (#1879) +- 🛠 Fix a warning in our schema generation script. (#1866) +- 🛠 Cleaner output on pytest 8-8.2. (#1865) + + +### v2.19.0 + +_10 June 2024_ + +See the [release post](https://iscinumpy.dev/post/cibuildwheel-2-19-0/) for more info on new features! + +- 🌟 Add Pyodide platform. Set with `--platform pyodide` or `CIBW_PLATFORM: pyodide` on Linux with a host Python 3.12 to build WebAssembly wheels. Not accepted on PyPI currently, but usable directly in a website using Pyodide, for live docs, etc. (#1456, #1859) +- 🌟 Add `build[uv]` backend, which will take a pre-existing uv install (or install `cibuildwheel[uv]`) and use `uv` for all environment setup and installs on Python 3.8+. This is significantly faster in most cases. (#1856) +- ✨ Add free-threaded macOS builds and update CPython to 3.13.0b2. (#1854) +- 🐛 Issue copying a wheel to a non-existent output dir fixed. (#1851, #1862) +- 🐛 Better determinism for the test environment seeding. (#1835) +- 🛠 `VIRTUAL_ENV` variable now set. (#1842) +- 🛠 Remove a pip<21.3 workaround. (#1842) +- 🛠 Error handling was refactored to use exceptions. (#1719) +- 🛠 Hardcoded paths in tests avoided. (#1834) +- 🛠 Single Python tests made more generic. (#1835) +- 🛠 Sped up our ci by splitting up emulation tests. (#1839) + + + +### v2.18.1 + +_20 May 2024_ + +- 🌟 Add free-threaded Linux and Windows builds for 3.13. New identifiers `cp313t-*`, new option `CIBW_FREE_THREADED_SUPPORT`/`tool.cibuildwheel.free-threaded-support` required to opt-in. [See the docs](https://cibuildwheel.pypa.io/en/stable/options/#free-threaded-support) for more information. (#1831) +- ✨ The `container-engine` is now a build (non-global) option. (#1792) +- 🛠 The build backend for cibuildwheel is now hatchling. (#1297) +- 🛠 Significant improvements and modernization to our noxfile. (#1823) +- 🛠 Use pylint's new GitHub Actions reporter instead of a custom matcher. (#1823) +- 🛠 Unpin virtualenv updates for Python 3.7+ (#1830) +- 🐛 Fix running linux tests from Windows or macOS ARM. (#1788) +- 📚 Fix our documentation build. (#1821) + + +### v2.18.0 + +_12 May 2024_ + +- ✨ Adds CPython 3.13 support, under the prerelease flag [CIBW_PRERELEASE_PYTHONS](https://cibuildwheel.pypa.io/en/stable/options/#prerelease-pythons). This version of cibuildwheel uses 3.13.0b1. Free-threading mode is not available yet (#1657), waiting on official binaries (planned for beta 2) and pip support. + + _While CPython is in beta, the ABI can change, so your wheels might not be compatible with the final release. For this reason, we don't recommend distributing wheels until RC1, at which point 3.13 will be available in cibuildwheel without the flag._ (#1815) + +- ✨ Musllinux now defaults to `musllinux_1_2`. You can set the older `musllinux_1_1` via config if needed. (#1817) +- 🛠 No longer pre-seed setuptools/wheel in virtual environments (#1819) +- 🛠 Respect the constraints file when building with pip, matching build (#1818) +- 🛠 Use uv to compile our pinned dependencies, 10x faster and doesn't require special setup (#1778) +- 🐛 Fix an issue with the schema (#1788) +- 📚 Document the new delocate error checking macOS versions (#1766) +- 📚 Document Rust builds (#1816) +- 📚 Speed up our readthedocs builds with uv, 26 seconds -> 6 seconds to install dependencies (#1816) + + +### v2.17.0 + +_11 March 2024_ + +- 🌟 Adds the ability to inherit configuration in TOML overrides. This makes certain configurations much simpler. If you're overriding an option like `before-build` or `environment`, and you just want to add an extra command or environment variable, you can just append (or prepend) to the previous config. See [the docs](https://cibuildwheel.pypa.io/en/stable/options/#inherit) for more information. (#1730) +- 🌟 Adds official support for native `arm64` macOS GitHub runners. To use them, just specify `macos-14` as an `os` of your job in your workflow file. You can also keep `macos-13` in your build matrix to build `x86_64`. Check out the new [GitHub Actions example config](https://cibuildwheel.pypa.io/en/stable/setup/#github-actions). +- ✨ You no longer need to specify `--platform` to run cibuildwheel locally! Instead it will detect your platform automatically. This was a safety feature, no longer necessary. (#1727) +- 🛠 Removed setuptools and wheel pinned versions. This only affects old-style projects without a `pyproject.toml`, projects with `pyproject.toml` are already getting fresh versions of their `build-system.requires` installed into an isolated environment. (#1725) +- 🛠 Improve how the GitHub Action passes arguments (#1757) +- 🛠 Remove a system-wide install of pipx in the GitHub Action (#1745) +- 🐛 No longer will cibuildwheel override the `PIP_CONSTRAINT` environment variable when using the `build` frontend. Instead it will be extended. (#1675) +- 🐛 Fix a bug where building and testing both `x86_86` and `arm64` wheels on the same runner caused the wrong architectures in the test environment (#1750) +- 🐛 Fix a bug that prevented testing a CPython 3.8 wheel targeting macOS 11+ on `x86_64` (#1768) +- 📚 Moved the docs onto the official PyPA domain - they're now available at https://cibuildwheel.pypa.io . (#1775) +- 📚 Docs and examples improvements (#1762, #1734) + + +### v2.16.5 + +_30 January 2024_ + +- 🐛 Fix an incompatibility with the GitHub Action and new GitHub Runner images for Windows that bundle Powershell 7.3+ (#1741) +- 🛠 Preliminary support for new `macos-14` arm64 runners (#1743) + +### v2.16.4 + +_28 January 2024_ + +- 🛠 Update manylinux pins to upgrade from a problematic PyPy version. (#1737) + +### v2.16.3 + +_26 January 2024_ + +- 🐛 Fix a bug when building from sdist, where relative paths to files in the package didn't work because the working directory was wrong (#1687) +- 🛠 Adds the ability to disable mounting the host filesystem in containers to `/host`, through the `disable_host_mount` suboption on [`CIBW_CONTAINER_ENGINE`](https://cibuildwheel.pypa.io/en/stable/options/#container-engine). +- 📚 A lot of docs improvements! (#1708, #1705, #1686, #1679, #1667, #1665) + +### v2.16.2 + +_3 October 2023_ + +- 🛠 Updates CPython 3.12 version to 3.12.0, final release (#1635) +- ✨ Adds a debug option [`CIBW_DEBUG_KEEP_CONTAINER`](https://cibuildwheel.pypa.io/en/stable/options/#cibw_debug_keep_container) to stop cibuildwheel deleting build containers after the build finishes. (#1620) +- 📚 Adds support for `[tool.cibuildwheel]` checking by adding a schema compatible with the [validate-pyproject](https://github.com/abravalheri/validate-pyproject/) tool (#1622, #1628, #1629) +- 🐛 Fix parsing of `CIBW_CONTAINER_ENGINE` and `CIBW_BUILD_FRONTEND` options to not break arguments on `:` characters (#1621) +- 🐛 Fix the evaluation order of `CIBW_ENVIRONMENT` and `CIBW_ENVIRONMENT_PASS` so that `CIBW_ENVIRONMENT` assignments can reference environment variables passed through from the host machine. (#1617) +- 🛠 Supports manylinux images' deferred installation of interpreters through the `manylinux-interpreters` tool (#1630) + +### v2.16.1 + +_26 September 2023_ + +- 🛠 Updates the prerelease CPython 3.12 version to 3.12.0rc3 (#1625) +- 🛠 Only calls `linux32` in containers when necessary (#1599) + +### v2.16.0 + +_18 September 2023_ + +- ✨ Add the ability to pass additional flags to a build frontend through the [CIBW_BUILD_FRONTEND](https://cibuildwheel.pypa.io/en/stable/options/#build-frontend) option (#1588). +- ✨ The environment variable SOURCE_DATE_EPOCH is now automatically passed through to container Linux builds (useful for [reproducible builds](https://reproducible-builds.org/docs/source-date-epoch/)!) (#1589) +- 🛠 Updates the prerelease CPython 3.12 version to 3.12.0rc2 (#1604) +- 🐛 Fix `requires_python` auto-detection from setup.py when the call to `setup()` is within an `if __name__ == "__main__"` block (#1613) +- 🐛 Fix a bug that prevented building Linux wheels in Docker on a Windows host (#1573) +- 🐛 `--only` can now select prerelease-pythons (#1564) +- 📚 Docs & examples updates (#1582, #1593, #1598, #1615) + +### v2.15.0 + +_8 August 2023_ + +- 🌟 CPython 3.12 wheels are now built by default - without the CIBW_PRERELEASE_PYTHONS flag. It's time to build and upload these wheels to PyPI! This release includes CPython 3.12.0rc1, which is guaranteed to be ABI compatible with the final release. (#1565) +- ✨ Adds musllinux_1_2 support - this allows packagers to build for musl-based Linux distributions on a more recent Alpine image, and a newer musl libc. (#1561) + +### v2.14.1 + +_15 July 2023_ + +- 🛠 Updates the prerelease CPython 3.12 version to 3.12.0b4 (#1550) + +### v2.14.0 + +_10 July 2023_ + +- ✨ Adds support for building PyPy 3.10 wheels. (#1525) +- 🛠 Updates the prerelease CPython 3.12 version to 3.12.0b3. +- ✨ Allow the use of the `{wheel}` placeholder in CIBW_TEST_COMMAND (#1533) +- 📚 Docs & examples updates (#1532, #1416) +- ⚠️ Removed support for running cibuildwheel in Python 3.7. Python 3.7 is EOL. However, cibuildwheel continues to build Python 3.7 wheels for the moment. (#1175) + +### v2.13.1 + +_10 June 2023_ + +- 🛠 Updates the prerelease CPython 3.12 version to 3.12.0b2. (#1516) +- 🛠 Adds a moving `v.` tag for use in GitHub Actions workflow files. If you use this, you'll get the latest patch release within a minor version. Additionally, Dependabot won't send you PRs for patch releases. (#1517) + +### v2.13.0 + +_28 May 2023_ + +- ✨ Adds CPython 3.12 support, under the prerelease flag [CIBW_PRERELEASE_PYTHONS](https://cibuildwheel.pypa.io/en/stable/options/#prerelease-pythons). This version of cibuildwheel uses 3.12.0b1. + + While CPython is in beta, the ABI can change, so your wheels might not be compatible with the final release. For this reason, we don't recommend distributing wheels until RC1, at which point 3.12 will be available in cibuildwheel without the flag. (#1507) + +- ✨ Adds the ability to pass arguments to the container engine when the container is created, using the [CIBW_CONTAINER_ENGINE](https://cibuildwheel.pypa.io/en/stable/options/#container-engine) option. (#1499) + +### v2.12.3 + +_19 April 2023_ + +- 🐛 Fix an import error when running on Python 3.7. (#1479) + +### v2.12.2 + +_18 April 2023_ + +- 🐛 Fix a bug that caused an extra empty config-setting to be passed to the backend when CIBW_BUILD_FRONTEND is set to `build`. (#1474) +- 🐛 Fix a crash that occurred when overwriting an existing wheel on Windows. (#1464) +- 🛠 Pinned version updates, including CPython 3.10.11, 3.11.3, pip 23.1 and wheel 0.40.0. + +### v2.12.1 + +_11 March 2023_ + +- 🐛 Fix a bug that prevented the use of CIBW_CONFIG_SETTINGS with the 'pip' build backend. (#1430) + +### v2.12.0 + +_16 Jan 2023_ + +- ✨ Adds support for PyPy arm64 wheels. This means that you can build PyPy wheels for Apple Silicon machines. Cross-compilation is not supported for these wheels, so you'll have to build on an Apple Silicon machine. (#1372) +- 🛠 Pinned version updates, including PyPy to v7.3.11 and setuptools to 66.0.0. + +### v2.11.4 + +_24 Dec 2022_ + +- 🐛 Fix a bug that caused missing wheels on Windows when a test was skipped using CIBW_TEST_SKIP (#1377) +- 🛠 Updates CPython 3.11 to 3.11.1 (#1371) +- 🛠 Updates PyPy to 7.3.10, except on macOS which remains on 7.3.9 due to a bug on that platform. (#1371) +- 📚 Added a reference to abi3audit to the docs (#1347) + +### v2.11.3 + +_5 Dec 2022_ + +- ✨ Improves the 'build options' log output that's printed at the start of each run (#1352) +- ✨ Added a friendly error message to a common misconfiguration of the `CIBW_TEST_COMMAND` option - not specifying path using the `{project}` placeholder (#1336) +- 🛠 The GitHub Action now uses Powershell on Windows to avoid occasional incompabilities with bash (#1346) + +### v2.11.2 + +_26 October 2022_ + +- 🛠 Updates CPython 3.11 to 3.11.0 - final release (#1327) +- 🛠 Simplify the default macOS repair command (#1322) +- 🛠 Fix the default `MACOSX_DEPLOYMENT_TARGET` on arm64 (#1312) +- 🛠 Hide irrelevant pip warnings on linux (#1311) +- 🐛 Fix a bug that caused the stdout and stderr of commands in containers to be in the wrong order Previously, stdout could appear after stderr. (#1324) +- 📚 Added [a FAQ entry](https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64) describing how to perform native builds of CPython 3.8 wheels on Apple Silicon. (#1323) +- 📚 Other docs improvements + +### v2.11.1 + +_13 October 2022_ + +- 🛠 Updates to the latest manylinux images, and updates CPython 3.10 to 3.10.8. + +### v2.11.0 + +_13 October 2022_ + +- 🌟 Adds support for cross-compiling Windows ARM64 wheels. To use this feature, add `ARM64` to the [CIBW_ARCHS](https://cibuildwheel.pypa.io/en/stable/options/#archs) option on a Windows Intel runner. (#1144) +- ✨ Adds support for building Linux aarch64 wheels on Circle CI. (#1307) +- ✨ Adds support for building Windows wheels on Gitlab CI. (#1295) +- ✨ Adds support for building Linux aarch64 wheels under emulation on Gitlab CI. (#1295) +- ✨ Adds the ability to test `cp38-macosx_arm64` wheels on a native arm64 runner. To do this, you'll need to preinstall the (experimental) [universal2 version of CPython 3.8](https://www.python.org/ftp/python/3.8.10/python-3.8.10-macos11.pkg) on your arm64 runner before invoking cibuildwheel. Note: it is not recommended to build x86_64 wheels with this setup, your wheels will have limited compatibility wrt macOS versions. (#1283) +- 🛠 Improved error messages when using custom Docker images and Python cannot be found at the correct path. (#1298) +- 📚 Sample configs for Azure Pipelines and Travis CI updated (#1296) +- 📚 Other docs improvements - including more information about using Homebrew for build dependencies (#1290) + +### v2.10.2 + +_25 September 2022_ + +- 🐛 Fix a bug that caused `win32` identifiers to fail when used with `--only`. (#1282) +- 🐛 Fix computation of `auto`/`auto64`/`auto32` archs when targeting a different platform to the one that you're running cibuildwheel on. (#1266) +- 📚 Fix an mistake in the 'how it works' diagram. (#1274) + +### v2.10.1 + +_18 September 2022_ + +- 🐛 Fix a bug that stopped environment variables specified in TOML from being expanded. (#1273) + +### v2.10.0 + +_13 September 2022_ + +- 🌟 Adds support for [building wheels on Cirrus CI](https://cibuildwheel.pypa.io/en/stable/setup/#cirrus-ci). This is exciting for us, as it's the first public CI platform that natively supports macOS Apple Silicon (aka. M1, `arm64`) runners. As such, it's the first platform that you can natively build _and test_ macOS `arm64` wheels. It also has native Linux ARM (aarch64) runners, for fast, native builds there. (#1191) +- 🌟 Adds support for running cibuildwheel on Apple Silicon machines. For a while, we've supported cross-compilation of Apple Silicon wheels on `x86_64`, but now that we have Cirrus CI we can run our test suite and officially support running cibuildwheel on `arm64`. (#1191) +- ✨ Adds the `--only` [command line option](https://cibuildwheel.pypa.io/en/stable/options/#command-line), to specify a single build to run. Previously, it could be cumbersome to set all the build selection options to target a specific build - for example, you might have to run something like `CIBW_BUILD=cp39-manylinux_x86_64 cibuildwheel --platform linux --archs x86_64`. The new `--only` option overrides all the build selection options to simplify running a single build, which now looks like `cibuildwheel --only cp39-manylinux_x86_64`. (#1098) +- ✨ Adds the [`CIBW_CONFIG_SETTINGS`](https://cibuildwheel.pypa.io/en/stable/options/#config-settings) option, so you can pass arguments to your package's build backend (#1244) +- 🛠 Updates the CPython 3.11 version to the latest release candidate - v3.11.0rc2. (#1265) +- 🐛 Fix a bug that can cause a RecursionError on Windows when building from an sdist. (#1253) +- 🛠 Add support for the s390x architecture on manylinux_2_28 (#1255) + +### v2.9.0 + +_11 August 2022_ + +- 🌟 CPython 3.11 wheels are now built by default - without the CIBW_PRERELEASE_PYTHONS flag. It's time to build and upload these wheels to PyPI! This release includes CPython 3.11.0rc1, which is guaranteed to be ABI compatible with the final release. (#1226) +- ⚠️ Removed support for running cibuildwheel in Python 3.6. Python 3.6 is EOL. However, cibuildwheel continues to build CPython 3.6 wheels for the moment. (#1175) +- ✨ Improved error messages when misspelling TOML options, suggesting close matches (#1205) +- 🛠 When running on Apple Silicon (so far, an unsupported mode of operation), cibuildwheel no longer builds universal2 wheels by default - just arm64. See [#1204](https://github.com/pypa/cibuildwheel/issues/1204) for discussion. We hope to release official support for native builds on Apple Silicon soon! (#1217) + +### v2.8.1 + +_18 July 2022_ + +- 🐛 Fix a bug when building CPython 3.8 wheels on an Apple Silicon machine where testing would always fail. cibuildwheel will no longer attempt to test the arm64 part of CPython 3.8 wheels because we use the x86_64 installer of CPython 3.8 due to its macOS system version backward-compatibility. See [#1169](https://github.com/pypa/cibuildwheel/pull/1169) for more details. (#1171) +- 🛠 Update the prerelease CPython 3.11 to 3.11.0b4. (#1180) +- 🛠 The GitHub Action will ensure a compatible version of Python is installed on the runner (#1114) +- 📚 A few docs improvements + +### v2.8.0 + +_5 July 2022_ + +- ✨ You can now run cibuildwheel on Podman, as an alternate container engine to Docker (which remains the default). This is useful in environments where a Docker daemon isn't available, for example, it can be run inside a Docker container, or without root access. To use Podman, set the [`CIBW_CONTAINER_ENGINE`](https://cibuildwheel.pypa.io/en/stable/options/#container-engine) option. (#966) +- ✨ Adds support for building `py3-none-{platform}` wheels. This works the same as ABI3 - wheels won't be rebuilt, but tests will still be run across all selected versions of Python. + + These wheels contain native extension code, but don't use the Python APIs. Typically, they're bridged to Python using a FFI module like [ctypes](https://docs.python.org/3/library/ctypes.html) or [cffi](https://cffi.readthedocs.io/en/latest/). Because they don't use Python ABI, the wheels are more compatible - they work across many Python versions. + + Check out this [example ctypes project](https://github.com/joerick/python-ctypes-package-sample) to see an example of how it works. (#1151) + +- 🛠 cibuildwheel will now error if multiple builds in a single run produce the same wheel filename, as this indicates a misconfiguration. (#1152) +- 📚 A few docs improvements and updates to keep things up-to-date. + +### v2.7.0 + +_17 June 2022_ + +- 🌟 Added support for the new `manylinux_2_28` images. These new images are based on AlmaLinux, the community-driven successor to CentOS, unlike manylinux_2_24, which was based on Debian. To build on these images, set your [`CIBW_MANYLINUX_*_IMAGE`](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) option to `manylinux_2_28`. (#1026) +- 🐛 Fix a bug where tests were not being run on CPython 3.11 (when CIBW_PRERELEASE_PYTHONS was set) (#1138) +- ✨ You can now build Linux wheels on Windows, as long as you have Docker installed and set to 'Linux containers' (#1117) +- 🐛 Fix a bug on macOS that caused cibuildwheel to crash trying to overwrite a previously-built wheel of the same name. (#1129) + +### v2.6.1 + +_7 June 2022_ + +- 🛠 Update the prerelease CPython 3.11 to 3.11.0b3 + +### v2.6.0 + +_25 May 2022_ + +- 🌟 Added the ability to test building wheels on CPython 3.11! Because CPython 3.11 is in beta, these wheels should not be distributed, because they might not be compatible with the final release, but it's available to build for testing purposes. Use the flag [`--prerelease-pythons` or `CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.pypa.io/en/stable/options/#prerelease-pythons) to test. This version of cibuildwheel includes CPython 3.11.0b1. (#1109) +- 📚 Added an interactive diagram showing how cibuildwheel works to the [docs](https://cibuildwheel.pypa.io/en/stable/#how-it-works) (#1100) + +### v2.5.0 + +_29 April 2022_ + +- ✨ Added support for building ABI3 wheels. cibuildwheel will now recognise when an ABI3 wheel was produced, and skip subsequent build steps where the previously built wheel is compatible. Tests still will run on all selected versions of Python, using the ABI3 wheel. Check [this entry](https://cibuildwheel.pypa.io/en/stable/faq/#abi3) in the docs for more info. (#1091) +- ✨ You can now build wheels directly from sdist archives, in addition to source directories. Just call cibuildwheel with an sdist argument on the command line, like `cibuildwheel mypackage-1.0.0.tar.gz`. For more details, check the [`--help` output](https://cibuildwheel.pypa.io/en/stable/options/#command-line) (#1096) +- 🐛 Fix a bug where cibuildwheel would crash when no builds are selected and `--allow-empty` is passed (#1086) +- 🐛 Workaround a permissions issue on Linux relating to newer versions of git and setuptools_scm (#1095) +- 📚 Minor docs improvements + +### v2.4.0 + +_2 April 2022_ + +- ✨ cibuildwheel now supports running locally on Windows and macOS (as well as Linux). On macOS, you'll have to install the versions of Pythons that you want to use from Python.org, and cibuildwheel will use them. On Windows, cibuildwheel will install it's own versions of Python. Check out [the documentation](https://cibuildwheel.pypa.io/en/stable/setup/#local) for instructions. (#974) +- ✨ Added support for building PyPy 3.9 wheels. (#1031) +- ✨ Listing at the end of the build now displays the size of each wheel (#975) +- 🐛 Workaround a connection timeout bug on Travis CI ppc64le runners (#906) +- 🐛 Fix an encoding error when reading setup.py in the wrong encoding (#977) +- 🛠 Setuptools updated to 61.3.0, including experimental support for reading config from pyproject.toml(PEP 621). This could change the behaviour of your build if you have a pyproject.toml with a `[project]` table, because that takes precedence over setup.py and setup.cfg. Check out the [setuptools docs](https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html) and the [project metadata specification](https://packaging.python.org/en/latest/specifications/declaring-project-metadata/) for more info. +- 🛠 Many other dependency updates. +- 📚 Minor docs improvements + ### v2.3.1 _14 December 2021_ @@ -13,12 +515,12 @@ _14 December 2021_ _26 November 2021_ -- 📈 cibuildwheel now defaults to manylinux2014 image for linux builds, rather than manylinux2010. If you want to stick with manylinux2010, it's simple to set this using [the image options](https://cibuildwheel.readthedocs.io/en/stable/options/#linux-image). (#926) -- ✨ You can now pass environment variables from the host machine into the Docker container during a Linux build. Check out [the docs for `CIBW_ENVIRONMENT_PASS_LINUX `](https://cibuildwheel.readthedocs.io/en/latest/options/#environment-pass) for the details. (#914) +- 📈 cibuildwheel now defaults to manylinux2014 image for linux builds, rather than manylinux2010. If you want to stick with manylinux2010, it's simple to set this using [the image options](https://cibuildwheel.pypa.io/en/stable/options/#linux-image). (#926) +- ✨ You can now pass environment variables from the host machine into the Docker container during a Linux build. Check out [the docs for `CIBW_ENVIRONMENT_PASS_LINUX `](https://cibuildwheel.pypa.io/en/stable/options/#environment-pass) for the details. (#914) - ✨ Added support for building PyPy 3.8 wheels. (#881) - ✨ Added support for building Windows arm64 CPython wheels on a Windows arm64 runner. We can't test this in CI yet, so for now, this is experimental. (#920) - 📚 Improved the deployment documentation (#911) -- 🛠 Changed the escaping behaviour inside cibuildwheel's option placeholders e.g. `{project}` in `before_build` or `{dest_dir}` in `repair_wheel_command`. This allows bash syntax like `${SOME_VAR}` to passthrough without being interpreted as a placeholder by cibuildwheel. See [this section](https://cibuildwheel.readthedocs.io/en/stable/options/#placeholders) in the docs for more info. (#889) +- 🛠 Changed the escaping behaviour inside cibuildwheel's option placeholders e.g. `{project}` in `before_build` or `{dest_dir}` in `repair_wheel_command`. This allows bash syntax like `${SOME_VAR}` to passthrough without being interpreted as a placeholder by cibuildwheel. See [this section](https://cibuildwheel.pypa.io/en/stable/options/#placeholders) in the docs for more info. (#889) - 🛠 Pip updated to 21.3, meaning it now defaults to in-tree builds again. If this causes an issue with your project, setting environment variable `PIP_USE_DEPRECATED=out-of-tree-build` is available as a temporary flag to restore the old behaviour. However, be aware that this flag will probably be removed soon. (#881) - 🐛 You can now access the current Python interpreter using `python3` within a build on Windows (#917) @@ -41,15 +543,15 @@ _22 October 2021_ - 🌟 Added support for [musllinux](https://www.python.org/dev/peps/pep-0656/). Support for this new wheel format lets projects build wheels for Linux distributions that use [musl libc](https://musl.libc.org/), notably, [Alpine](https://alpinelinux.org/) Docker containers. (#768) - Musllinux builds are enabled by default. If you're not ready to build musllinux, add `*-musllinux_*` to your [`CIBW_SKIP`/`skip`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) option. Or, you might have to make some changes to your options - to simplify that process, you can use... + Musllinux builds are enabled by default. If you're not ready to build musllinux, add `*-musllinux_*` to your [`CIBW_SKIP`/`skip`](https://cibuildwheel.pypa.io/en/stable/options/#build-skip) option. Or, you might have to make some changes to your options - to simplify that process, you can use... - 🌟 TOML option overrides! This provides much greater flexibility in configuration via pyproject.toml. (#854) - You can now set build options for any subset of your builds using a match pattern. So, for example, you can customise CPython 3.8 builds with an override on `cp38-*` or musllinux builds by selecting `*musllinux*`. Check out [the docs](https://cibuildwheel.readthedocs.io/en/latest/options/#overrides) for more info on the specifics. + You can now set build options for any subset of your builds using a match pattern. So, for example, you can customise CPython 3.8 builds with an override on `cp38-*` or musllinux builds by selecting `*musllinux*`. Check out [the docs](https://cibuildwheel.pypa.io/en/stable/options/#overrides) for more info on the specifics. - 🛠 Added support for building PyPy wheels on macOS 11 CI runners. (#875) -- 🛠 Setting an empty string for the [`CIBW_*_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) option will now fallthrough to the config file or cibuildwheel's default, rather than causing an error. This makes the option easier to use in CI build matricies. (#829) +- 🛠 Setting an empty string for the [`CIBW_*_IMAGE`](https://cibuildwheel.pypa.io/en/stable/options/#manylinux-image) option will now fallthrough to the config file or cibuildwheel's default, rather than causing an error. This makes the option easier to use in CI build matrices. (#829) - 🛠 Support for TOML 1.0 when reading config files, via the `tomli` package. (#876) @@ -88,9 +590,9 @@ _25 July 2021_ _16 July 2021_ -- 🌟 You can now configure cibuildwheel options inside your project's `pyproject.toml`! Environment variables still work of course. Check out the [documentation](https://cibuildwheel.readthedocs.io/en/stable/options/#setting-options) for more info. -- 🌟 Added support for building wheels with [build](https://github.com/pypa/build), as well as pip. This feature is controlled with the [`CIBW_BUILD_FRONTEND`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-frontend) option. -- 🌟 Added the ability to test building wheels on CPython 3.10! Because CPython 3.10 is in beta, these wheels should not be distributed, because they might not be compatible with the final release, but it's available to build for testing purposes. Use the flag [`--prerelease-pythons` or `CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons) to test. (#675) This version of cibuildwheel includes CPython 3.10.0b4. +- 🌟 You can now configure cibuildwheel options inside your project's `pyproject.toml`! Environment variables still work of course. Check out the [documentation](https://cibuildwheel.pypa.io/en/stable/options/#setting-options) for more info. +- 🌟 Added support for building wheels with [build](https://github.com/pypa/build), as well as pip. This feature is controlled with the [`CIBW_BUILD_FRONTEND`](https://cibuildwheel.pypa.io/en/stable/options/#build-frontend) option. +- 🌟 Added the ability to test building wheels on CPython 3.10! Because CPython 3.10 is in beta, these wheels should not be distributed, because they might not be compatible with the final release, but it's available to build for testing purposes. Use the flag [`--prerelease-pythons` or `CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.pypa.io/en/stable/options/#prerelease-pythons) to test. (#675) This version of cibuildwheel includes CPython 3.10.0b4. - ⚠️ **Removed support for building Python 2.7 and Python 3.5 wheels**, for both CPython and PyPy. If you still need to build on these versions, please use the latest v1.x version. (#596) - ✨ Added the ability to build CPython 3.8 wheels for Apple Silicon. (#704) - 🛠 Update to the latest build dependencies, including Auditwheel 4. (#633) @@ -117,7 +619,7 @@ _1 May 2021_ - 📚 Lots of docs improvements! (#650, #623, #616, #609, #606) - 🐛 Fix nuget "Package is not found" error on Windows. (#653) -- ⚠️ cibuildwheel will no longer build Windows 2.7 wheels, unless you specify a custom toolchain using `DISTUTILS_USE_SDK=1` and `MSSdk=1`. This is because Microsoft have stopped distributing Visual C++ Compiler for Python 2.7. See [this FAQ entry](https://cibuildwheel.readthedocs.io/en/stable/faq/#windows-and-python-27) for more details. (#649) +- ⚠️ cibuildwheel will no longer build Windows 2.7 wheels, unless you specify a custom toolchain using `DISTUTILS_USE_SDK=1` and `MSSdk=1`. This is because Microsoft have stopped distributing Visual C++ Compiler for Python 2.7. See [this FAQ entry](https://cibuildwheel.pypa.io/en/stable/faq/#windows-and-python-27) for more details. (#649) - 🐛 Fix crash on Windows due to missing `which` command (#641). ### v1.10.0 @@ -125,7 +627,7 @@ _1 May 2021_ _22 Feb 2021_ - ✨ Added `manylinux_2_24` support. To use these new Debian-based manylinux - images, set your [manylinux image](https://cibuildwheel.readthedocs.io/en/stable/options/#linux-image) + images, set your [manylinux image](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) options to `manylinux_2_24`. - 🛠 On macOS, we now set `MACOSX_DEPLOYMENT_TARGET` in before running `CIBW_BEFORE_ALL`. This is useful when using `CIBW_BEFORE_ALL` to build a @@ -142,10 +644,10 @@ _5 February 2021_ - 🌟 Added support for Apple Silicon wheels on macOS! You can now cross-compile `universal2` and `arm64` wheels on your existing macOS Intel runners, by setting - [CIBW_ARCHS_MACOS](https://cibuildwheel.readthedocs.io/en/stable/options/#archs). + [CIBW_ARCHS_MACOS](https://cibuildwheel.pypa.io/en/stable/options/#archs). Xcode 12.2 or later is required, but you don't need macOS 11.0 - you can still build on macOS 10.15. See - [this FAQ entry](https://cibuildwheel.readthedocs.io/en/stable/faq/#apple-silicon) + [this FAQ entry](https://cibuildwheel.pypa.io/en/stable/faq/#apple-silicon) for more information. (#484) - 🌟 Added auto-detection of your package's Python compatibility, via declared [`requires-python`](https://www.python.org/dev/peps/pep-0621/#requires-python) @@ -155,7 +657,7 @@ _5 February 2021_ will automatically skip builds on versions of Python that your package doesn't support. Hopefully this makes the first-run experience of cibuildwheel a bit easier. If you need to override this for any reason, - look at [`CIBW_PROJECT_REQUIRES_PYTHON`](https://cibuildwheel.readthedocs.io/en/stable/options/#requires-python). + look at [`CIBW_PROJECT_REQUIRES_PYTHON`](https://cibuildwheel.pypa.io/en/stable/options/#requires-python). (#536) - 🌟 cibuildwheel can now be invoked as a native GitHub Action! You can now invoke cibuildwheel in a GHA build step like: @@ -170,10 +672,10 @@ _5 February 2021_ ``` This saves a bit of boilerplate, and you can [use Dependabot to keep the - pinned version up-to-date](https://cibuildwheel.readthedocs.io/en/stable/faq/#automatic-updates). + pinned version up-to-date](https://cibuildwheel.pypa.io/en/stable/faq/#automatic-updates). - ✨ Added `auto64` and `auto32` shortcuts to the - [CIBW_ARCHS](https://cibuildwheel.readthedocs.io/en/stable/options/#archs) + [CIBW_ARCHS](https://cibuildwheel.pypa.io/en/stable/options/#archs) option. (#553) - ✨ cibuildwheel now prints a list of the wheels built at the end of each run. (#570) @@ -186,9 +688,9 @@ _22 January 2021_ - 🌟 Added support for emulated builds! You can now build manylinux wheels on ARM64`aarch64`, as well as `ppc64le` and 's390x'. To build under emulation, register QEMU via binfmt_misc and set the - [`CIBW_ARCHS_LINUX`](https://cibuildwheel.readthedocs.io/en/stable/options/#archs) + [`CIBW_ARCHS_LINUX`](https://cibuildwheel.pypa.io/en/stable/options/#archs) option to the architectures you want to run. See - [this FAQ entry](https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation) + [this FAQ entry](https://cibuildwheel.pypa.io/en/stable/faq/#emulation) for more information. (#482) - ✨ Added `CIBW_TEST_SKIP` option. This allows you to choose certain builds whose tests you'd like to skip. This might be useful when running a slow @@ -214,7 +716,7 @@ _1 January 2021_ in a virtualenv. (#502) - 🛠 Some preparatory work towards using cibuildwheel as a GitHub Action. Check out - [the FAQ](https://cibuildwheel.readthedocs.io/en/stable/faq/#option-1-github-action) + [the FAQ](https://cibuildwheel.pypa.io/en/stable/faq/#option-1-github-action) for information on how to use it. We'll be fully updating the docs to this approach in a subsequent release (#494) @@ -336,7 +838,7 @@ _25 June 2020_ _24 June 2020_ -- 🌟 Add [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all) +- 🌟 Add [`CIBW_BEFORE_ALL`](https://cibuildwheel.pypa.io/en/stable/options/#before-all) option, which lets you run a command on the build machine before any wheels are built. This is especially useful when building on Linux, to `make` something external to Python, or to `yum install` a dependency. (#342) @@ -379,15 +881,15 @@ _2 May 2020_ environment on macOS and Windows, where the version of Xcode and Visual Studio can still effect things. - This can be controlled using the [CIBW_DEPENDENCY_VERSIONS](https://cibuildwheel.readthedocs.io/en/stable/options/#dependency-versions) - and [manylinux image](https://cibuildwheel.readthedocs.io/en/stable/options/#linux-image) + This can be controlled using the [CIBW_DEPENDENCY_VERSIONS](https://cibuildwheel.pypa.io/en/stable/options/#dependency-versions) + and [manylinux image](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) options - if you always want to use the latest toolchain, you can still do that, or you can specify your own pip constraints file and manylinux image. (#256) - ✨ Added `package_dir` command line option, meaning we now support building a package that lives in a subdirectory and pulls in files from the wider - project. See [the `package_dir` option help](https://cibuildwheel.readthedocs.io/en/stable/options/#command-line-options) + project. See [the `package_dir` option help](https://cibuildwheel.pypa.io/en/stable/options/#command-line-options) for more information. Note that this change makes the working directory (where you call @@ -405,7 +907,7 @@ _2 May 2020_ _12 March 2020_ - 🌟 Add support for building on GitHub Actions! Check out the - [docs](https://cibuildwheel.readthedocs.io/en/stable/setup/#github-actions) + [docs](https://cibuildwheel.pypa.io/en/stable/setup/#github-actions) for information on how to set it up. (#194) - ✨ Add the `CIBW_BEFORE_TEST` option, which lets you run a command to prepare the environment before your tests are run. (#242) @@ -455,9 +957,9 @@ _10 November 2019_ build using the manylinux2010 images by default. If your project is still manylinux1 compatible, you should get both manylinux1 and manylinux2010 wheels - you can upload both to PyPI. If you always require manylinux1 wheels, you can - build using the old manylinux1 image using the [manylinux image](https://cibuildwheel.readthedocs.io/en/stable/options/#linux-image) option. + build using the old manylinux1 image using the [manylinux image](https://cibuildwheel.pypa.io/en/stable/options/#linux-image) option. (#155) -- 📚 Documentation is now on its [own mini-site](https://cibuildwheel.readthedocs.io), +- 📚 Documentation is now on its [own mini-site](https://cibuildwheel.pypa.io), rather than on the README (#169) - ✨ Add support for building Windows wheels on Travis CI. (#160) - 🛠 If you set `CIBW_TEST_COMMAND`, your tests now run in a virtualenv. (#164) @@ -638,6 +1140,18 @@ _31 March 2017_ - 🌟 First public release! + + diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..1f1736056 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,227 @@ +# Configuration methods + +cibuildwheel can either be configured using environment variables, or from +config file such as `pyproject.toml`. + +This page describes how to set options. For a full list of available options, see the [options reference](options.md). + +## Environment variables {: #environment-variables} + +Environment variables can be set in your CI config. For example, to configure +cibuildwheel to run tests, add the following YAML to your CI config file: + +!!! tab "GitHub Actions" + + > .github/workflows/*.yml ([docs](https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables)) (can be global, in job, or in step) + + ```yaml + env: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +!!! tab "Azure Pipelines" + + > azure-pipelines.yml ([docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables)) + + ```yaml + variables: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +!!! tab "Travis CI" + + > .travis.yml ([docs](https://docs.travis-ci.com/user/environment-variables/)) + + ```yaml + env: + global: + - CIBW_TEST_REQUIRES=pytest + - CIBW_TEST_COMMAND="pytest {project}/tests" + ``` + +!!! tab "CircleCI" + + > .circleci/config.yml ([docs](https://circleci.com/docs/2.0/configuration-reference/#environment)) + + ```yaml + jobs: + job_name: + environment: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +!!! tab "Gitlab CI" + + > .gitlab-ci.yml ([docs](https://docs.gitlab.com/ci/yaml/#variables)) + + ```yaml + linux: + variables: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +!!! tab "Cirrus CI" + + > .cirrus.yml ([docs](https://cirrus-ci.org/guide/writing-tasks/#environment-variables)) + + ```yaml + env: + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "pytest {project}/tests" + ``` + +## Configuration file {: #configuration-file} + +You can configure cibuildwheel with a config file, such as `pyproject.toml`. +Options have the same names as the environment variable overrides, but are +placed in `[tool.cibuildwheel]` and are lower case, with dashes, following +common [TOML][https://toml.io] practice. Anything placed in subsections `linux`, `windows`, +`macos`, or `pyodide` will only affect those platforms. Lists can be used +instead of strings for items that are naturally a list. Multiline strings also +work just like in the environment variables. Environment variables will take +precedence if defined. + +The example above using environment variables could have been written like this: + +```toml +[tool.cibuildwheel] +test-requires = "pytest" +test-command = "pytest ./tests" +``` + +The complete set of defaults for the current version of cibuildwheel are shown below: + +```toml +{% include "../cibuildwheel/resources/defaults.toml" %} +``` + + +!!! tip + Static configuration works across all CI systems, and can be used locally if + you run `cibuildwheel --platform linux`. This is preferred, but environment + variables are better if you need to change per-matrix element + (`CIBW_BUILD` is often in this category, for example), or if you cannot or do + not want to change a `pyproject.toml` file. You can specify a different file to + use with `--config-file` on the command line, as well. + +## Configuration overrides {: #overrides } + +One feature specific to the configuration files is the ability to override +settings based on selectors. To use, add a ``tool.cibuildwheel.overrides`` +array, and specify a ``select`` string. Then any options you set will only +apply to items that match that selector. These are applied in order, with later +matches overriding earlier ones if multiple selectors match. Environment +variables always override static configuration. + +A few of the options below have special handling in overrides. A different +`before-all` will trigger a new container to launch on Linux, and cannot be +overridden on macOS or Windows. Overriding the image on linux will also +trigger new containers, one per image. Some commands are not supported; +`output-dir`, build/skip/test_skip selectors, and architectures cannot be +overridden. + +You can specify a table of overrides in `inherit={}`, any list or table in this +list will inherit from previous overrides or the main configuration. The valid +options are `"none"` (the default), `"append"`, and `"prepend"`. + +#### Examples: + +```toml +[tool.cibuildwheel.linux] +before-all = "yum install mylib" +test-command = "echo 'installed'" + +[[tool.cibuildwheel.overrides]] +select = "*-musllinux*" +before-all = "apk add mylib" +``` + +This example will override the before-all command on musllinux only, but will +still run the test-command. Note the double brackets, this is an array in TOML, +which means it can be given multiple times. + +```toml +[tool.cibuildwheel] +# Normal options, etc. +manylinux-x86_64-image = "manylinux_2_34" + +[[tool.cibuildwheel.overrides]] +select = "cp38-*" +manylinux-x86_64-image = "manylinux2014" + +[[tool.cibuildwheel.overrides]] +select = "cp3{9,10}-*" +manylinux-x86_64-image = "manylinux_2_28" +``` + +This example will build CPython 3.8 wheels on manylinux2014, CPython 3.9-3.10 +wheels on manylinux_2_28, and manylinux_2_34 wheels for any newer Python +(like 3.10). + +```toml +[tool.cibuildwheel] +environment = {FOO="BAR", "HAM"="EGGS"} +test-command = ["pyproject"] + +[[tool.cibuildwheel.overrides]] +select = "cp311*" + +inherit.test-command = "prepend" +test-command = ["pyproject-before"] + +inherit.environment="append" +environment = {FOO="BAZ", "PYTHON"="MONTY"} + +[[tool.cibuildwheel.overrides]] +select = "cp311*" +inherit.test-command = "append" +test-command = ["pyproject-after"] +``` + +This example will provide the command `"pyproject-before && pyproject && pyproject-after"` +on Python 3.11, and will have `environment = {FOO="BAZ", "PYTHON"="MONTY", "HAM"="EGGS"}`. + + +## Extending existing options {: #inherit } + +In the TOML configuration, you can choose how tables and lists are inherited. +By default, all values are overridden completely (`"none"`) but sometimes you'd +rather `"append"` or `"prepend"` to an existing list or table. You can do this +with the `inherit` table in overrides. For example, if you want to add an environment +variable for CPython 3.11, without `inherit` you'd have to repeat all the +original environment variables in the override. With `inherit`, it's just: + +```toml +[[tool.cibuildwheel.overrides]] +select = "cp311*" +inherit.environment = "append" +environment.NEWVAR = "Added!" +``` + +For a table, `"append"` will replace a key if it exists, while `"prepend"` will +only add a new key, older keys take precedence. + +Lists are also supported (and keep in mind that commands are lists). For +example, you can print a message before and after a wheel is repaired: + +```toml +[[tool.cibuildwheel.overrides]] +select = "*" +inherit.repair-wheel-command = "prepend" +repair-wheel-command = "echo 'Before repair'" + +[[tool.cibuildwheel.overrides]] +select = "*" +inherit.repair-wheel-command = "append" +repair-wheel-command = "echo 'After repair'" +``` + +As seen in this example, you can have multiple overrides match - they match top +to bottom, with the config being accumulated. If you need platform-specific +inheritance, you can use `select = "*-????linux_*"` for Linux, `select = +"*-win_*"` for Windows, and `select = "*-macosx_*"` for macOS. As always, +environment variables will completely override any TOML configuration. diff --git a/docs/contributing.md b/docs/contributing.md index c6a5b9308..31c879a79 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -2,6 +2,8 @@ title: Contributing --- +# Contributing + Wheel-building can be pretty complex. We expect users to find edge-cases - please help the rest of the community out by documenting these, adding features to support them, and reporting bugs. If you have an idea for a modification or feature, it's probably best to raise an issue first and discuss it with the maintainer team. Once we have rough consensus on a design, begin work in a PR. @@ -10,17 +12,16 @@ If you have an idea for a modification or feature, it's probably best to raise a Everyone contributing to the cibuildwheel project is expected to follow the [PSF Code of Conduct](https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md). -Design Goals ------------- +## Design Goals - `cibuildwheel` should wrap the complexity of wheel building. - The user interface to `cibuildwheel` is the build script (e.g. `.travis.yml`). Feature additions should not increase the complexity of this script. -- Options should be environment variables (these lend themselves better to YML config files). They should be prefixed with `CIBW_`. -- Options should be generalise to all platforms. If platform-specific options are required, they should be namespaced e.g. `CIBW_TEST_COMMAND_MACOS` +- Options should be environment variables (these lend themselves better to YAML config files). They should be prefixed with `CIBW_`. +- Options should be generalised to all platforms. If platform-specific options are required, they should be namespaced e.g. `CIBW_TEST_COMMAND_MACOS` Other notes: -- The platforms are very similar, until they're not. I'd rather have straight-forward code than totally DRY code, so let's keep airy platform abstractions to a minimum. +- The platforms are very similar, until they're not. I'd rather have straightforward code than totally DRY code, so let's keep airy platform abstractions to a minimum. - I might want to break the options into a shared config file one day, so that config is more easily shared. That has motivated some of the design decisions. ### cibuildwheel's relationship with build errors @@ -31,41 +32,114 @@ We're not responsible for errors in those tools, for fixing errors/crashes there So, if we can, I'd like to improve the experience on errors as well. In [this](https://github.com/pypa/cibuildwheel/issues/139) case, it takes a bit of knowledge to understand that the Linux builds are happening in a different OS via Docker, that the linked symbols won't match, that auditwheel will fail because of this. A problem with how the tools fit together, instead of the tools themselves. -Maintainer notes ----------------- +## Development + +### Running the tests + +When making a change to the codebase, you can run tests locally for quicker feedback than the CI runs on a PR. You can run them directly, but the easiest way to run tests is using [nox](https://nox.thea.codes/). + +You can run all the tests locally by doing: + +```bash +nox -s tests +``` + +However, because this takes a while, you might prefer to be more specific. + +#### Unit tests + +To run the project's unit tests, do: + +```bash +nox -s tests -- unit_test +``` + +There are a few custom options to enable different parts of the test suite - check `nox -s tests -- unit_test --help` for details. + +If you're calling this a lot, you might consider using the `-r` or `-R` arguments to nox to make it a bit faster. This calls pytest under the hood, so to target a specific test, use pytest's `-k` option after the `--` above to select a specific test. + +#### Integration tests + +To run the project's integration tests, do: + +```bash +nox -s tests -- test +``` + +The integration test suite is big - it can take more than 30 minutes to run the whole thing. -### Nox support +Because it takes such a long time, normally you'd choose specific tests to run locally, and rely on the project's CI for the rest. Use pytest's `-k` option to choose specific tests. You can pass a test name or a filename, it'll run everything that matches. -Most developer tasks have a nox interface. This allows you to very simply run tasks without worrying about setting up a development environment (as shown below). This is a slower than setting up a development environment and reusing it, but has the (important) benefit of being highly reproducible; an earlier run does not affect a current run, or anything else on your machine. +```bash +nox -s tests -- test -k +# e.g. +nox -s tests -- test -k before_build +``` + +A few notes- + +- Because they run inside a container, Linux tests can run on all platforms where Docker is installed, so they're convenient for running integration tests locally. Set the `--platform` flag on pytest to do this: `nox -s tests -- test --platform linux`. + +- Running the macOS integration tests requires _system installs_ of Python from python.org for all the versions that are tested. We won't attempt to install these when running locally, but you can do so manually using the URL in the error message that is printed when the install is not found. + +- The ['enable groups'](options.md#enable) run by default are just 'cpython-prerelease' and 'cpython-freethreading'. You can add other groups like pypy or graalpy by passing the `--enable` argument to pytest, i.e. `nox -s tests -- test --enable pypy`. On GitHub PRs, you can add a label to the PR to enable these groups. + +#### Running pytest directly + +More advanced users might prefer to invoke pytest directly. Set up a [dev environment](#setting-up-a-dev-environment), then, + +```bash +# run the unit tests +pytest unit_test +# run the whole integration test suite +pytest test +# run a specific integration test +pytest test -k test_build_frontend_args +# run a specific integration test on a different platform +CIBW_PLATFORM=linux pytest test -k test_build_frontend_args +``` -Install [nox](https://nox.thea.codes); homebrew is recommend on macOS, otherwise, pipx is a great choice - in fact, you can use `pipx run nox` and avoid installing completely. +### Linting, docs + +Most developer tasks have a nox interface. This allows you to very simply run tasks without worrying about setting up a development environment (as shown below). This is slower than setting up a development environment and reusing it, but has the (important) benefit of being highly reproducible; an earlier run does not affect a current run, or anything else on your machine. You can see a list of sessions by typing `nox -l`; here are a few common ones: ```console -nox -s lint # Run the linters (default) -nox -s tests # Run the tests (default) -nox -s docs -- serve # Build and serve the documentation -nox -s build # Make SDist and wheel +nox -s lint # Run the linters (default) +nox -s tests [-- PYTEST-ARGS] # Run the tests (default) +nox -s docs # Build and serve the documentation +nox -s build # Make SDist and wheel ``` -More advanced users can run the update scripts. `update_pins` should work directly, but `update_constraints` needs all versions of Python installed. If you don't want to do that locally, a fast way to run it to use docker to run nox: +More advanced users can run the update scripts: ```console -docker run --rm -itv $PWD:/src -w /src quay.io/pypa/manylinux_2_24_x86_64:latest pipx run nox -s update_constraints +nox -s update_constraints # update all constraints files in cibuildwheel/resources +nox -s update_pins # update tools, python interpreters & docker images used by cibuildwheel ``` -### Local testing +### Setting up a dev environment -You should run: +A dev environment isn't required for any of the `nox` tasks above. However, a dev environment is still useful, to be able to point an editor at, and a few other jobs. -```console -python3 -m venv venv -. venv/bin/activate -pip install -e .[dev] +cibuildwheel uses dependency groups. Set up a dev environment with UV by doing + +```bash +uv sync +``` + +Or, if you're not using `uv`, you can do: + +```bash +python3 -m venv .venv +source .venv/bin/activate +pipx run dependency-groups dev | xargs pip install -e. ``` -To prepare a development environment. +Your virtualenv is at `.venv`. + +## Maintainer notes ### Testing sample configs @@ -94,7 +168,6 @@ This has been moved to using docker, so you only need the following instructions The dependency update script in the next section requires multiple python versions installed. One way to do this is to use `pyenv`: ```bash -pyenv install 3.6.11 pyenv install 3.7.8 # Optionally add 3.8 and make it the local version; # otherwise assuming 3.8+ already is your current python version @@ -103,7 +176,6 @@ pyenv install 3.7.8 Then, you need to make the required virtual environments: ```bash -$(pyenv prefix 3.6.11)/bin/python -m venv env36 $(pyenv prefix 3.7.8)/bin/python -m venv env37 ``` @@ -125,7 +197,7 @@ Then, increment the project version number using: bin/bump_version.py ``` -You'll be prompted to enter the new version number. Update the changelog when prompted. The script will create a 'bump version' commit and version tag. +(or `nox -s bump_version`) You'll be prompted to enter the new version number. Update the changelog when prompted. The script will create a 'bump version' commit and version tag. Finally, cut the release and push to GitHub. @@ -134,3 +206,5 @@ git push && git push --tags ``` Then head to https://github.com/pypa/cibuildwheel/releases and create a GitHub release from the new tag, pasting in the changelog entry. Once the release is created inside GitHub, a CI job will create the assets and upload them to PyPI. + +If there were any schema updates, run `pipx run ./bin/generate_schema.py --schemastore > partial-cibuildwheel.json` and contribute the changes to SchemaStore. diff --git a/docs/cpp_standards.md b/docs/cpp_standards.md index 1f1406181..b22b2ff72 100644 --- a/docs/cpp_standards.md +++ b/docs/cpp_standards.md @@ -2,23 +2,24 @@ title: Modern C++ standards --- +# Modern C++ standards + Building Python wheels with modern C++ standards (C++11 and later) requires a few tricks. -## manylinux1 and C++14 -The old `manylinux1` image (based on CentOS 5) contains a version of GCC and libstdc++ that only supports C++11 and earlier standards. There are however ways to compile wheels with the C++14 standard (and later): https://github.com/pypa/manylinux/issues/118 +## manylinux2014 and C++20 -`manylinux2010` and `manylinux2014` are newer and support all C++ standards (up to C++17). +The past end-of-life `manylinux2014` image (based on CentOS 7) contains a version of GCC and libstdc++ that only supports C++17 and earlier standards. -## macOS and deployment target versions +`manylinux_2_28` are newer and support all C++ standards (up to C++20). -OS X/macOS allows you to specify a so-called "deployment target" version that will ensure backwards compatibility with older versions of macOS. One way to do this is by setting the `MACOSX_DEPLOYMENT_TARGET` environment variable. +## macOS and deployment target versions -However, to enable modern C++ standards, the deploment target needs to be set high enough (since older OS X/macOS versions did not have the necessary modern C++ standard library). +The [`MACOSX_DEPLOYMENT_TARGET` environment variable](platforms.md#macos-version-compatibility) is used to set the minimum deployment target for macOS. -To get C++11 and C++14 support, `MACOSX_DEPLOYMENT_TARGET` needs to be set to (at least) `"10.9"`. By default, `cibuildwheel` already does this, building 64-bit-only wheels for macOS 10.9 and later. +However, to enable modern C++ standards, the deployment target needs to be set high enough (since older OS X/macOS versions did not have the necessary modern C++ standard library). -To get C++17 support, Xcode 9.3+ is needed, requiring at least macOS 10.13 on the build machine. To use C++17 library features and link against the C++ runtime library, set `MACOSX_DEPLOYMENT_TARGET` to `"10.13"` or `"10.14"` (or higher) - macOS 10.13 offers partial C++17 support (e.g., the filesystem header is in experimental, offering `#include ` instead of `#include `); macOS 10.14 has full C++17 support. +To get C++17 support, Xcode 9.3+ is needed, requiring at least macOS 10.13 on the build machine. To use C++17 library features and link against the C++ runtime library, set `MACOSX_DEPLOYMENT_TARGET` to `"10.13"` or `"10.14"` (or higher) - macOS 10.13 offers partial C++17 support (e.g., the filesystem header is in experimental, offering `#include ` instead of `#include `); macOS 10.14 has full C++17 support. CPython 3.12+ require 10.13+ anyway. However, if only C++17 compiler and standard template library (STL) features are used (not needing a C++17 runtime) it might be possible to set `MACOSX_DEPLOYMENT_TARGET` to a lower value, such as `"10.9"`. To find out if this is the case, try compiling and running with a lower `MACOSX_DEPLOYMENT_TARGET`: if C++17 features are used that require a more recent deployment target, building the wheel should fail. diff --git a/docs/data/how-it-works.png b/docs/data/how-it-works.png new file mode 100644 index 000000000..41d589904 Binary files /dev/null and b/docs/data/how-it-works.png differ diff --git a/docs/data/projects.schema.json b/docs/data/projects.schema.json new file mode 100644 index 000000000..452b52c52 --- /dev/null +++ b/docs/data/projects.schema.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/docs/data/projects.schema.json", + "type": "array", + "description": "The projects file for cibuildwheel", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { "type": "string" }, + "gh": { "type": "string", "pattern": "[^/]*/[^/]*" }, + "stars": { "$ref": "#/items/properties/gh" }, + "pypi": { "type": "string" }, + "notes": { "type": "string" }, + "ci": { + "type": "array", + "items": { + "enum": [ + "github", + "travisci", + "circleci", + "gitlab", + "cirrusci", + "azurepipelines" + ] + } + }, + "ci_config": { "type": "string" }, + "os": { + "type": "array", + "items": { "enum": ["windows", "apple", "linux"] } + } + }, + "required": ["name", "gh", "ci", "os"] + } +} diff --git a/docs/data/projects.yml b/docs/data/projects.yml index 48ea51437..e6ce488d9 100644 --- a/docs/data/projects.yml +++ b/docs/data/projects.yml @@ -4,23 +4,29 @@ # stars: GitHub repo (optional, if different from package, such as for Twisted) # pypi: The pypi name, if different from the GitHub package name # os: Operating system list, [windows, apple, linux] (optional) -# ci: [appveyor, github, azurepipelines, circleci, gitlab, travisci] (optional) +# ci: [github, azurepipelines, circleci, gitlab, travisci, cirrusci] (optional) # notes: (text, optional) +- name: abess + gh: abess-team/abess + notes: A fast best-subset selection library. It uses cibuildwheel to build a large project with C++ extensions. + ci: [github] + os: [windows, apple, linux] + - name: Matplotlib gh: matplotlib/matplotlib notes: The venerable Matplotlib, a Python library with C++ portions ci: [github] os: [windows, apple, linux] -- name: pyinstrument_cext - gh: joerick/pyinstrument_cext - ci: [travisci, appveyor] +- name: pyinstrument + gh: joerick/pyinstrument + ci: [github] os: [windows, apple, linux] - notes: A simple C extension, without external dependencies + notes: Python profiler with a C extension. No external dependencies. - name: websockets - gh: aaugustin/websockets + gh: python-websockets/websockets ci: [travisci] ci_config: .travis.yml os: [apple, linux] @@ -166,15 +172,17 @@ - name: pyzmq gh: zeromq/pyzmq - ci: [github] + ci: [github, circleci] os: [windows, apple, linux] ci_config: .github/workflows/wheels.yml notes: | - Python bindings for zeromq, the networking library. Uses Cython and CFFI. + Python bindings for zeromq, the networking library. + Uses Cython on CPython and CFFI on PyPy. + ARM wheels for linux are built natively on CircleCI. - name: python-rapidjson gh: python-rapidjson/python-rapidjson - ci: [travisci, gitlab, appveyor] + ci: [travisci, gitlab] os: [windows, linux] - name: jq.py @@ -198,6 +206,12 @@ ci: [github] os: [windows, apple, linux] +- name: pybind11 cross build example + gh: wbarnha/pybind_cmake_example_crossbuild + ci: [github, gitlab] + os: [windows, apple, linux] + notes: Same as pybind11 cmake_example but used to demo Linux ARM + Windows + macOS builds on GitLab + - name: iminuit gh: scikit-hep/iminuit ci: [github] @@ -281,7 +295,7 @@ pypi: mypy ci: [github] os: [apple, linux, windows] - notes: MyPyC, the compiled component of MyPy. + notes: The compiled version of MyPy using MyPyC. - name: Imagecodecs (fork) gh: czaki/imagecodecs_build @@ -311,12 +325,12 @@ notes: Full range of wheels for setuptools rust, with auto release and PyPI deploy. - name: python-snappy - gh: andrix/python-snappy + gh: intake/python-snappy ci: [github] os: [apple, linux, windows] - name: sourmash - gh: dib-lab/sourmash + gh: sourmash-bio/sourmash ci: [github] os: [apple, linux, windows] @@ -387,9 +401,9 @@ - name: pillow-heif gh: bigcat88/pillow_heif pypi: pillow-heif - ci: [github] - os: [apple, linux] - notes: Python CFFI binding to libheif library with third party dependencies like `libde265`, `x265`, `libaom` with test & publishing on PyPi. + ci: [github, cirrusci] + os: [apple, linux, windows] + notes: Bindings to libheif library with third party dependencies. Fully automated CI for tests and publishing including Apple Silicon builds. - name: clang-format gh: ssciwr/clang-format-wheel @@ -399,11 +413,12 @@ - name: Tornado gh: tornadoweb/tornado - ci: [travisci] - os: [apple, linux] + ci: [github] + os: [linux, apple, windows] + notes: Tornado is a Python web framework and asynchronous networking library. Uses stable ABI for a small C extension. - name: pytorch-fairseq - gh: pytorch/fairseq + gh: facebookresearch/fairseq ci: [github] os: [apple, linux] @@ -417,11 +432,6 @@ ci: [github] os: [apple, linux, windows] -- name: pydantic - gh: samuelcolvin/pydantic - ci: [github] - os: [apple, linux, windows] - - name: vaex gh: vaexio/vaex ci: [github] @@ -465,7 +475,7 @@ os: [apple, linux, windows] - name: OpenSpiel - gh: deepmind/open_spiel + gh: google-deepmind/open_spiel ci: [github] os: [apple, linux] @@ -495,7 +505,7 @@ os: [apple, linux, windows] - name: OpenTimelineIO - gh: PixarAnimationStudios/OpenTimelineIO + gh: AcademySoftwareFoundation/OpenTimelineIO ci: [github] os: [apple, linux, windows] @@ -510,3 +520,147 @@ notes: A modern implementation of a PostgreSQL adapter for Python ci: [github] os: [windows, apple, linux] + +- name: Arbor + gh: arbor-sim/arbor + ci: [github] + os: [apple, linux] + notes: > + Arbor is a multi-compartment neuron simulation library; compatible with + next-generation accelerators; best-practices applied to research software; + focused on community-driven development. Includes a + [small script](https://github.com/arbor-sim/arbor/blob/master/scripts/patchwheel.py) + patching `rpath` in bundled libraries. + +- name: Kivy + gh: kivy/kivy + ci: [github] + os: [windows, apple, linux] + +- name: NCNN + gh: Tencent/ncnn + ci: [github] + os: [windows, apple, linux] + +- name: Prophet + gh: facebook/prophet + ci: [github] + os: [windows, apple, linux] + +- name: MemRay + gh: bloomberg/memray + ci: [github] + os: [linux] + +- name: PyGame + gh: pygame/pygame + ci: [github] + os: [apple, linux] + +- name: UltraJSON + gh: ultrajson/ultrajson + ci: [github] + os: [windows, apple, linux] + +- name: NumPy + gh: numpy/numpy + ci: [github, travisci] + os: [windows, apple, linux] + +- name: Wrapt + gh: GrahamDumpleton/wrapt + ci: [github] + os: [windows, apple, linux] + +- name: SimpleJSON + gh: simplejson/simplejson + ci: [github] + os: [windows, apple, linux] + +- name: Implicit + gh: benfred/implicit + ci: [github] + os: [windows, apple, linux] + notes: Includes GPU support for linux wheels + +- name: power-grid-model + gh: PowerGridModel/power-grid-model + ci: [github] + os: [windows, apple, linux] + notes: Python/C++ library for distribution power system analysis + +- name: Python-WebRTC + gh: MarshalX/python-webrtc + ci: [github] + os: [windows, apple, linux] + +- name: cf-units + gh: SciTools/cf-units + ci: [github] + os: [apple, linux] + +- name: envd + gh: tensorchord/envd + ci: [github] + os: [apple, linux, windows] + notes: A machine learning development environment build tool + +- name: mosec + gh: mosecorg/mosec + ci: [github] + os: [linux, apple] + notes: A machine learning model serving framework powered by Rust + +- name: ril + gh: Cryptex-github/ril-py + ci: [github] + os: [windows, apple, linux] + pypi: pyril + notes: A python binding to Rust Imaging library using maturin and Pyo3, utilizes Github Action cache to improve speed. Builds abi3 wheels. + +- name: Picologging + gh: microsoft/picologging + ci: [github] + os: [windows, apple, linux] + pypi: picologging + notes: A high-performance logging library for Python. + +- name: aalink + gh: artfwo/aalink + ci: [github] + os: [windows, apple, linux] + pypi: aalink + notes: Async Python interface for Ableton Link. + +- name: werpy + gh: analyticsinmotion/werpy + ci: [github] + os: [windows, linux, apple] + pypi: werpy + notes: An ultra-fast python package using optimized dynamic programming to compute the Word Error Rate (WER). + +- name: keyvi + gh: KeyviDev/keyvi + ci: [github] + os: [linux, apple] + pypi: keyvi + notes: FST based key value index highly optimized for size and lookup performance, utilizes ccache action for improved runtime + +- name: pedalboard + gh: spotify/pedalboard + ci: [github] + os: [windows, linux, apple] + pypi: pedalboard + notes: > + A Python library for working with audio data and audio plugins + by wrapping the [JUCE](https://github.com/juce-framework/JUCE/) + C++ framework. Uses cibuildwheel to deploy on as many operating + systems and Python versions as possible with only one dependency + (any NumPy). + +- name: streaming-form-data + gh: siddhantgoel/streaming-form-data + ci: [github] + os: [apple, linux, windows] + pypi: streaming-form-data + notes: Streaming parser for multipart/form-data written in Cython diff --git a/docs/data/readme_icons/appveyor.svg b/docs/data/readme_icons/appveyor.svg deleted file mode 100644 index 6fa0bac27..000000000 --- a/docs/data/readme_icons/appveyor.svg +++ /dev/null @@ -1 +0,0 @@ -AppVeyor icon \ No newline at end of file diff --git a/docs/data/readme_icons/cirrusci.svg b/docs/data/readme_icons/cirrusci.svg new file mode 100644 index 000000000..c92d3fe7e --- /dev/null +++ b/docs/data/readme_icons/cirrusci.svg @@ -0,0 +1 @@ +Cirrus CI icon \ No newline at end of file diff --git a/docs/deliver-to-pypi.md b/docs/deliver-to-pypi.md index 610fe6e5d..edd59b384 100644 --- a/docs/deliver-to-pypi.md +++ b/docs/deliver-to-pypi.md @@ -2,51 +2,34 @@ title: Delivering to PyPI --- -After you've built your wheels, you'll probably want to deliver them to PyPI. - -## Manual method - -On your development machine, install [pipx](https://pypa.github.io/pipx/) and do the following: - -```bash -# Either download the SDist from your CI, or make it: -# Clear out your 'dist' folder. -rm -rf dist -# Make a source distribution -pipx run build --sdist - -# 🏃🏻 -# Go and download your wheel files from wherever you put them. e.g. your CI -# provider can be configured to store them for you. Put them all into the -# 'dist' folder. +# Delivering to PyPI -# Upload using 'twine' (you may need to 'pip install twine') -pipx run twine upload dist/* -``` +After you've built your wheels, you'll probably want to deliver them to PyPI. ## Automatic method If you don't need much control over the release of a package, you can set up -cibuildwheel to deliver the wheels straight to PyPI. You just need to bump the +your CI provider to deliver the wheels straight to PyPI. You just need to bump the version and tag it. -### Generic instructions +The exact way to set it up varies, depending on which CI provider you're using. But generally, the process goes like this: -Make your SDist with the [build](https://github.com/pypa/build) tool, and your wheels with cibuildwheel. If you can make the files available as -downloadable artifacts, this make testing before releases easier (depending on your CI provider's options). The "publish" job/step should collect the -files, and then run `twine upload ` (possibly via [pipx](https://github.com/pypa/pipx)); this should only happen on tags or "releases". +- Build your wheels with cibuildwheel +- Build an sdist with the [build](https://github.com/pypa/build) tool +- Check that the current CI run is happening during a release (e.g. it's in response to a vXX tag) +- Collect these assets together onto one runner +- Upload them to PyPI using `twine upload ` ### GitHub Actions -GitHub actions has pipx in all the runners as a supported package manager, as -well as several useful actions. Alongside your existing job(s) that runs cibuildwheel to make wheels, you will probably want to build an SDist: +GitHub actions has pipx in all the runners as a supported package manager, as well as `pypa/gh-action-pypi-publish`, which can be used instead of twine. Alongside your existing job(s) that runs cibuildwheel to make wheels, you will probably want to build an sdist: ```yaml make_sdist: name: Make SDist runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Optional, use if you use setuptools_scm submodules: true # Optional, use if you have submodules @@ -54,39 +37,37 @@ well as several useful actions. Alongside your existing job(s) that runs cibuild - name: Build SDist run: pipx run build --sdist - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-sdist path: dist/*.tar.gz ``` -Then, you need to publish the artifacts that the previous jobs have built. This final job should run only on release or tag, depending on your preference. It gathers the artifacts from the sdist and wheel jobs and uploads them to PyPI. - -This requires a [PyPI upload token](https://pypi.org/manage/account/token/), stored in your [GitHub repo's secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository) as `pypi_password`. +Then, you need to publish the artifacts that the previous jobs have built. This final job should run only on release or tag, depending on your preference. It gathers the artifacts from the sdist and wheel jobs and uploads them to PyPI. The release environment (`pypi` in the example below) will be created the first time this workflow runs. ```yaml upload_all: needs: [build_wheels, make_sdist] + environment: pypi + permissions: + id-token: write runs-on: ubuntu-latest if: github.event_name == 'release' && github.event.action == 'published' steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: - name: artifact + pattern: cibw-* path: dist + merge-multiple: true - - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.pypi_password }} + - uses: pypa/gh-action-pypi-publish@release/v1 ``` -You should use dependabot to keep the publish action up to date. In the above -example, the same name (the default, "artifact" is used for all upload-artifact -runs, so we can just download all of the in one step into a common directory. +The above example uses PyPI Trusted Publishing to deliver the wheels, which requires some configuration on the PyPI side for a [new project](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc) or an [existing project](https://docs.pypi.org/trusted-publishers/adding-a-publisher). You can use Dependabot to keep the publish action up to date. See [`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) -for an example configuration that automatically upload wheels to PyPI. Also see +for an example configuration that automatically uploads wheels to PyPI. Also see [scikit-hep.org/developer/gha_wheels](https://scikit-hep.org/developer/gha_wheels) for a complete guide. @@ -95,3 +76,23 @@ for a complete guide. See [`examples/travis-ci-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/travis-ci-deploy.yml) for an example configuration. + +## Manual method + +On your development machine, install [pipx](https://pipx.pypa.io/) and do the following: + +```bash +# Either download the SDist from your CI, or make it: +# Clear out your 'dist' folder. +rm -rf dist +# Make a source distribution +pipx run build --sdist + +# 🏃🏻 +# Go and download your wheel files from wherever you put them. e.g. your CI +# provider can be configured to store them for you. Put them all into the +# 'dist' folder. + +# Upload using 'twine' +pipx run twine upload dist/* +``` diff --git a/docs/diagram.html b/docs/diagram.html new file mode 100644 index 000000000..52c70b9e7 --- /dev/null +++ b/docs/diagram.html @@ -0,0 +1,487 @@ +
+
+ + +
+
+
+ +
+
+ Manylinux container +
+
+ +
+
Test virtualenv
+
+
+
+ + +
Linux
+
macOS
+
Windows
+ +
+
For each version of Python
+
+
+
If tests are configured
+
+ +
+ +
+
+
+
+
+
+
+ {{action.label}} +
+
+ +
+
+
+ + + + + + diff --git a/docs/extra.css b/docs/extra.css index e3599627e..770508a63 100644 --- a/docs/extra.css +++ b/docs/extra.css @@ -3,6 +3,8 @@ body { overflow-wrap: break-word; + overflow-x: hidden; + -webkit-font-smoothing: antialiased; } p { @@ -43,6 +45,42 @@ h1, h2, h3, h4, h5, h6 { border-bottom: 1px solid #e1e4e5; margin-bottom: 0.8em; padding-bottom: 0.2em; + scroll-margin: 1.5em 0; +} + +/* make it so tables can overflow the content area, make the most of the content area */ +.wy-nav-content-wrap { + container-type: inline-size; +} +.wy-table-responsive { + overflow-x: auto; + overflow-y: hidden; + position: relative; + left: -3.236em; + max-width: calc(100cqw); + width: fit-content; + min-width: min(calc(100% + 3.236em + 3.236em), 100cqw); +} +.rst-content .section .wy-table-responsive .docutils { + padding-left: 3.236em; + padding-right: 3.236em; +} +.rst-content .section .wy-table-responsive .docutils thead, +.rst-content .section .wy-table-responsive .docutils tbody { + background-color: white; +} +@media screen and (max-width: 768px) { + .wy-table-responsive { + left: -1em; + } + .rst-content .section .wy-table-responsive .docutils { + padding-left: 1em; + padding-right: 1em; + } +} + +.wy-nav-content { + max-width: 900px; } .rst-content blockquote { @@ -61,6 +99,13 @@ h1, h2, h3, h4, h5, h6 { border: none; } +@media screen and (max-width: 768px) { + .wy-nav-content-wrap .wy-nav-content { + padding: 1em; + } +} + + /* Code block filename style @@ -104,10 +149,10 @@ h1, h2, h3, h4, h5, h6 { .code-block-filename + pre code { padding-top: 0.9em; } - +/* .toctree-l3 { border-left: 10px solid transparent; -} +} */ /* import font awesome 4 for icons */ @import url(https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css); @@ -225,3 +270,51 @@ h1, h2, h3, h4, h5, h6 { padding-right: 0; margin-bottom: 0.5em; } + +/* mkdocs update theme fix-ups */ + +.rst-content code, .rst-content tt, code { + /* border: none; */ + /* background-color: #f0f1f1; */ + color: inherit; +} +.rst-content pre code { + border: none; +} + +.rst-content h1 code, +.rst-content h2 code, +.rst-content h3 code, +.rst-content h4 code, +.rst-content h5 code, +.rst-content h6 code { + border: none; + color: inherit; + background-color: #f0f1f1; + font-size: 80%; +} + +.rst-content table.docutils td code, +.rst-content table.docutils th code, +.rst-content table.field-list td code, +.rst-content table.field-list th code, +.wy-table td code, +.wy-table th code { + /* table elements are already made smaller, the code styling on top of that makes the text too small */ + font-size: 82.5%; +} + + +/* hide the l1 buttons */ +.wy-menu-vertical li.toctree-l1.current>a button.toctree-expand, +.wy-menu-vertical li.toctree-l1.on>a button.toctree-expand, +.wy-menu-vertical li.toctree-l1>a button.toctree-expand { + display: none; +} + + +/* word wrap in table cells */ +.wy-table-responsive table td, .wy-table-responsive table th { + white-space: normal; + line-height: 1.4; +} diff --git a/docs/extra.js b/docs/extra.js index 23bed98ce..c6013f535 100644 --- a/docs/extra.js +++ b/docs/extra.js @@ -58,3 +58,54 @@ while (true) { // this will catch infinite loops which can occur when editing the above if (tabConversionIterations++ > 1000) throw 'too many iterations' } + +/** + * Redirects the current page based on the path and fragment identifier (hash) in the URL. + * + * Example usage: + * fragmentRedirect([ + * { source: 'setup/#github-actions', destination: 'ci-services' } + * { source: 'faq/#macosx', destination: 'platforms#apple' } + * ]) + */ +function fragmentRedirect(redirects) { + const href = window.location.href; + const hash = window.location.hash; + + for (const redirect of redirects) { + const source = redirect.source; + const destination = redirect.destination; + + if (endswith(href, source)) { + // Redirect to the destination path, with the same fragment identifier + // specified in the destination path, otherwise, keep the same hash + // from the current URL. + const destinationIncludesHash = destination.includes('#'); + let newUrl = href.replace(source, destination); + if (!destinationIncludesHash) { + newUrl += hash; + } + console.log('Redirecting to:', newUrl); + window.location.replace(newUrl); + return + } + } +} + +function endswith(str, suffix) { + return str.indexOf(suffix, str.length - suffix.length) !== -1; +} + +fragmentRedirect([ + { source: 'setup/#github-actions', destination: 'ci-services/' }, + { source: 'setup/#azure-pipelines', destination: 'ci-services/' }, + { source: 'setup/#travis-ci', destination: 'ci-services/' }, + { source: 'setup/#appveyor', destination: 'ci-services/' }, + { source: 'setup/#circleci', destination: 'ci-services/' }, + { source: 'setup/#gitlab-ci', destination: 'ci-services/' }, + { source: 'setup/#cirrus-ci', destination: 'ci-services/' }, + + { source: 'faq/#linux-builds-in-containers', destination: 'platforms/#linux-containers' }, + { source: 'faq/#apple-silicon', destination: 'platforms/#macos-architectures' }, + { source: 'faq/#windows-arm64', destination: 'platforms/#windows-arm64' }, +]); diff --git a/docs/faq.md b/docs/faq.md index 82e45852b..25d9da73b 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -2,87 +2,11 @@ title: Tips and tricks --- -## Tips - -### Linux builds on Docker - -Linux wheels are built in the [`manylinux`/`musllinux` docker images](https://github.com/pypa/manylinux) to provide binary compatible wheels on Linux, according to [PEP 600](https://www.python.org/dev/peps/pep-0600/) / [PEP 656](https://www.python.org/dev/peps/pep-0656/). Because of this, when building with `cibuildwheel` on Linux, a few things should be taken into account: - -- Programs and libraries are not installed on the CI runner host, but rather should be installed inside of the Docker image - using `yum` for `manylinux2010` or `manylinux2014`, `apt-get` for `manylinux_2_24` and `apk` for `musllinux_1_1`, or manually. The same goes for environment variables that are potentially needed to customize the wheel building. - - `cibuildwheel` supports this by providing the [`CIBW_ENVIRONMENT`](options.md#environment) and [`CIBW_BEFORE_ALL`](options.md#before-all) options to setup the build environment inside the running Docker image. - -- The project directory is mounted in the running Docker instance as `/project`, the output directory for the wheels as `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that `/host` is not available on CircleCI due to their Docker policies. - -- Alternative Docker images can be specified with the `CIBW_MANYLINUX_*_IMAGE`/`CIBW_MUSLLINUX_*_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#linux-image) for more details. - -### Building macOS wheels for Apple Silicon {: #apple-silicon} - -`cibuildwheel` supports cross-compiling `universal2` and `arm64` wheels on `x86_64` runners. With the introduction of Apple Silicon, you now have several choices for wheels for Python 3.8+: - -#### `x86_64` - -The traditional wheel for Apple, loads on Intel machines, and on -Apple Silicon when running Python under Rosetta 2 emulation. - -Due to a change in naming, Pip 20.3+ (or an installer using packaging 20.5+) -is required to install a binary wheel on macOS Big Sur. - -#### `arm64` - -The native wheel for macOS on Apple Silicon. - -Requires Pip 20.3+ (or packaging 20.5+) to install. - -#### `universal2` - -This wheel contains both architectures, causing it to be up to twice the -size (data files do not get doubled, only compiled code). It requires -Pip 20.3 (Packaging 20.6+) to load on Intel, and Pip 21.0.1 (Packaging 20.9+) -to load on Apple Silicon. - -!!! note - The dual-architecture `universal2` has a few benefits, but a key benefit - to a universal wheel is that a user can bundle these wheels into an - application and ship a single binary. - - However, if you have a large library, then you might prefer to ship - the two single-arch wheels instead - `x86_64` and `arm64`. In rare cases, - you might want to build all three, but in that case, pip will not download - the universal wheels, because it prefers the most specific wheel - available. - -Generally speaking, because Pip 20.3 is required for the `universal2` wheel, -most packages should provide both `x86_64` and `universal2` wheels for now. -Once Pip 20.3+ is common on macOS, then it should be possible to ship only the -`universal2` wheel. - -**Apple Silicon wheels are not built by default**, but can be enabled by adding extra archs to the [`CIBW_ARCHS_MACOS` option](options.md#archs) - e.g. `x86_64 arm64 universal2`. Cross-compilation is provided by the Xcode toolchain. - -!!! important - When cross-compiling on Intel, it is not possible to test `arm64` and the `arm64` part of a `universal2` wheel. - - `cibuildwheel` will raise a warning to notify you of this - these warnings be be silenced by skipping testing on these platforms: `CIBW_TEST_SKIP: *_arm64 *_universal2:arm64`. - -Hopefully, cross-compilation is a temporary situation. Once we have widely -available Apple Silicon CI runners, we can build and test `arm64` and -`universal2` wheels natively. That's why `universal2` wheels are not yet built -by default, and require opt-in by setting `CIBW_ARCHS_MACOS`. +# Tips and tricks -!!! note - Your runner needs Xcode Command Line Tools 12.2 or later to build `universal2` or `arm64`. - - Only CPython 3.8 and newer support `universal2` and `arm64` wheels. - -Here's an example GitHub Actions workflow with a job that builds for Apple Silicon: - -> .github/workflows/build_macos.yml - -```yml -{% include "../examples/github-apple-silicon.yml" %} -``` +## Tips -### Building non-native architectures using emulation {: #emulation} +### Building Linux wheels for non-native archs using emulation {: #emulation} cibuildwheel supports building non-native architectures on Linux, via emulation through the binfmt_misc kernel feature. The easiest way to use this @@ -103,7 +27,15 @@ Linux), and the other architectures are emulated automatically. {% include "../examples/github-with-qemu.yml" %} ``` -### Building packages with optional C extensions +### Building CPython ABI3 wheels (Limited API) {: #abi3} + +The CPython Limited API is a subset of the Python C Extension API that's declared to be forward-compatible, meaning you can compile wheels for one version of Python, and they'll be compatible with future versions. Wheels that use the Limited API are known as ABI3 wheels. + +To create a package that builds ABI3 wheels, you'll need to configure your build backend to compile libraries correctly create wheels with the right tags. [Check this repo](https://github.com/joerick/python-abi3-package-sample) for an example of how to do this with setuptools. + +You could also consider running [abi3audit](https://github.com/trailofbits/abi3audit) against the produced wheels in order to check for abi3 violations or inconsistencies. You can run it alongside the default in your [repair-wheel-command](options.md#repair-wheel-command). + +### Packages with optional C extensions {: #optional-extensions} `cibuildwheel` defines the environment variable `CIBUILDWHEEL` to the value `1` allowing projects for which the C extension is optional to make it mandatory when building wheels. @@ -117,17 +49,68 @@ myextension = Extension( ) ``` -### Automatic updates {: #automatic-updates} +### Building with NumPy + +If using NumPy, there are a couple of things that can help. + +First, if you require the `numpy` package at build-time (some binding tools, like `pybind11` and `nanobind`, do not), then the backward compatibility for your `build-backend.build-requires` is a little complicated for Python <3.9: + +* NumPy <1.25: You must build with the oldest version of NumPy you want to support at runtime. +* NumPy 1.25 and 1.26: Anything you build will be compatible with 1.19+ by default, and you can set the minimum target to, for example, 1.22 with `#define NPY_TARGET_VERSION NPY_1_22_API_VERSION`. +* NumPy 2.x: You must build with NumPy 2 to support NumPy 2; otherwise the same as 1.25+. + +So the rule is: + +* Python <3.8: Use the oldest supported NumPy (via helper `oldest-supported-numpy` if you want) +* Python 3.9+: Use latest supported NumPy (2+). + +Second, there might be platforms you want to ship for that NumPy (or some other scientific Python libraries) are not shipping yet for. This is often true for beta candidates of new Python releases, for example. To work with this, you can use the Scientific Python Nightly wheels. Here's an example, depending on what frontend you use: + +!!! tab "pip based" + For frontends like `build` (the default) and `pip`: + + ```toml + [tool.cibuildwheel] + environment.PIP_ONLY_BINARY = "numpy" + environment.PIP_PREFER_BINARY = "1" + + [[tool.cibuildwheel.overrides]] + select = ["cp314*"] + inherit.environment = "append" + environment.PIP_EXTRA_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" + environment.PIP_PRERELEASE = "allow" + ``` + +!!! tab "uv based" + For frontends like `build[uv]`: + + ```toml + [tool.cibuildwheel] + environment.UV_ONLY_BINARY = "numpy" + environment.UV_PREFER_BINARY = "1" + + [[tool.cibuildwheel.overrides]] + select = ["cp314*"] + inherit.environment = "append" + environment.UV_INDEX = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" + environment.UV_INDEX_STRATEGY = "unsafe-best-match" + environment.UV_PRERELEASE = "allow" + ``` + +(Note the `*_ONLY_BINARY` variable also supports `":all:"`, and you don't need both that and `*_PREFER_BINARY`, you can use either one, depending on if you want a missing wheel to be a failure or an attempt to build in CI.) + +### Automatic updates using Dependabot {: #automatic-updates} + +Selecting a moving target (like the latest release) is generally a bad idea in CI. If something breaks, you can't tell whether it was your code or an upstream update that caused the breakage, and in a worst-case scenario, it could occur during a release. -Selecting a moving target (like the latest release) is generally a bad idea in CI. If something breaks, you can't tell whether it was your code or an upstream update that caused the breakage, and in a worse-case scenario, it could occur during a release. -There are two suggested methods for keeping cibuildwheel up to date that instead involve scheduled pull requests using GitHub's dependabot. +There are two suggested methods for keeping cibuildwheel up to date that instead involve scheduled pull requests using GitHub's Dependabot. #### Option 1: GitHub Action If you use GitHub Actions for builds, you can use cibuildwheel as an action: ```yaml -uses: pypa/cibuildwheel@v2.3.1 +uses: pypa/cibuildwheel@v3.0.0 ``` This is a composite step that just runs cibuildwheel using pipx. You can set command-line options as `with:` parameters, and use `env:` as normal. @@ -141,10 +124,6 @@ updates: directory: "/" schedule: interval: "weekly" - ignore: - # Optional: Official actions have moving tags like v1; - # if you use those, you don't need updates. - - dependency-name: "actions/*" ``` #### Option 2: Requirement files @@ -153,7 +132,7 @@ The second option, and the only one that supports other CI systems, is using a ` ```bash # requirements-cibw.txt -cibuildwheel==2.3.1 +cibuildwheel==3.0.0 ``` Then your install step would have `python -m pip install -r requirements-cibw.txt` in it. Your `.github/dependabot.yml` file could look like this: @@ -173,13 +152,13 @@ This will also try to update other pins in all requirement files, so be sure you ### Alternatives to cibuildwheel options {: #cibw-options-alternatives} cibuildwheel provides lots of opportunities to configure the build -environment. However, you might consider adding this build configuration into +environment. However, you might consider adding this build configuration into the package itself - in general, this is preferred, because users of your package 'sdist' will also benefit. #### Missing build dependencies {: #cibw-options-alternatives-deps} -If your build needs Python dependencies, rather than using CIBW_BEFORE_BUILD, it's best to add these to the +If your build needs Python dependencies, rather than using `before-build`, it's best to add these to the [`build-system.requires`](https://www.python.org/dev/peps/pep-0518/#build-system-table) section of your pyproject.toml. For example, if your project requires Cython to build, your pyproject.toml might include a section like this: @@ -188,7 +167,6 @@ to build, your pyproject.toml might include a section like this: [build-system] requires = [ "setuptools>=42", - "wheel", "Cython", ] @@ -201,59 +179,102 @@ You might need to run some other commands before building, like running a script that performs codegen or downloading some data that's not stored in your source tree. -Rather than using CIBW_BEFORE_ALL or CIBW_BEFORE_BUILD, you could incorporate +Rather than using `before-all` or `before-build`, you could incorporate these steps into your package's build process. For example, if you're using setuptools, you can add steps to your package's `setup.py` using a structure like this: - ```python - import subprocess - import setuptools - import setuptools.command.build_py +```python +import subprocess +import setuptools +import setuptools.command.build_py - class BuildPyCommand(setuptools.command.build_py.build_py): - """Custom build command.""" +class BuildPyCommand(setuptools.command.build_py.build_py): + """Custom build command.""" - def run(self): + def run(self): # your custom build steps here # e.g. # subprocess.run(['python', 'scripts/my_custom_script.py'], check=True) setuptools.command.build_py.build_py.run(self) - setuptools.setup( - cmdclass={ - 'build_py': BuildPyCommand, - }, - # Usual setup() args. - # ... - ) - ``` +setuptools.setup( + cmdclass={ + 'build_py': BuildPyCommand, + }, + # Usual setup() args. + # ... +) +``` #### Compiler flags Your build might need some compiler flags to be set through environment variables. Consider incorporating these into your package, for example, in `setup.py` using [`extra_compile_args` or -`extra_link_args`](https://docs.python.org/3/distutils/setupscript.html#other-options). - -### Python 2.7 / PyPy2 wheels - -See the [cibuildwheel version 1 docs](https://cibuildwheel.readthedocs.io/en/1.x/) for information about building Python 2.7 or PyPy2 wheels. There are lots of tricks and workaround there that are no longer required for Python 3 in cibuildwheel 2. +`extra_link_args`](https://setuptools.pypa.io/en/latest/userguide/ext_modules.html#setuptools.Extension). ## Troubleshooting If your wheel didn't compile, you might have a mistake in your config. -To quickly test your config without doing a git push and waiting for your code to build on CI, you can [test the Linux build in a local Docker container](setup.md#local). +To quickly test your config without doing a git push and waiting for your code to build on CI, you can [test the Linux build in a local Docker container](platforms.md#linux). ### Missing dependencies -You might need to install something on the build machine. You can do this with apt/yum, brew or choco, using the [`CIBW_BEFORE_ALL`](options.md#before-all) option. Or, for a Python dependency, consider [adding it to pyproject.toml](#cibw-options-alternatives-deps). +Sometimes a build will fail due to a missing dependency. + +**If the build is missing a Python package**, you should [add it to pyproject.toml](#cibw-options-alternatives-deps). + +**If you need a build tool** (e.g. cmake, automake, ninja), you can install it through a package manager like apt/yum, brew or choco, using the [`before-all`](options.md#before-all) option. + +**If your build is linking into a native library dependency**, you can build/install that in [`before-all`](options.md#before-all). However, on Linux, Mac (and Windows if you're using [delvewheel]), the library that you install will be bundled into the wheel in the [repair step]. So take care to ensure that + +- the bundled library doesn't accidentally increase the minimum system requirements (such as the minimum macOS version) +- the bundled library matches the architecture of the wheel you're building when cross-compiling + +This is particularly an issue on macOS, where de facto package manager Homebrew will install libraries that are compiled for the specific version of macOS that the build machine is running, rendering the wheels useless for any previous version. And brew will not install the right arch for cross compilation of Apple Silicon wheels. + +For these reasons, it's strongly recommended to not use brew for native library dependencies. Instead, we recommend compiling the library yourself. If you compile in the [`before-all`](options.md#before-all) step, cibuildwheel will have already set the appropriate `MACOSX_DEPLOYMENT_TARGET` env var, so the library will target the correct version of macOS. + +!!! tip + For build steps, Homebrew is still a great resource - you can [look up the build formula](https://formulae.brew.sh/) and use that as a starting point. + +[delvewheel]: https://github.com/adang1345/delvewheel +[repair step]: options.md#repair-wheel-command +[Homebrew]: https://brew.sh/ +[delocate]: https://github.com/matthew-brett/delocate + +### Building Rust wheels + +If you build Rust wheels, you need to download the Rust compilers in manylinux. +If you support 32-bit Windows, you need to add this as a potential target. You +can do this on GitHub Actions, for example, with: + +```yaml +CIBW_BEFORE_ALL_LINUX: curl -sSf https://sh.rustup.rs | sh -s -- -y +CIBW_BEFORE_ALL_WINDOWS: rustup target add i686-pc-windows-msvc +CIBW_ENVIRONMENT_LINUX: "PATH=$HOME/.cargo/bin:$PATH" +``` + +Rust's minimum macOS target is 10.12, while CPython supports 10.9 before +Python 3.12, so you'll need to raise the minimum: + +```toml +[tool.cibuildwheel.macos.environment] +MACOSX_DEPLOYMENT_TARGET = "10.12" +``` + +And Rust does not provide Cargo for musllinux 32-bit, so that needs to be +skipped: -### macOS: ModuleNotFoundError +```toml +[tool.cibuildwheel] +skip = ["*-musllinux_i686"] +``` -Calling cibuildwheel from a python3 script and getting a `ModuleNotFoundError`? Due to a (fixed) [bug](https://bugs.python.org/issue22490) in CPython, you'll need to [unset the `__PYVENV_LAUNCHER__` variable](https://github.com/pypa/cibuildwheel/issues/133#issuecomment-478288597) before activating a venv. +Also see [maturin-action](https://github.com/PyO3/maturin-action) which is optimized for Rust wheels, builds the non-Python Rust modules once, and can cross-compile (and can build 32-bit musl, for example). ### macOS: 'No module named XYZ' errors after running cibuildwheel @@ -270,46 +291,81 @@ python3 -m twine upload wheelhouse/*.whl This doesn't work because while `cibuildwheel` was running, it installed a few new versions of 'python3', so the `python3` run on line 3 isn't the same as the `python3` that ran on line 1. -Solutions to this vary, but the simplest is to install tools immediately before they're used: +Solutions to this vary, but the simplest is to use pipx: ```bash -python3 -m pip install cibuildwheel -python3 -m cibuildwheel --output-dir wheelhouse -python3 -m pip install twine -python3 -m twine upload wheelhouse/*.whl +# most runners have pipx preinstalled, but in case you don't +python3 -m pip install pipx + +pipx run cibuildwheel==3.0.0 --output-dir wheelhouse +pipx run twine upload wheelhouse/*.whl ``` ### macOS: Passing DYLD_LIBRARY_PATH to delocate -macOS has built-in [System Integrity protections](https://developer.apple.com/library/archive/documentation/Security/Conceptual/System_Integrity_Protection_Guide/RuntimeProtections/RuntimeProtections.html) which limits the use of `DYLD_LIBRARY_PATH` and `LD_LIBRARY_PATH` so that it does not automatically pass to children processes. This means if you set `DYLD_LIBRARY_PATH` before running cibuildwheel, or even set it in `CIBW_ENVIRONMENT`, it will be stripped out of the environment before delocate is called. +macOS has built-in [System Integrity protections](https://developer.apple.com/library/archive/documentation/Security/Conceptual/System_Integrity_Protection_Guide/RuntimeProtections/RuntimeProtections.html) which limits the use of `DYLD_LIBRARY_PATH` and `LD_LIBRARY_PATH` so that it does not automatically pass to children processes. This means if you set `DYLD_LIBRARY_PATH` before running cibuildwheel, or even set it in `environment`, it will be stripped out of the environment before delocate is called. -To work around this, use a different environment variable such as `REPAIR_LIBRARY_PATH` to store the library path, and set `DYLD_LIBRARY_PATH` in [`CIBW_REPAIR_WHEEL_COMMAND_MACOS`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command), like this: +To work around this, use a different environment variable such as `REPAIR_LIBRARY_PATH` to store the library path, and set `DYLD_LIBRARY_PATH` in [`macos.repair-wheel-command`](https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command), like this: !!! tab examples "Environment variables" ```yaml CIBW_REPAIR_WHEEL_COMMAND_MACOS: > - DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && - DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} + DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} ``` !!! tab examples "pyproject.toml" ```toml [tool.cibuildwheel.macos] - repair-wheel-command = [ - "DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel}", - "DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" - ] + repair-wheel-command = """\ + DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel \ + --require-archs {delocate_archs} -w {dest_dir} -v {wheel}\ + """ ``` See [#816](https://github.com/pypa/cibuildwheel/issues/816), thanks to @phoerious for reporting. +### macOS: Building CPython 3.8 wheels on arm64 + +If you're building on an arm64 runner, you might notice something strange about CPython 3.8 - unlike Python 3.9+, it's cross-compiled to arm64 from an x86_64 version of Python running under Rosetta emulation. This is because (despite the prevalence of arm64 versions of Python 3.8 from Apple and Homebrew) there is no officially supported Python.org installer of Python 3.8 for arm64. + +This is fine for simple C extensions, but for more complicated builds on arm64 it becomes an issue. + +So, if you want to build macOS arm64 wheels on an arm64 runner (e.g., `macos-14`) on Python 3.8, before invoking cibuildwheel, you should install a native arm64 Python 3.8 interpreter on the runner: + + +!!! tab "GitHub Actions" + + ```yaml + - uses: actions/setup-python@v5 + with: + python-version: 3.8 + if: runner.os == 'macOS' && runner.arch == 'ARM64' + ``` + +!!! tab "Generic" + + ```bash + curl -o /tmp/Python38.pkg https://www.python.org/ftp/python/3.8.10/python-3.8.10-macos11.pkg + sudo installer -pkg /tmp/Python38.pkg -target / + sh "/Applications/Python 3.8/Install Certificates.command" + ``` + +Then cibuildwheel will detect that it's installed and use it instead. However, you probably don't want to build x86_64 wheels on this Python, unless you're happy with them only supporting macOS 11+. + +### macOS: Library dependencies do not satisfy target MacOS + +Since delocate 0.11.0 there is added verification that the library binary dependencies match the target macOS version. This is to prevent the situation where a wheel platform tag is lower than the actual minimum macOS version required by the library. To resolve this error you need to build the library to the same macOS version as the target wheel (for example using `MACOSX_DEPLOYMENT_TARGET` environment variable). +Alternatively, you could set `MACOSX_DEPLOYMENT_TARGET` in `environment` to correctly label the wheel as incompatible with older macOS versions. + +This error may happen when you install a library using a package manager like Homebrew, which compiles the library for the macOS version of the build machine. This is not suitable for wheels, as the library will only work on the same macOS version as the build machine. You should compile the library yourself, or use a precompiled binary that matches the target macOS version. + ### Windows: 'ImportError: DLL load failed: The specific module could not be found' -Visual Studio and MSVC link the compiled binary wheels to the Microsoft Visual C++ Runtime. Normally, these are included with Python, but when compiling with a newer version of Visual Studio, it is possible users will run into problems on systems that do not have these runtime libraries installed. The solution is to ask users to download the corresponding Visual C++ Redistributable from the [Microsoft website](https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads) and install it. Since a Python installation normally includes these VC++ Redistributable files for [the version of the MSVC compiler used to compile Python](https://wiki.python.org/moin/WindowsCompilers), this is typically only a problem when compiling a Python C extension with a newer compiler. +Visual Studio and MSVC link the compiled binary wheels to the Microsoft Visual C++ Runtime. Normally, the C parts of the runtime are included with Python, but the C++ components are not. When compiling modules using C++, it is possible users will run into problems on systems that do not have the full set of runtime libraries installed. The solution is to ask users to download the corresponding Visual C++ Redistributable from the [Microsoft website](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist) and install it. -Additionally, Visual Studio 2019 started linking to an even newer DLL, `VCRUNTIME140_1.dll`, besides the `VCRUNTIME140.dll` that is included with recent Python versions (starting from Python 3.5; see [here](https://wiki.python.org/moin/WindowsCompilers) for more details on the corresponding Visual Studio & MSVC versions used to compile the different Python versions). To avoid this extra dependency on `VCRUNTIME140_1.dll`, the [`/d2FH4-` flag](https://devblogs.microsoft.com/cppblog/making-cpp-exception-handling-smaller-x64/) can be added to the MSVC invocations (check out [this issue](https://github.com/pypa/cibuildwheel/issues/423) for details and references). +Additionally, Visual Studio 2019 started linking to an even newer DLL, `VCRUNTIME140_1.dll`, besides the `VCRUNTIME140.dll` that is included with recent Python versions (starting from Python 3.5; see [here](https://wiki.python.org/moin/WindowsCompilers) for more details on the corresponding Visual Studio & MSVC versions used to compile the different Python versions). To avoid this extra dependency on `VCRUNTIME140_1.dll`, the [`/d2FH4-` flag](https://devblogs.microsoft.com/cppblog/making-cpp-exception-handling-smaller-x64/) can be added to the MSVC invocations (check out [this issue](https://github.com/pypa/cibuildwheel/issues/423) for details and references). CPython 3.8.3 and all versions after it have this extra DLL, so it is only needed for 3.8 and earlier. To add the `/d2FH4-` flag to a standard `setup.py` using `setuptools`, the `extra_compile_args` option can be used: @@ -323,4 +379,4 @@ To add the `/d2FH4-` flag to a standard `setup.py` using `setuptools`, the `extr ], ``` -To investigate the dependencies of a C extension (i.e., the `.pyd` file, a DLL in disguise) on Windows, [Dependency Walker](http://www.dependencywalker.com/) is a great tool. +To investigate the dependencies of a C extension (i.e., the `.pyd` file, a DLL in disguise) on Windows, [Dependency Walker](http://www.dependencywalker.com/) is a great tool. For diagnosing a failing import, the [dlltracer](https://pypi.org/project/dlltracer/) tool may also provide additional details. diff --git a/docs/index.md b/docs/index.md index 6cc7ccfd5..49f01bb48 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,6 +2,8 @@ title: Home --- +# cibuildwheel + {% include-markdown "../README.md" start="" @@ -9,3 +11,14 @@ title: Home %} To get started, head over to the [setup guide](setup.md). + +How it works +------------ + +This diagram summarises the steps that cibuildwheel takes on each platform to build your package's wheels. + +{% + include "diagram.html" +%} + +This isn't exhaustive, for a full list of the things cibuildwheel can do, check the [options](options.md) page. diff --git a/docs/main.py b/docs/main.py index b17982372..11556dec0 100644 --- a/docs/main.py +++ b/docs/main.py @@ -1,13 +1,30 @@ +import os +import re import subprocess +import sysconfig from typing import Any -# Requires Python 3.7+ +import rich.console +import rich.text def define_env(env: Any) -> None: "Hook function for mkdocs-macros" - @env.macro + @env.macro # type: ignore[misc] def subprocess_run(*args: str) -> str: "Run a subprocess and return the stdout" - return subprocess.run(args, check=True, capture_output=True, text=True).stdout + env = os.environ.copy() + scripts = sysconfig.get_path("scripts") + env.pop("NO_COLOR", None) + env["PATH"] = f"{scripts}{os.pathsep}{env.get('PATH', '')}" + env["PYTHON_COLORS"] = "1" + output = subprocess.run(args, check=True, capture_output=True, text=True, env=env).stdout + rich_text = rich.text.Text.from_ansi(output) + console = rich.console.Console(record=True, force_terminal=True) + console.print(rich_text) + page = console.export_html(inline_styles=True) + result = re.search(r"(.*?)", page, re.DOTALL | re.IGNORECASE) + assert result + txt = result.group(1).strip() + return txt.replace("code ", 'code class="nohighlight" ') diff --git a/docs/options.md b/docs/options.md index 46f2d24e1..bd99997cc 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1,316 +1,170 @@ -## Setting options - -cibuildwheel can either be configured using environment variables, or from -config file such as `pyproject.toml`. - -### Environment variables {: #environment-variables} - -Environment variables can be set in your CI config. For example, to configure -cibuildwheel to run tests, add the following YAML to your CI config file: - -!!! tab "GitHub Actions" - - > .github/workflows/*.yml ([docs](https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables)) (can be global, in job, or in step) - - ```yaml - env: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" - ``` - -!!! tab "Azure Pipelines" - - > azure-pipelines.yml ([docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables)) - - ```yaml - variables: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" - ``` - -!!! tab "Travis CI" - - > .travis.yml ([docs](https://docs.travis-ci.com/user/environment-variables/)) - - ```yaml - env: - global: - - CIBW_TEST_REQUIRES=pytest - - CIBW_TEST_COMMAND="pytest {project}/tests" - ``` - -!!! tab "AppVeyor" - - > appveyor.yml ([docs](https://www.appveyor.com/docs/build-configuration/#environment-variables)) - - ```yaml - environment: - global: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}\\tests" - ``` - -!!! tab "CircleCI" - - > .circleci/config.yml ([docs](https://circleci.com/docs/2.0/configuration-reference/#environment)) - - ```yaml - jobs: - job_name: - environment: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" - ``` - -!!! tab "Gitlab CI" - - > .gitlab-ci.yml ([docs](https://docs.gitlab.com/ee/ci/variables/README.html#create-a-custom-variable-in-gitlab-ciyml)) - - ```yaml - linux: - variables: - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: "pytest {project}/tests" - ``` - -### Configuration file {: #configuration-file} - -You can configure cibuildwheel with a config file, such as `pyproject.toml`. -Options have the same names as the environment variable overrides, but are -placed in `[tool.cibuildwheel]` and are lower case, with dashes, following -common [TOML][] practice. Anything placed in subsections `linux`, `windows`, -or `macos` will only affect those platforms. Lists can be used instead of -strings for items that are natually a list. Multiline strings also work just -like in in the environment variables. Environment variables will take -precedence if defined. - -The example above using environment variables could have been written like this: - -```toml -[tool.cibuildwheel] -test-requires = "pytest" -test-command = "pytest {project}/tests" -``` - -The complete set of defaults for the current version of cibuildwheel are shown below: - -```toml -{% include "../cibuildwheel/resources/defaults.toml" %} -``` - - -!!! tip - Static configuration works across all CI systems, and can be used locally if - you run `cibuildwheel --plat linux`. This is preferred, but environment - variables are better if you need to change per-matrix element - (`CIBW_BUILD` is often in this category, for example), or if you cannot or do - not want to change a `pyproject.toml` file. You can specify a different file to - use with `--config-file` on the command line, as well. - -### Configuration overrides {: #overrides } - -One feature specific to the configuration files is the ability to override -settings based on selectors. To use, add a ``tool.cibuildwheel.overrides`` -array, and specify a ``select`` string. Then any options you set will only -apply to items that match that selector. These are applied in order, with later -matches overriding earlier ones if multiple selectors match. Environment -variables always override static configuration. - -A few of the options below have special handling in overrides. A different -`before-all` will trigger a new docker launch on Linux, and cannot be -overridden on macOS or Windows. Overriding the image on linux will also -generate new docker launches, one per image. Some commands are not supported; -`output-dir`, build/skip/test_skip selectors, and architectures cannot be -overridden. - -##### Examples: - -```toml -[tool.cibuildwheel.linux] -before-all = "yum install mylib" -test-command = "echo 'installed'" - -[[tool.cibuildwheel.overrides]] -select = "*-musllinux*" -before-all = "apk add mylib" -``` - -This example will override the before-all command on musllinux only, but will -still run the test-command. Note the double brackets, this is an array in TOML, -which means it can be given multiple times. - -```toml -[tool.cibuildwheel] -# Normal options, etc. -manylinux-x86_64-image = "manylinux2014" - -[[tool.cibuildwheel.overrides]] -select = "cp36-*" -manylinux-x86_64-image = "manylinux1" - -[[tool.cibuildwheel.overrides]] -select = "cp3{7,8,9}-*" -manylinux-x86_64-image = "manylinux2010" -``` - -This example will build CPython 3.6 wheels on manylinux1, CPython 3.7-3.9 -wheels on manylinux2010, and manylinux2014 wheels for any newer Python -(like 3.10). - - -## Options summary +# Options
## Build selection -### `CIBW_PLATFORM` {: #platform} +### `platform` {: #platform cmd-line env-var } > Override the auto-detected target platform -Options: `auto` `linux` `macos` `windows` +Options: `auto` `linux` `macos` `windows` `ios` `pyodide` Default: `auto` -`auto` will auto-detect platform using environment variables, such as `TRAVIS_OS_NAME`/`APPVEYOR`/`CIRCLECI`. +`auto` will build wheels for the current platform. -- For `linux`, you need Docker running, on Linux, macOS, or Windows. +- For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. +- For `ios` you need to be running on macOS, with Xcode and the iOS simulator installed. +- For `pyodide`, you need a Linux or macOS machine. + +Check the [platforms](platforms.md) page for more information on platform requirements. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. !!! tip - You can use this option to locally debug your cibuildwheel config, instead of pushing to CI to test every change. For example: + You can use this option to locally debug your cibuildwheel config on Linux, instead of pushing to CI to test every change. For example: ```bash export CIBW_BUILD='cp37-*' - export CIBW_TEST_COMMAND='pytest {package}/tests' + export CIBW_TEST_COMMAND='pytest {project}/tests' cibuildwheel --platform linux . ``` - This is even more convenient if you store your cibuildwheel config in [`pyproject.toml`](#configuration-file). + Linux builds are the easiest to test locally, because all the build tools are supplied in the container, and they run exactly the same locally as in CI. + + This is even more convenient if you store your cibuildwheel config in [`pyproject.toml`](configuration.md#configuration-file). + + You can also run a single identifier with `--only `. This will + not require `--platform` or `--arch`, and will override any build/skip + configuration. -### `CIBW_BUILD`, `CIBW_SKIP` {: #build-skip} +### `build`, `skip` {: #build-skip toml env-var } > Choose the Python versions to build List of builds to build and skip. Each build has an identifier like `cp38-manylinux_x86_64` or `cp37-macosx_x86_64` - you can list specific ones to build and cibuildwheel will only build those, and/or list ones to skip and cibuildwheel won't try to build them. -When both options are specified, both conditions are applied and only builds with a tag that matches `CIBW_BUILD` and does not match `CIBW_SKIP` will be built. +When both options are specified, both conditions are applied and only builds with a tag that matches `build` and does not match `skip` will be built. When setting the options, you can use shell-style globbing syntax, as per [fnmatch](https://docs.python.org/3/library/fnmatch.html) with the addition of curly bracket syntax `{option1,option2}`, provided by [bracex](https://pypi.org/project/bracex/). All the build identifiers supported by cibuildwheel are shown below:
- -| | macOS | Windows | Linux Intel | Linux Other | -|--------------|------------------------------------------------------------------------|----------------------------------|-----------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Python 3.6 | cp36-macosx_x86_64 | cp36-win_amd64
cp36-win32 | cp36-manylinux_x86_64
cp36-manylinux_i686
cp36-musllinux_x86_64
cp36-musllinux_i686 | cp36-manylinux_aarch64
cp36-manylinux_ppc64le
cp36-manylinux_s390x
cp36-musllinux_aarch64
cp36-musllinux_ppc64le
cp36-musllinux_s390x | -| Python 3.7 | cp37-macosx_x86_64 | cp37-win_amd64
cp37-win32 | cp37-manylinux_x86_64
cp37-manylinux_i686
cp37-musllinux_x86_64
cp37-musllinux_i686 | cp37-manylinux_aarch64
cp37-manylinux_ppc64le
cp37-manylinux_s390x
cp37-musllinux_aarch64
cp37-musllinux_ppc64le
cp37-musllinux_s390x | -| Python 3.8 | cp38-macosx_x86_64
cp38-macosx_universal2
cp38-macosx_arm64 | cp38-win_amd64
cp38-win32 | cp38-manylinux_x86_64
cp38-manylinux_i686
cp38-musllinux_x86_64
cp38-musllinux_i686 | cp38-manylinux_aarch64
cp38-manylinux_ppc64le
cp38-manylinux_s390x
cp38-musllinux_aarch64
cp38-musllinux_ppc64le
cp38-musllinux_s390x | -| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32
cp39-win_arm64 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x | -| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x | -| PyPy3.7 v7.3 | pp37-macosx_x86_64 | pp37-win_amd64 | pp37-manylinux_x86_64
pp37-manylinux_i686 | pp37-manylinux_aarch64 | -| PyPy3.8 v7.3 | pp38-macosx_x86_64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | +| | macOS | Windows | Linux Intel | Linux Other | iOS | pyodide (WASM) | +|---------------|------------------------------------------------------------------------|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|----------------------| +| Python 3.8 | cp38-macosx_x86_64
cp38-macosx_universal2
cp38-macosx_arm64 | cp38-win_amd64
cp38-win32 | cp38-manylinux_x86_64
cp38-manylinux_i686
cp38-musllinux_x86_64
cp38-musllinux_i686 | cp38-manylinux_aarch64
cp38-manylinux_ppc64le
cp38-manylinux_s390x
cp38-manylinux_armv7l
cp38-manylinux_riscv64
cp38-musllinux_aarch64
cp38-musllinux_ppc64le
cp38-musllinux_s390x
cp38-musllinux_armv7l
cp38-musllinux_riscv64 | | | +| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32
cp39-win_arm64 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-manylinux_armv7l
cp39-manylinux_riscv64
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x
cp39-musllinux_armv7l
cp39-musllinux_riscv64 | | | +| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-manylinux_armv7l
cp310-manylinux_riscv64
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x
cp310-musllinux_armv7l
cp310-musllinux_riscv64 | | | +| Python 3.11 | cp311-macosx_x86_64
cp311-macosx_universal2
cp311-macosx_arm64 | cp311-win_amd64
cp311-win32
cp311-win_arm64 | cp311-manylinux_x86_64
cp311-manylinux_i686
cp311-musllinux_x86_64
cp311-musllinux_i686 | cp311-manylinux_aarch64
cp311-manylinux_ppc64le
cp311-manylinux_s390x
cp311-manylinux_armv7l
cp311-manylinux_riscv64
cp311-musllinux_aarch64
cp311-musllinux_ppc64le
cp311-musllinux_s390x
cp311-musllinux_armv7l
cp311-musllinux_riscv64 | | | +| Python 3.12 | cp312-macosx_x86_64
cp312-macosx_universal2
cp312-macosx_arm64 | cp312-win_amd64
cp312-win32
cp312-win_arm64 | cp312-manylinux_x86_64
cp312-manylinux_i686
cp312-musllinux_x86_64
cp312-musllinux_i686 | cp312-manylinux_aarch64
cp312-manylinux_ppc64le
cp312-manylinux_s390x
cp312-manylinux_armv7l
cp312-manylinux_riscv64
cp312-musllinux_aarch64
cp312-musllinux_ppc64le
cp312-musllinux_s390x
cp312-musllinux_armv7l
cp312-musllinux_riscv64 | | cp312-pyodide_wasm32 | +| Python 3.13 | cp313-macosx_x86_64
cp313-macosx_universal2
cp313-macosx_arm64 | cp313-win_amd64
cp313-win32
cp313-win_arm64 | cp313-manylinux_x86_64
cp313-manylinux_i686
cp313-musllinux_x86_64
cp313-musllinux_i686 | cp313-manylinux_aarch64
cp313-manylinux_ppc64le
cp313-manylinux_s390x
cp313-manylinux_armv7l
cp313-manylinux_riscv64
cp313-musllinux_aarch64
cp313-musllinux_ppc64le
cp313-musllinux_s390x
cp313-musllinux_armv7l
cp313-musllinux_riscv64 | cp313-ios_arm64_iphoneos
cp313-ios_arm64_iphonesimulator
cp313-ios_x86_64_iphonesimulator | cp313-pyodide_wasm32 | +| Python 3.14 | cp314-macosx_x86_64
cp314-macosx_universal2
cp314-macosx_arm64 | cp314-win_amd64
cp314-win32
cp314-win_arm64 | cp314-manylinux_x86_64
cp314-manylinux_i686
cp314-musllinux_x86_64
cp314-musllinux_i686 | cp314-manylinux_aarch64
cp314-manylinux_ppc64le
cp314-manylinux_s390x
cp314-manylinux_armv7l
cp314-manylinux_riscv64
cp314-musllinux_aarch64
cp314-musllinux_ppc64le
cp314-musllinux_s390x
cp314-musllinux_armv7l
cp314-musllinux_riscv64 | | | +| PyPy3.8 v7.3 | pp38-macosx_x86_64
pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | | | +| PyPy3.9 v7.3 | pp39-macosx_x86_64
pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | | | +| PyPy3.10 v7.3 | pp310-macosx_x86_64
pp310-macosx_arm64 | pp310-win_amd64 | pp310-manylinux_x86_64
pp310-manylinux_i686 | pp310-manylinux_aarch64 | | | +| PyPy3.11 v7.3 | pp311-macosx_x86_64
pp311-macosx_arm64 | pp311-win_amd64 | pp311-manylinux_x86_64
pp311-manylinux_i686 | pp311-manylinux_aarch64 | | | +| GraalPy 3.11 v24.2 | gp311_242-macosx_x86_64
gp311_242-macosx_arm64 | gp311_242-win_amd64 | gp311_242-manylinux_x86_64 | gp311_242-manylinux_aarch64 | | | The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to cibuildwheel. The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). -For CPython, the minimally supported macOS version is 10.9; for PyPy 3.7, macOS 10.13 or higher is required. - Windows arm64 platform support is experimental. +Linux riscv64 platform support is experimental and requires an explicit opt-in through [`enable`](#enable). -See the [cibuildwheel 1 documentation](https://cibuildwheel.readthedocs.io/en/1.x/) for past end of life versions of Python, and PyPy2.7. +See the [cibuildwheel 2 documentation](https://cibuildwheel.pypa.io/en/2.x/) for past end-of-life versions of Python. #### Examples -!!! tab examples "Environment variables" - - ```yaml - # Only build on CPython 3.6 - CIBW_BUILD: cp36-* +!!! tab examples "pyproject.toml" - # Skip building on CPython 3.6 on the Mac - CIBW_SKIP: cp36-macosx_x86_64 + ```toml + [tool.cibuildwheel] + # Only build on CPython 3.8 + build = "cp38-*" # Skip building on CPython 3.8 on the Mac - CIBW_SKIP: cp38-macosx_x86_64 + skip = "cp38-macosx_x86_64" - # Skip building on CPython 3.6 on all platforms - CIBW_SKIP: cp36-* + # Skip building on CPython 3.8 on all platforms + skip = "cp38-*" - # Skip CPython 3.6 on Windows - CIBW_SKIP: cp36-win* + # Skip CPython 3.8 on Windows + skip = "cp38-win*" - # Skip CPython 3.6 on 32-bit Windows - CIBW_SKIP: cp36-win32 + # Skip CPython 3.8 on 32-bit Windows + skip = "cp38-win32" - # Skip CPython 3.6 and CPython 3.7 - CIBW_SKIP: cp36-* cp37-* + # Skip CPython 3.8 and CPython 3.9 + skip = ["cp38-*", "cp39-*"] - # Skip Python 3.6 on Linux - CIBW_SKIP: cp36-manylinux* + # Skip Python 3.8 on Linux + skip = "cp38-manylinux*" # Skip 32-bit builds - CIBW_SKIP: "*-win32 *-manylinux_i686" + skip = ["*-win32", "*-manylinux_i686"] # Disable building PyPy wheels on all platforms - CIBW_SKIP: pp* + skip = "pp*" ``` - Separate multiple selectors with a space. - -!!! tab examples "pyproject.toml" - - ```toml - [tool.cibuildwheel] - # Only build on CPython 3.6 - build = "cp36-*" +!!! tab examples "Environment variables" - # Skip building on CPython 3.6 on the Mac - skip = "cp36-macosx_x86_64" + ```yaml + # Only build on CPython 3.8 + CIBW_BUILD: cp38-* # Skip building on CPython 3.8 on the Mac - skip = "cp38-macosx_x86_64" + CIBW_SKIP: cp38-macosx_x86_64 - # Skip building on CPython 3.6 on all platforms - skip = "cp36-*" + # Skip building on CPython 3.8 on all platforms + CIBW_SKIP: cp38-* - # Skip CPython 3.6 on Windows - skip = "cp36-win*" + # Skip CPython 3.8 on Windows + CIBW_SKIP: cp38-win* - # Skip CPython 3.6 on 32-bit Windows - skip = "cp36-win32" + # Skip CPython 3.8 on 32-bit Windows + CIBW_SKIP: cp38-win32 - # Skip CPython 3.6 and CPython 3.7 - skip = ["cp36-*", "cp37-*"] + # Skip CPython 3.8 and CPython 3.9 + CIBW_SKIP: cp38-* cp39-* - # Skip Python 3.6 on Linux - skip = "cp36-manylinux*" + # Skip Python 3.8 on Linux + CIBW_SKIP: cp38-manylinux* # Skip 32-bit builds - skip = ["*-win32", "*-manylinux_i686"] + CIBW_SKIP: "*-win32 *-manylinux_i686" # Disable building PyPy wheels on all platforms - skip = "pp*" + CIBW_SKIP: pp* ``` + Separate multiple selectors with a space. + + + It is generally recommended to set `CIBW_BUILD` as an environment variable, though `skip` tends to be useful in a config file; you can statically declare that you don't - support pypy, for example. + support a specific build, for example. -### `CIBW_ARCHS` {: #archs} + +### `archs` {: #archs cmd-line env-var toml } > Change the architectures built on your machine by default. A list of architectures to build. -On macOS, this option can be used to cross-compile between `x86_64`, -`universal2` and `arm64` for Apple Silicon support. +On macOS, this option can be used to [cross-compile](platforms.md#macos-architectures) between `x86_64`, `universal2` and `arm64`. + +On Linux, this option can be used to build [non-native architectures under emulation](faq.md#emulation). -On Linux, this option can be used to build non-native architectures under -emulation. See [this guide](faq.md#emulation) for more information. +On Windows, this option can be used to [compile for `ARM64` from an Intel machine](platforms.md#windows-arm64), provided the cross-compiling tools are installed. Options: -- Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x` +- Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x` `armv7l` `riscv64` - macOS: `x86_64` `arm64` `universal2` -- Windows: `AMD64` `x86` -- `auto`: The default archs for your machine - see the table below. - - `auto64`: Just the 64-bit auto archs - - `auto32`: Just the 32-bit auto archs -- `native`: the native arch of the build machine - Matches [`platform.machine()`](https://docs.python.org/3/library/platform.html#platform.machine). +- Windows: `AMD64` `x86` `ARM64` +- Pyodide: `wasm32` +- iOS: `arm64_iphoneos` `arm64_iphonesimulator` `x86_64_iphonesimulator` +- `auto`: The recommended archs for your machine - see the table below. +- `auto64`: The 64-bit arch(s) supported by your machine (includes device and simulator for iOS) +- `auto32`: The 32-bit arch supported by your machine +- `native`: the native arch of the build machine - matches [`platform.machine()`](https://docs.python.org/3/library/platform.html#platform.machine). - `all` : expands to all the architectures supported on this OS. You may want - to use [CIBW_BUILD](#build-skip) with this option to target specific + to use [`build`](#build-skip) with this option to target specific architectures via build selectors. +Linux riscv64 platform support is experimental and requires an explicit opt-in through [`enable`](#enable). + Default: `auto` | Runner | `native` | `auto` | `auto64` | `auto32` | |---|---|---|---|---| -| Linux / Intel | `x86_64` | `x86_64` `i686` | `x86_64` | `i686` | -| Windows / Intel | `AMD64` | `AMD64` `x86` | `AMD64` | `x86` | +| Linux / Intel 64-bit | `x86_64` | `x86_64` | `x86_64` | `i686` | +| Linux / Intel 32-bit | `i686` | `i686` | | `i686` | +| Linux / Arm 64-bit | `aarch64` | `aarch64` | `aarch64` | `armv7l`¹ | +| Linux / Arm 32-bit | `armv7l` | `armv7l` | | `armv7l` | +| Windows / Intel 64-bit | `AMD64` | `AMD64` `x86` | `AMD64` | `x86` | +| Windows / Intel 32-bit | `x86` | `x86` | | `x86` | +| Windows / ARM64 | `ARM64` | `ARM64` | `ARM64` | | | macOS / Intel | `x86_64` | `x86_64` | `x86_64` | | -| macOS / Apple Silicon | `arm64` | `arm64` `universal2` | `arm64` `universal2`| | +| macOS / Apple Silicon | `arm64` | `arm64` | `arm64` | | +| iOS on macOS / Intel | `x86_64_iphonesimulator` | `x86_64_iphonesimulator` | `x86_64_iphonesimulator` | | +| iOS on macOS / Apple Silicon | `arm64_iphonesimulator` | `arm64_iphoneos` `arm64_iphonesimulator` | `arm64_iphoneos` `arm64_iphonesimulator` | + +¹: This will only be included if the runner supports it. If not listed above, `auto` is the same as `native`. +!!! warning + The `auto` option only includes 32-bit architectures if they are + commonly built. `cibuildwheel` 3.0 removed 32-bit Linux builds from `auto`, + and a future release may remove 32-bit Windows builds from `auto` as well. + If you know you need them, please include `auto32`. Note that modern + manylinux image do not support 32-bit builds. If you want to avoid 32-bit + Windows builds today, feel free to use `auto64`. + +!!! note + Pyodide currently ignores the architecture setting, as it always builds for + `wasm32`. + [setup-qemu-action]: https://github.com/docker/setup-qemu-action [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt Platform-specific environment variables are also available:
- `CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX` + `CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX` | `CIBW_ARCHS_IOS` This option can also be set using the [command-line option](#command-line) `--archs`. This option cannot be set in an `overrides` section in `pyproject.toml`. #### Examples -!!! tab examples "Environment variables" +!!! tab examples "pyproject.toml" - ```yaml + ```toml # Build `universal2` and `arm64` wheels on an Intel runner. # Note that the `arm64` wheel and the `arm64` part of the `universal2` # wheel cannot be tested in this configuration. - CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" + [tool.cibuildwheel.macos] + archs = ["x86_64", "universal2", "arm64"] # On an Linux Intel runner with qemu installed, build Intel and ARM wheels - CIBW_ARCHS_LINUX: "auto aarch64" - ``` + [tool.cibuildwheel.linux] + archs = ["auto", "aarch64"] - Separate multiple archs with a space. + # Build all 32-bit and 64-bit wheels natively buildable on the image + [tool.cibuildwheel] + archs = ["auto64", "auto32"] + ``` -!!! tab examples "pyproject.toml" +!!! tab examples "Environment variables" - ```toml + ```yaml # Build `universal2` and `arm64` wheels on an Intel runner. # Note that the `arm64` wheel and the `arm64` part of the `universal2` # wheel cannot be tested in this configuration. - [tool.cibuildwheel.macos] - archs = ["x86_64", "universal2", "arm64"] + CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" # On an Linux Intel runner with qemu installed, build Intel and ARM wheels - [tool.cibuildwheel.linux] - archs = ["auto", "aarch64"] + CIBW_ARCHS_LINUX: "auto aarch64" + + # Build all 32-bit and 64-bit wheels natively buildable on the image + CIBW_ARCHS: "auto64 auto32" ``` + Separate multiple archs with a space. + It is generally recommended to use the environment variable or command-line option for Linux, as selecting archs often depends on your specific runner having qemu installed. -### `CIBW_PROJECT_REQUIRES_PYTHON` {: #requires-python} +### `project-requires-python` {: #requires-python env-var} > Manually set the Python compatibility of your project By default, cibuildwheel reads your package's Python compatibility from -`pyproject.toml` following [PEP621](https://www.python.org/dev/peps/pep-0621/) +`pyproject.toml` following the [project metadata specification](https://packaging.python.org/en/latest/specifications/declaring-project-metadata/) or from `setup.cfg`; finally it will try to inspect the AST of `setup.py` for a simple keyword assignment in a top level function call. If you need to override this behaviour for some reason, you can use this option. When setting this option, the syntax is the same as `project.requires-python`, -using 'version specifiers' like `>=3.6`, according to +using 'version specifiers' like `>=3.8`, according to [PEP440](https://www.python.org/dev/peps/pep-0440/#version-specifiers). Default: reads your package's Python compatibility from `pyproject.toml` @@ -415,160 +302,279 @@ Default: reads your package's Python compatibility from `pyproject.toml` `setup.py` `setup(python_requires="...")`. If not found, cibuildwheel assumes the package is compatible with all versions of Python that it can build. - !!! note - Rather than using this option, it's recommended you set - `project.requires-python` in `pyproject.toml` instead: - Example `pyproject.toml`: + Rather than using this environment variable, it's recommended you set this value + statically in a way that your build backend can use it, too. This ensures + that your package's metadata is correct when published on PyPI. This + cibuildwheel-specific option is provided as an override, and therefore is only + available in environment variable form. + + - If you have a `pyproject.toml` containing a `[project]` table, you can + specify `requires-python` there. + ```toml [project] - requires-python = ">=3.6" + ... + requires-python = ">=3.8" + ``` - # Aside - in pyproject.toml you should always specify minimal build - # system options, like this: + Note that not all build backends fully support using a `[project]` table yet; + specifically setuptools just added experimental support in version 61. + Adding `[project]` to `pyproject.toml` requires all the other supported + values to be specified there, or to be listed in `dynamic`. - [build-system] - requires = ["setuptools>=42", "wheel"] - build-backend = "setuptools.build_meta" + - If you're using setuptools, [you can set this value in `setup.cfg` (preferred) or `setup.py`](https://setuptools.pypa.io/en/latest/userguide/dependency_management.html#python-requirement) + and cibuildwheel will read it from there. + +#### Examples + +!!! tab examples "Environment variables" + + ```yaml + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.8" + ``` + +### `enable` {: #enable toml env-var} +> Enable building with extra categories of selectors present. + +This option lets you opt-in to non-default builds, like pre-releases and +free-threaded Python. These are not included by default to give a nice default +for new users, but can be added to the selectors available here. The allowed +values are: + + +- `cpython-prerelease`: Enables beta versions of Pythons if any are available + (May-July, approximately). +- `cpython-freethreading`: [PEP 703](https://www.python.org/dev/peps/pep-0703) + introduced variants of CPython that can be built without the Global + Interpreter Lock (GIL). Those variants are also known as free-threaded / + no-gil. This will enable building these wheels while they are experimental. + The build identifiers for those variants have a `t` suffix in their + `python_tag` (e.g. `cp313t-manylinux_x86_64`). +- `pypy`: Enable PyPy. +- `pypy-eol`: Enable PyPy versions that have passed end of life (if still available). +- `cpython-experimental-riscv64`: Enable experimental riscv64 builds. Those builds + are disabled by default as they can't be uploaded to PyPI and a PEP will most likely + be required before this can happen. +- `graalpy`: Enable GraalPy. +- `pyodide-prerelease`: Pyodide versions that haven't released yet, if one is + available. Safe if you are shipping a site with an early build, not for + general distribution. +- `all`: Enable all of the above. +!!! caution + `cpython-prerelease` is provided for testing purposes only. It is not + recommended to distribute wheels built with beta releases, such as + uploading to PyPI. Please _do not_ upload these wheels to PyPI (except for + pre-releases), as they are not guaranteed to work with the final Python + release. Once Python is ABI stable and enters the release candidate phase, + that version of Python will become available without this flag. - Currently, setuptools has not yet added support for reading this value from - pyproject.toml yet, and so does not copy it to Requires-Python in the wheel - metadata. This mechanism is used by pip to scan through older versions of - your package until it finds a release compatible with the current version - of Python compatible when installing, so it is an important value to set if - you plan to drop support for a version of Python in the future. +!!! note + Free threading is experimental: [What’s New In Python 3.13](https://docs.python.org/3.13/whatsnew/3.13.html#free-threaded-cpython) - If you don't want to list this value twice, you can also use the setuptools - specific location in `setup.cfg` and cibuildwheel will detect it from - there. Example `setup.cfg`: +Default: empty. - [options] - python_requires = ">=3.6" +This option doesn't support overrides or platform specific variants; it is +intended as a way to acknowledge that a project is aware that these extra +selectors exist. If you need to enable/disable it per platform or python +version, set this option to `true` and use +[`build`](#build-skip)/[`skip`](#build-skip) options to filter the +builds. +Unlike all other cibuildwheel options, the environment variable setting will +only add to the TOML config; you can't remove an enable by setting an empty or +partial list in environment variables; use `CIBW_SKIP` instead. This way, if +you apply `cpython-prerelease` during the beta period using `CIBW_ENABLE` +without disabling your other enables. -This option is not available in `pyproject.toml` under -`tool.cibuildwheel.project-requires-python`, since it should be set with the -[PEP621](https://www.python.org/dev/peps/pep-0621/) location instead, -`project.requires-python`. #### Examples +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # Enable free-threaded support + enable = ["cpython-freethreading"] + + # Skip building free-threaded compatible wheels on Windows + enable = ["cpython-freethreading"] + skip = "*t-win*" + + # Include all PyPy versions + enable = ["pypy", "pypy-eol"] + ``` + + !!! tab examples "Environment variables" ```yaml - CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" + # Include latest Python beta + CIBW_ENABLE: cpython-prerelease + + # Include free-threaded support + CIBW_ENABLE: cpython-freethreading + + # Include both + CIBW_ENABLE: cpython-prerelease cpython-freethreading + + # Skip building free-threaded compatible wheels on Windows + CIBW_ENABLE: cpython-freethreading + CIBW_SKIP: *t-win* + + # Include all PyPy versions + CIBW_ENABLE = pypy pypy-eol ``` -### `CIBW_PRERELEASE_PYTHONS` {: #prerelease-pythons} -> Enable building with pre-release versions of Python if available -During the beta period, when new versions of Python are being tested, -cibuildwheel will often gain early support for beta releases. If you would -like to test wheel building with these versions, you can enable this flag. -!!! caution - This option is provided for testing purposes only. It is not - recommended to distribute wheels built when `CIBW_PRERELEASE_PYTHONS` is - set, such as uploading to PyPI. Please _do not_ upload these wheels to - PyPI, as they are not guaranteed to work with the final Python release. - Once Python is ABI stable and enters the release candidate phase, that - version of Python will become available without this flag. +### `allow-empty` {: #allow-empty cmd-line env-var} +> Suppress the error code if no wheels match the specified build identifiers + +When none of the specified build identifiers match any available versions, +cibuildwheel will typically return error code 3, indicating that there are +no wheels to build. Enabling this option will suppress this error, allowing +the build process to complete without signaling an error. -Default: Off (0) if Python is available in beta phase. No effect otherwise. +Default: Off (0). Error code 3 is returned when no builds are selected. -This option can also be set using the [command-line option](#command-line) `--prerelease-pythons`. This option is not available in the `pyproject.toml` config. +This option can also be set using the [command-line option](#command-line) +`--allow-empty`. This option is not available in the `pyproject.toml` config. #### Examples !!! tab examples "Environment variables" ```yaml - # Include latest Python beta - CIBW_PRERELEASE_PYTHONS: True + # Prevent an error code if the build does not match any wheels + CIBW_ALLOW_EMPTY: True ``` ## Build customization -### `CIBW_BUILD_FRONTEND` {: #build-frontend} -> Set the tool to use to build, either "pip" (default for now) or "build" +### `build-frontend` {: #build-frontend toml env-var} +> Set the tool to use to build, either "build" (default), "build\[uv\]", or "pip" -Choose which build backend to use. Can either be "pip", which will run -`python -m pip wheel`, or "build", which will run `python -m build --wheel`. +Options: -!!! tip - Until v2.0.0, [pip] was the only way to build wheels, and is still the - default. However, we expect that at some point in the future, cibuildwheel - will change the default to [build], in line with the PyPA's recommendation. - If you want to try `build` before this, you can use this option. +- `build[;args: ...]` +- `build[uv][;args: ...]` +- `pip[;args: ...]` + +Default: `build` + +Choose which build frontend to use. + +You can use "build\[uv\]", which will use an external [uv][] everywhere +possible, both through `--installer=uv` passed to build, as well as when making +all build and test environments. This will generally speed up cibuildwheel. +Make sure you have an external uv on Windows and macOS, either by +pre-installing it, or installing cibuildwheel with the uv extra, +`cibuildwheel[uv]`. You cannot use uv currently on Windows for ARM, for +musllinux on s390x, or for iOS, as binaries are not provided by uv. Legacy dependencies like +setuptools on Python < 3.12 and pip are not installed if using uv. + +Pyodide ignores this setting, as only "build" is supported. + +You can specify extra arguments to pass to the build frontend using the +optional `args` option. + +!!! warning + If you are using `build[uv]` and are passing `--no-isolation` or `-n`, we + will detect this and avoid passing `--installer=uv` to build, but still + install all packages with uv. We do not currently detect combined short + options, like `-xn`! [pip]: https://pip.pypa.io/en/stable/cli/pip_wheel/ [build]: https://github.com/pypa/build/ +[uv]: https://github.com/astral-sh/uv #### Examples +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # Switch to using pip + build-frontend = "pip" + + # supply an extra argument to 'pip wheel' + build-frontend = { name = "pip", args = ["--no-build-isolation"] } + + # Use uv and build + build-frontend = "build[uv]" + + # Use uv and build with an argument + build-frontend = { name = "build[uv]", args = ["--no-isolation"] } + ``` + !!! tab examples "Environment variables" ```yaml - # Switch to using build - CIBW_BUILD_FRONTEND: "build" - - # Ensure pip is used even if the default changes in the future + # Switch to using pip CIBW_BUILD_FRONTEND: "pip" - ``` -!!! tab examples "pyproject.toml" + # supply an extra argument to 'pip wheel' + CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation" - ```toml - [tool.cibuildwheel] - # Switch to using build - build-frontend = "build" + # Use uv and build + CIBW_BUILD_FRONTEND: "build[uv]" - # Ensure pip is used even if the default changes in the future - build-frontend = "pip" + # Use uv and build with an argument + CIBW_BUILD_FRONTEND: "build[uv]; args: --no-isolation" ``` -### `CIBW_ENVIRONMENT` {: #environment} -> Set environment variables needed during the build -A list of environment variables to set during the build. Bash syntax should be used, even on Windows. +### `config-settings` {: #config-settings env-var toml} +> Specify config-settings for the build backend. -You must set this variable to pass variables to Linux builds (since they execute in a Docker container). It also works for the other platforms. +Specify config settings for the build backend. Each space separated +item will be passed via `--config-setting`. In TOML, you can specify +a table of items, including arrays. -You can use `$PATH` syntax to insert other variables, or the `$(pwd)` syntax to insert the output of other shell commands. +!!! tip + Currently, "build" supports arrays for options, but "pip" only supports + single values. -To specify more than one environment variable, separate the assignments by spaces. +Platform-specific environment variables also available:
+`CIBW_CONFIG_SETTINGS_MACOS` | `CIBW_CONFIG_SETTINGS_WINDOWS` | `CIBW_CONFIG_SETTINGS_LINUX` | `CIBW_CONFIG_SETTINGS_IOS` | `CIBW_CONFIG_SETTINGS_PYODIDE` -Platform-specific environment variables are also available:
-`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` #### Examples +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel.config-settings] + --build-option = "--use-mypyc" + ``` + !!! tab examples "Environment variables" ```yaml - # Set some compiler flags - CIBW_ENVIRONMENT: CFLAGS='-g -Wall' CXXFLAGS='-Wall' + CIBW_CONFIG_SETTINGS: "--build-option=--use-mypyc" + ``` - # Append a directory to the PATH variable (this is expanded in the build environment) - CIBW_ENVIRONMENT: PATH=$PATH:/usr/local/bin - # Prepend a directory containing spaces on Windows. - CIBW_ENVIRONMENT_WINDOWS: > - PATH="C:\\Program Files\\PostgreSQL\\13\\bin;$PATH" - # Set BUILD_TIME to the output of the `date` command - CIBW_ENVIRONMENT: BUILD_TIME="$(date)" - # Supply options to `pip` to affect how it downloads dependencies - CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.myorg.com/simple +### `environment` {: #environment env-var toml} +> Set environment variables - # Set two flags on linux only - CIBW_ENVIRONMENT_LINUX: BUILD_TIME="$(date)" SAMPLE_TEXT="sample text" - ``` +A list of environment variables to set during the build and test phases. Bash syntax should be used, even on Windows. - Separate multiple values with a space. +You must use this variable to pass variables to Linux builds, since they execute in a container. It also works for the other platforms. + +You can use `$PATH` syntax to insert other variables, or the `$(pwd)` syntax to insert the output of other shell commands. + +To specify more than one environment variable, separate the assignments by spaces. + +Platform-specific environment variables are also available:
+`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` | `CIBW_ENVIRONMENT_IOS` | `CIBW_ENVIRONMENT_PYODIDE` + +#### Examples !!! tab examples "pyproject.toml" @@ -593,6 +599,10 @@ Platform-specific environment variables are also available:
# Supply options to `pip` to affect how it downloads dependencies environment = { PIP_EXTRA_INDEX_URL="https://pypi.myorg.com/simple" } + # Any pip command-line option can be set using the PIP_ prefix + # https://pip.pypa.io/en/stable/topics/configuration/#environment-variables + environment = { PIP_GLOBAL_OPTION="build_ext -j4" } + # Set two flags on linux only [tool.cibuildwheel.linux] environment = { BUILD_TIME="$(date)", SAMPLE_TEXT="sample text" } @@ -603,32 +613,55 @@ Platform-specific environment variables are also available:
SAMPLE_TEXT = "sample text" ``` - In configuration mode, you can use a [TOML][] table instead of a raw string as shown above. + In configuration files, you can use a [TOML][] table instead of a raw string as shown above. -!!! note - cibuildwheel always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions). +!!! tab examples "Environment variables" -### `CIBW_ENVIRONMENT_PASS_LINUX` {: #environment-pass} -> Set environment variables on the host to pass-through to the container during the build. + ```yaml + # Set some compiler flags + CIBW_ENVIRONMENT: CFLAGS='-g -Wall' CXXFLAGS='-Wall' -A list of environment variables to pass into the linux container during the build. It has no affect on the other platforms, which can already access all environment variables directly. + # Append a directory to the PATH variable (this is expanded in the build environment) + CIBW_ENVIRONMENT: PATH=$PATH:/usr/local/bin -To specify more than one environment variable, separate the variable names by spaces. + # Prepend a directory containing spaces on Windows. + CIBW_ENVIRONMENT_WINDOWS: > + PATH="C:\\Program Files\\PostgreSQL\\13\\bin;$PATH" -#### Examples + # Set BUILD_TIME to the output of the `date` command + CIBW_ENVIRONMENT: BUILD_TIME="$(date)" -!!! tab examples "Environment passthrough" + # Supply options to `pip` to affect how it downloads dependencies + CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.myorg.com/simple - ```yaml - # Export a variable - CIBW_ENVIRONMENT_PASS_LINUX: CFLAGS + # Any pip command-line options can be set using the PIP_ prefix + # https://pip.pypa.io/en/stable/topics/configuration/#environment-variables + CIBW_ENVIRONMENT: PIP_GLOBAL_OPTION="build_ext -j4" - # Set two flags variables - CIBW_ENVIRONMENT_PASS_LINUX: BUILD_TIME SAMPLE_TEXT + # Set two flags on linux only + CIBW_ENVIRONMENT_LINUX: BUILD_TIME="$(date)" SAMPLE_TEXT="sample text" ``` Separate multiple values with a space. +!!! note + cibuildwheel always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#optional-extensions). + +!!! note + To do its work, cibuildwheel sets the variables `VIRTUALENV_PIP`, `DIST_EXTRA_CONFIG`, `SETUPTOOLS_EXT_SUFFIX`, `PIP_DISABLE_PIP_VERSION_CHECK`, `PIP_ROOT_USER_ACTION`, and it extends the variables `PATH` and `PIP_CONSTRAINT`. Your assignments to these options might be replaced or extended. + +### `environment-pass` {: #environment-pass env-var="CIBW_ENVIRONMENT_PASS_LINUX" toml} +> Set environment variables on the host to pass-through to the container. + +A list of environment variables to pass into the linux container during each build and test. It has no effect on the other platforms, which can already access all environment variables directly. + +To specify more than one environment variable, separate the variable names by spaces. + +!!! note + cibuildwheel automatically passes the environment variable [`SOURCE_DATE_EPOCH`](https://reproducible-builds.org/docs/source-date-epoch/) if defined. + +#### Examples + !!! tab examples "pyproject.toml" ```toml @@ -641,47 +674,43 @@ To specify more than one environment variable, separate the variable names by sp environment-pass = ["BUILD_TIME", "SAMPLE_TEXT"] ``` - In configuration mode, you can use a [TOML][] list instead of a raw string as shown above. + In configuration files, you can use a [TOML][] list instead of a raw string as shown above. -### `CIBW_BEFORE_ALL` {: #before-all} -> Execute a shell command on the build system before any wheels are built. - -Shell command to prepare a common part of the project (e.g. build or install libraries which does not depend on the specific version of Python). - -This option is very useful for the Linux build, where builds take place in isolated Docker containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both `x86_64` and `i686` wheels (the default), your build uses two different Docker images. In that case, this command will execute twice - once per build container. +!!! tab examples "Environment variables" + + ```yaml + # Export a variable + CIBW_ENVIRONMENT_PASS_LINUX: CFLAGS + + # Set two flags variables + CIBW_ENVIRONMENT_PASS_LINUX: BUILD_TIME SAMPLE_TEXT + ``` + + Separate multiple values with a space. + +### `before-all` {: #before-all env-var toml} +> Execute a shell command on the build system before any wheels are built. + +Shell command that runs before any builds are run, to build or install parts that do not depend on the specific version of Python. + +This option is very useful for the Linux build, where builds take place in isolated containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both `x86_64` and `i686` wheels (the default), your build uses two different container images. In that case, this command will execute twice - once per build container. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by cibuildwheel. -On Windows and macOS, the version of Python available inside `CIBW_BEFORE_ALL` is whatever is available on the host machine. On Linux, a modern Python version is available on PATH. +On Windows and macOS, the version of Python available inside `before-all` is whatever is available on the host machine. On Linux, a modern Python version is available on PATH. This option has special behavior in the overrides section in `pyproject.toml`. -On linux, overriding it triggers a new docker launch. It cannot be overridden +On linux, overriding it triggers a new container launch. It cannot be overridden on macOS and Windows. Platform-specific environment variables also available:
-`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` - -#### Examples +`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` | `CIBW_BEFORE_ALL_IOS` | `CIBW_BEFORE_ALL_PYODIDE` -!!! tab examples "Environment variables" - - ```yaml - # Build third party library - CIBW_BEFORE_ALL: make -C third_party_lib - - # Install system library - CIBW_BEFORE_ALL_LINUX: yum install -y libffi-devel +!!! note - # Chain multiple commands using && and > in a YAML file, like: - CIBW_BEFORE_ALL: > - yum install bzip2 -y && - make third_party - ``` + This command is executed in a different Python environment from the builds themselves. So you can't `pip install` a Python dependency in `before-all` and use it in the build. Instead, look at [`before-build`](#before-build), or, if your project uses pyproject.toml, the [build-system.requires](https://peps.python.org/pep-0518/#build-system-table) field. - For multiline commands, see the last example. The character `>` means that - whitespace is collapsed to a single line, and '&&' between each command - ensures that errors are not ignored. [Further reading on multiline YAML - here.](https://yaml-multiline.info). +#### Examples !!! tab examples "pyproject.toml" @@ -703,12 +732,33 @@ Platform-specific environment variables also available:
In configuration files, you can use a TOML array, and each line will be run sequentially - joined with `&&`. -Note that manylinux2_24 builds occur inside a Debian9 docker, where -manylinux2010 and manylinux2014 builds occur inside a CentOS one. So for -`manylinux2_24` the `CIBW_BEFORE_ALL_LINUX` command must use `apt-get -y` +!!! tab examples "Environment variables" + + ```yaml + # Build third party library + CIBW_BEFORE_ALL: make -C third_party_lib + + # Install system library + CIBW_BEFORE_ALL_LINUX: yum install -y libffi-devel + + # Chain multiple commands using && and > in a YAML file, like: + CIBW_BEFORE_ALL: > + yum install bzip2 -y && + make third_party + ``` + + For multiline commands, see the last example. The character `>` means that + whitespace is collapsed to a single line, and '&&' between each command + ensures that errors are not ignored. [Further reading on multiline YAML + here.](https://yaml-multiline.info). + + +Note that `manylinux_2_31` builds occur inside a Debian derivative docker +container, where `manylinux2014` builds occur inside a CentOS one. So for +`manylinux_2_31` the `before-all` command must use `apt-get -y` instead. -### `CIBW_BEFORE_BUILD` {: #before-build} +### `before-build` {: #before-build env-var toml} > Execute a shell command preparing each wheel's build A shell command to run before building the wheel. This option allows you to run a command in **each** Python environment before the `pip wheel` command. This is useful if you need to set up some dependency so it's available during the build. @@ -720,26 +770,10 @@ The active Python binary can be accessed using `python`, and pip with `pip`; cib The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
- `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` + `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` | `CIBW_BEFORE_BUILD_IOS` | `CIBW_BEFORE_BUILD_PYODIDE` #### Examples -!!! tab examples "Environment variables" - - ```yaml - # Install something required for the build (you might want to use pyproject.toml instead) - CIBW_BEFORE_BUILD: pip install pybind11 - - # Chain commands using && - CIBW_BEFORE_BUILD_LINUX: python scripts/install-deps.py && make clean - - # Run a script that's inside your project - CIBW_BEFORE_BUILD: bash scripts/prepare_for_build.sh - - # If cibuildwheel is called with a package_dir argument, it's available as {package} - CIBW_BEFORE_BUILD: "{package}/script/prepare_for_build.sh" - ``` - !!! tab examples "pyproject.toml" ```toml @@ -763,9 +797,27 @@ Platform-specific environment variables are also available:
before-build = "{package}/script/prepare_for_build.sh" ``` - In configuration mode, you can use a array, and the items will be joined with `&&`. In TOML, using a single-quote string will avoid escapes - useful for + In configuration files, you can use a array, and the items will be joined + with `&&`. In TOML, using a single-quote string will avoid escapes - useful for Windows paths. +!!! tab examples "Environment variables" + + ```yaml + # Install something required for the build (you might want to use pyproject.toml instead) + CIBW_BEFORE_BUILD: pip install pybind11 + + # Chain commands using && + CIBW_BEFORE_BUILD_LINUX: python scripts/install-deps.py && make clean + + # Run a script that's inside your project + CIBW_BEFORE_BUILD: bash scripts/prepare_for_build.sh + + # If cibuildwheel is called with a package_dir argument, it's available as {package} + CIBW_BEFORE_BUILD: "{package}/script/prepare_for_build.sh" + ``` + + !!! note If you need Python dependencies installed for the build, we recommend using `pyproject.toml`'s `build-system.requires` instead. This is an example @@ -774,10 +826,8 @@ Platform-specific environment variables are also available:
[build-system] requires = [ "setuptools>=42", - "wheel", "Cython", - "numpy==1.13.3; python_version<'3.5'", - "oldest-supported-numpy; python_version>='3.5'", + "numpy", ] build-backend = "setuptools.build_meta" @@ -785,23 +835,62 @@ Platform-specific environment variables are also available:
This [PEP 517][]/[PEP 518][] style build allows you to completely control the build environment in cibuildwheel, [PyPA-build][], and pip, doesn't force downstream users to install anything they don't need, and lets you do - more complex pinning (Cython, for example, requires a wheel to be built - with an equal or earlier version of NumPy; pinning in this way is the only - way to ensure your module works on all available NumPy versions). + more complex pinning. [PyPA-build]: https://pypa-build.readthedocs.io/en/latest/ [PEP 517]: https://www.python.org/dev/peps/pep-0517/ [PEP 518]: https://www.python.org/dev/peps/pep-0517/ +### `xbuild-tools` {: #xbuild-tools env-var toml} +> Binaries on the path that should be included in an isolated cross-build environment. + +When building in a cross-platform environment, it is sometimes necessary to isolate the ``PATH`` so that binaries from the build machine don't accidentally get linked into the cross-platform binary. However, this isolation process will also hide tools that might be required to build your wheel. + +If there are binaries present on the `PATH` when you invoke cibuildwheel, and those binaries are required to build your wheels, those binaries can be explicitly included in the isolated cross-build environment using `xbuild-tools`. The binaries listed in this setting will be linked into an isolated location, and that isolated location will be put on the `PATH` of the isolated environment. You do not need to provide the full path to the binary - only the executable name that would be found by the shell. + +If you declare a tool as a cross-build tool, and that tool cannot be found in the runtime environment, an error will be raised. + +If you do not define `xbuild-tools`, and you build for a platform that uses a cross-platform environment, a warning will be raised. If your project does not require any cross-build tools, you can set `xbuild-tools` to an empty list to silence this warning. + +*Any* tool used by the build process must be included in the `xbuild-tools` list, not just tools that cibuildwheel will invoke directly. For example, if your build invokes `cmake`, and the `cmake` script invokes `magick` to perform some image transformations, both `cmake` and `magick` must be included in your safe tools list. + +Platform-specific environment variables are also available on platforms that use cross-platform environment isolation:
+ `CIBW_XBUILD_TOOLS_IOS` + +#### Examples + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # Allow access to the cmake and rustc binaries in the isolated cross-build environment. + xbuild-tools = ["cmake", "rustc"] -### `CIBW_REPAIR_WHEEL_COMMAND` {: #repair-wheel-command} -> Execute a shell command to repair each (non-pure Python) built wheel + # No cross-build tools are required + xbuild-tools = [] + ``` + +!!! tab examples "Environment variables" + + ```yaml + # Allow access to the cmake and rustc binaries in the isolated cross-build environment. + CIBW_XBUILD_TOOLS: cmake rustc + + # No cross-build tools are required + CIBW_XBUILD_TOOLS: + ``` + + +### `repair-wheel-command` {: #repair-wheel-command env-var toml} +> Execute a shell command to repair each built wheel Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` -- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}'` +- on macOS: `'delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}'` - on Windows: `''` +- on iOS: `''` +- on Pyodide: `''` A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. The command is run on each built wheel (except for pure Python ones) before testing it. @@ -815,7 +904,7 @@ The following placeholders must be used inside the command and will be replaced The command is run in a shell, so you can run multiple commands like `cmd1 && cmd2`. Platform-specific environment variables are also available:
-`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` +`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` | `CIBW_REPAIR_WHEEL_COMMAND_IOS` | `CIBW_REPAIR_WHEEL_COMMAND_PYODIDE` !!! tip cibuildwheel doesn't yet ship a default repair command for Windows. @@ -826,26 +915,13 @@ Platform-specific environment variables are also available:
[Delvewheel]: https://github.com/adang1345/delvewheel -#### Examples - -!!! tab examples "Environment variables" - - ```yaml - # Use delvewheel on windows - CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" - CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" - - # Don't repair macOS wheels - CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" - - # Pass the `--lib-sdir .` flag to auditwheel on Linux - CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} {wheel}" +!!! tip + When using `--platform pyodide`, `pyodide build` is used to do the build, + which already uses `auditwheel-emscripten` to repair the wheel, so the default + repair command is empty. If there is a way to do this in two steps in the future, + this could change. - # Multi-line example - use && to join on all platforms - CIBW_REPAIR_WHEEL_COMMAND: > - python scripts/repair_wheel.py -w {dest_dir} {wheel} && - python scripts/check_repaired_wheel.py -w {dest_dir} {wheel} - ``` +#### Examples !!! tab examples "pyproject.toml" @@ -869,91 +945,131 @@ Platform-specific environment variables are also available:
'python scripts/repair_wheel.py -w {dest_dir} {wheel}', 'python scripts/check_repaired_wheel.py -w {dest_dir} {wheel}', ] + + # Use abi3audit to catch issues with Limited API wheels + [tool.cibuildwheel.linux] + repair-wheel-command = [ + "auditwheel repair -w {dest_dir} {wheel}", + "pipx run abi3audit --strict --report {wheel}", + ] + [tool.cibuildwheel.macos] + repair-wheel-command = [ + "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}", + "pipx run abi3audit --strict --report {wheel}", + ] + [tool.cibuildwheel.windows] + repair-wheel-command = [ + "copy {wheel} {dest_dir}", + "pipx run abi3audit --strict --report {wheel}", + ] ``` - In configuration mode, you can use an inline array, and the items will be joined with `&&`. + In configuration files, you can use an inline array, and the items will be joined with `&&`. + + +!!! tab examples "Environment variables" + + ```yaml + # Use delvewheel on windows + CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" + + # Don't repair macOS wheels + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + + # Pass the `--lib-sdir .` flag to auditwheel on Linux + CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} {wheel}" + + # Multi-line example - use && to join on all platforms + CIBW_REPAIR_WHEEL_COMMAND: > + python scripts/repair_wheel.py -w {dest_dir} {wheel} && + python scripts/check_repaired_wheel.py -w {dest_dir} {wheel} + + # Use abi3audit to catch issues with Limited API wheels + CIBW_REPAIR_WHEEL_COMMAND_LINUX: > + auditwheel repair -w {dest_dir} {wheel} && + pipx run abi3audit --strict --report {wheel} + CIBW_REPAIR_WHEEL_COMMAND_MACOS: > + delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} && + pipx run abi3audit --strict --report {wheel} + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: > + copy {wheel} {dest_dir} && + pipx run abi3audit --strict --report {wheel} + ``` -### `CIBW_MANYLINUX_*_IMAGE`, `CIBW_MUSLLINUX_*_IMAGE` {: #linux-image} -> Specify alternative manylinux / musllinux Docker images +### `manylinux-*-image`, `musllinux-*-image` {: #linux-image env-var toml} + +> Specify manylinux / musllinux container images + +The available options are: -The available options are (default value): +| Option | Default | +|--------------------------------|-----------------------------------------------------------------| +| `manylinux_x86_64-image` | [`manylinux_2_28`](https://quay.io/pypa/manylinux_2_28_x86_64) | +| `manylinux-i686-image` | [`manylinux2014`](https://quay.io/pypa/manylinux2014_i686) | +| `manylinux-pypy_x86_64-image` | [`manylinux_2_28`](https://quay.io/pypa/manylinux_2_28_x86_64) | +| `manylinux-aarch64-image` | [`manylinux_2_28`](https://quay.io/pypa/manylinux_2_28_aarch64) | +| `manylinux-ppc64le-image` | [`manylinux_2_28`](https://quay.io/pypa/manylinux_2_28_ppc64le) | +| `manylinux-s390x-image` | [`manylinux_2_28`](https://quay.io/pypa/manylinux_2_28_s390x) | +| `manylinux-armv7l-image` | [`manylinux_2_31`](https://quay.io/pypa/manylinux_2_31_armv7l) | +| `manylinux-riscv64-image` | No default | +| `manylinux-pypy_aarch64-image` | [`manylinux_2_28`](https://quay.io/pypa/manylinux_2_28_aarch64) | +| `manylinux-pypy_i686-image` | [`manylinux2014`](https://quay.io/pypa/manylinux2014_i686) | +| `musllinux_x86_64-image` | [`musllinux_1_2`](https://quay.io/pypa/musllinux_1_2_x86_64) | +| `musllinux-i686-image` | [`musllinux_1_2`](https://quay.io/pypa/musllinux_1_2_i686) | +| `musllinux-aarch64-image` | [`musllinux_1_2`](https://quay.io/pypa/musllinux_1_2_aarch64) | +| `musllinux-ppc64le-image` | [`musllinux_1_2`](https://quay.io/pypa/musllinux_1_2_ppc64le) | +| `musllinux-s390x-image` | [`musllinux_1_2`](https://quay.io/pypa/musllinux_1_2_s390x) | +| `musllinux-armv7l-image` | [`musllinux_1_2`](https://quay.io/pypa/musllinux_1_2_armv7l) | +| `musllinux-riscv64-image` | No default | -- `CIBW_MANYLINUX_X86_64_IMAGE` ([`quay.io/pypa/manylinux2014_x86_64`](https://quay.io/pypa/manylinux2014_x86_64)) -- `CIBW_MANYLINUX_I686_IMAGE` ([`quay.io/pypa/manylinux2014_i686`](https://quay.io/pypa/manylinux2014_i686)) -- `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` ([`quay.io/pypa/manylinux2014_x86_64`](https://quay.io/pypa/manylinux2014_x86_64)) -- `CIBW_MANYLINUX_AARCH64_IMAGE` ([`quay.io/pypa/manylinux2014_aarch64`](https://quay.io/pypa/manylinux2014_aarch64)) -- `CIBW_MANYLINUX_PPC64LE_IMAGE` ([`quay.io/pypa/manylinux2014_ppc64le`](https://quay.io/pypa/manylinux2014_ppc64le)) -- `CIBW_MANYLINUX_S390X_IMAGE` ([`quay.io/pypa/manylinux2014_s390x`](https://quay.io/pypa/manylinux2014_s390x)) -- `CIBW_MANYLINUX_PYPY_AARCH64_IMAGE` ([`quay.io/pypa/manylinux2014_aarch64`](https://quay.io/pypa/manylinux2014_aarch64)) -- `CIBW_MANYLINUX_PYPY_I686_IMAGE` ([`quay.io/pypa/manylinux2014_i686`](https://quay.io/pypa/manylinux2014_i686)) -- `CIBW_MUSLLINUX_X86_64_IMAGE` ([`quay.io/pypa/musllinux_1_1_x86_64`](https://quay.io/pypa/musllinux_1_1_x86_64)) -- `CIBW_MUSLLINUX_I686_IMAGE` ([`quay.io/pypa/musllinux_1_1_i686`](https://quay.io/pypa/musllinux_1_1_i686)) -- `CIBW_MUSLLINUX_AARCH64_IMAGE` ([`quay.io/pypa/musllinux_1_1_aarch64`](https://quay.io/pypa/musllinux_1_1_aarch64)) -- `CIBW_MUSLLINUX_PPC64LE_IMAGE` ([`quay.io/pypa/musllinux_1_1_ppc64le`](https://quay.io/pypa/musllinux_1_1_ppc64le)) -- `CIBW_MUSLLINUX_S390X_IMAGE` ([`quay.io/pypa/musllinux_1_1_s390x`](https://quay.io/pypa/musllinux_1_1_s390x)) +Set the Docker image to be used for building [manylinux / musllinux](https://github.com/pypa/manylinux) wheels. -Set an alternative Docker image to be used for building [manylinux / musllinux](https://github.com/pypa/manylinux) wheels. +For `manylinux-*-image`, except `manylinux-armv7l-image`, the value of this option can either be set to `manylinux2014`, `manylinux_2_28` or `manylinux_2_34` to use a pinned version of the [official manylinux images](https://github.com/pypa/manylinux). Alternatively, set these options to any other valid Docker image name. +`manylinux_2_28` and `manylinux_2_34` are not supported for `i686` architecture. -For `CIBW_MANYLINUX_*_IMAGE`, the value of this option can either be set to `manylinux1`, `manylinux2010`, `manylinux2014` or `manylinux_2_24` to use a pinned version of the [official manylinux images](https://github.com/pypa/manylinux). Alternatively, set these options to any other valid Docker image name. For PyPy, the `manylinux1` image is not available. For architectures other -than x86 (x86\_64 and i686) `manylinux2014` or `manylinux_2_24` must be used, because the first version of the manylinux specification that supports additional architectures is `manylinux2014`. +For `manylinux-armv7l-image`, the value of this option can either be set to `manylinux_2_31` or a custom image. Support is experimental for now. The `manylinux_2_31` value is only available for `armv7`. -For `CIBW_MUSLLINUX_*_IMAGE`, the value of this option can either be set to `musllinux_1_1` to use a pinned version of the [official musllinux images](https://github.com/pypa/musllinux). Alternatively, set these options to any other valid Docker image name. +For `musllinux-*-image`, the value of this option can either be set to `musllinux_1_2` or a custom image. If this option is blank, it will fall though to the next available definition (environment variable -> pyproject.toml -> default). -If setting a custom Docker image, you'll need to make sure it can be used in the same way as the official, default Docker images: all necessary Python and pip versions need to be present in `/opt/python/`, and the auditwheel tool needs to be present for cibuildwheel to work. Apart from that, the architecture and relevant shared system libraries need to be compatible to the relevant standard to produce valid manylinux1/manylinux2010/manylinux2014/manylinux_2_24/musllinux_1_1 wheels (see [pypa/manylinux on GitHub](https://github.com/pypa/manylinux), [PEP 513](https://www.python.org/dev/peps/pep-0513/), [PEP 571](https://www.python.org/dev/peps/pep-0571/), [PEP 599](https://www.python.org/dev/peps/pep-0599/), [PEP 600](https://www.python.org/dev/peps/pep-0600/) and [PEP 656](https://www.python.org/dev/peps/pep-0656/) for more details). +If setting a custom image, you'll need to make sure it can be used in the same way as the default images: all necessary Python and pip versions need to be present in `/opt/python/`, and the auditwheel tool needs to be present for cibuildwheel to work. Apart from that, the architecture and relevant shared system libraries need to be compatible to the relevant standard to produce valid manylinux2014/manylinux_2_28/manylinux_2_34/musllinux_1_2 wheels (see [pypa/manylinux on GitHub](https://github.com/pypa/manylinux), [PEP 599](https://www.python.org/dev/peps/pep-0599/), [PEP 600](https://www.python.org/dev/peps/pep-0600/) and [PEP 656](https://www.python.org/dev/peps/pep-0656/) for more details). -Auditwheel detects the version of the manylinux / musllinux standard in the Docker image through the `AUDITWHEEL_PLAT` environment variable, as cibuildwheel has no way of detecting the correct `--plat` command line argument to pass to auditwheel for a custom image. If a Docker image does not correctly set this `AUDITWHEEL_PLAT` environment variable, the `CIBW_ENVIRONMENT` option can be used to do so (e.g., `CIBW_ENVIRONMENT='AUDITWHEEL_PLAT="manylinux2010_$(uname -m)"'`). +Auditwheel detects the version of the manylinux / musllinux standard in the image through the `AUDITWHEEL_PLAT` environment variable, as cibuildwheel has no way of detecting the correct `--plat` command line argument to pass to auditwheel for a custom image. If a custom image does not correctly set this `AUDITWHEEL_PLAT` environment variable, the `CIBW_ENVIRONMENT` option can be used to do so (e.g., `CIBW_ENVIRONMENT='AUDITWHEEL_PLAT="manylinux2014_$(uname -m)"'`). -#### Examples +!!! warning + On x86_64, `manylinux_2_34` is using [x86-64-v2](https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels) target architecture. + While manylinux worked around that when building extensions from sources by intercepting compiler calls + to target x86_64 instead, every library installed with dnf will most likely target the more + recent x86-64-v2 which, if grafted into a wheel, will fail to run on older hardware. -!!! tab examples "Environment variables" + The workaround does not work for executables as they are always being linked with x86-64-v2 object files. - ```yaml - # Build using the manylinux1 image to ensure manylinux1 wheels are produced - # Not setting PyPy to manylinux1, since there is no manylinux1 PyPy image. - CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 - CIBW_MANYLINUX_I686_IMAGE: manylinux1 + There's no PEP to handle micro-architecture variants yet when it comes to packaging or + installing wheels. Auditwheel doesn't detect this either. - # Build using the manylinux2014 image - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - CIBW_MANYLINUX_I686_IMAGE: manylinux2014 - CIBW_MANYLINUX_PYPY_X86_64_IMAGE: manylinux2014 - CIBW_MANYLINUX_PYPY_I686_IMAGE: manylinux2014 + Please check the tracking issue in [pypa/manylinux](https://github.com/pypa/manylinux/issues/1725) - # Build using the latest manylinux2014 release, instead of the cibuildwheel - # pinned version - CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64:latest - CIBW_MANYLINUX_I686_IMAGE: quay.io/pypa/manylinux2014_i686:latest - CIBW_MANYLINUX_PYPY_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64:latest - CIBW_MANYLINUX_PYPY_I686_IMAGE: quay.io/pypa/manylinux2014_i686:latest - # Build using a different image from the docker registry - CIBW_MANYLINUX_X86_64_IMAGE: dockcross/manylinux-x64 - CIBW_MANYLINUX_I686_IMAGE: dockcross/manylinux-x86 - ``` +#### Examples + !!! tab examples "pyproject.toml" ```toml [tool.cibuildwheel] - # Build using the manylinux1 image to ensure manylinux1 wheels are produced - # Not setting PyPy to manylinux1, since there is no manylinux1 PyPy image. - manylinux-x86_64-image = "manylinux1" - manylinux-i686-image = "manylinux1" - # Build using the manylinux2014 image manylinux-x86_64-image = "manylinux2014" manylinux-i686-image = "manylinux2014" manylinux-pypy_x86_64-image = "manylinux2014" manylinux-pypy_i686-image = "manylinux2014" - # Build using the latest manylinux2010 release, instead of the cibuildwheel - # pinned version + # Build using the latest manylinux2010 release manylinux-x86_64-image = "quay.io/pypa/manylinux2010_x86_64:latest" manylinux-i686-image = "quay.io/pypa/manylinux2010_i686:latest" manylinux-pypy_x86_64-image = "quay.io/pypa/manylinux2010_x86_64:latest" @@ -962,19 +1078,112 @@ Auditwheel detects the version of the manylinux / musllinux standard in the Dock # Build using a different image from the docker registry manylinux-x86_64-image = "dockcross/manylinux-x64" manylinux-i686-image = "dockcross/manylinux-x86" + + # Build musllinux wheels using the musllinux_1_1 image + musllinux-x86_64-image = "quay.io/pypa/musllinux_1_1_x86_64:latest" + musllinux-i686-image = "quay.io/pypa/musllinux_1_1_i686:latest" ``` Like any other option, these can be placed in `[tool.cibuildwheel.linux]` if you prefer; they have no effect on `macos` and `windows`. -### `CIBW_DEPENDENCY_VERSIONS` {: #dependency-versions} -> Specify how cibuildwheel controls the versions of the tools it uses +!!! tab examples "Environment variables" + + ```yaml + # Build using the manylinux2014 image + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_MANYLINUX_I686_IMAGE: manylinux2014 + CIBW_MANYLINUX_PYPY_X86_64_IMAGE: manylinux2014 + CIBW_MANYLINUX_PYPY_I686_IMAGE: manylinux2014 + + # Build using the latest manylinux2010 release + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2010_x86_64:latest + CIBW_MANYLINUX_I686_IMAGE: quay.io/pypa/manylinux2010_i686:latest + CIBW_MANYLINUX_PYPY_X86_64_IMAGE: quay.io/pypa/manylinux2010_x86_64:latest + CIBW_MANYLINUX_PYPY_I686_IMAGE: quay.io/pypa/manylinux2010_i686:latest + + # Build using a different image from the docker registry + CIBW_MANYLINUX_X86_64_IMAGE: dockcross/manylinux-x64 + CIBW_MANYLINUX_I686_IMAGE: dockcross/manylinux-x86 + + # Build musllinux wheels using the musllinux_1_1 image + CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/musllinux_1_1_x86_64:latest + CIBW_MUSLLINUX_I686_IMAGE: quay.io/pypa/musllinux_1_1_i686:latest + ``` + + +### `container-engine` {: #container-engine env-var toml} +> Specify the container engine to use when building Linux wheels -Options: `pinned` `latest` `` +Options: + +- `docker[;create_args: ...][;disable_host_mount: true/false]` +- `podman[;create_args: ...][;disable_host_mount: true/false]` + +Default: `docker` + +Set the container engine to use. Docker is the default, or you can switch to +[Podman](https://podman.io/). To use Docker, you need to have a Docker daemon +running and `docker` available on PATH. To use Podman, it needs to be +installed and `podman` available on PATH. + +Options can be supplied after the name. + +| Option name | Description +|---|--- +| `create_args` | Space-separated strings, which are passed to the container engine on the command line when it's creating the container. If you want to include spaces inside a parameter, use shell-style quoting. +| `disable_host_mount` | By default, cibuildwheel will mount the root of the host filesystem as a volume at `/host` in the container. To disable the host mount, pass `true` to this option. + + +!!! tip + + While most users will stick with Docker, Podman is available in different + contexts - for example, it can be run inside a Docker container, or without + root access. Thanks to the [OCI], images are compatible between engines, so + you can still use the regular manylinux/musllinux containers. + +[OCI]: https://opencontainers.org/ + +#### Examples + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # use podman instead of docker + container-engine = "podman" + + # pass command line options to 'docker create' + container-engine = { name = "docker", create-args = ["--gpus", "all"]} + + # disable the /host mount + container-engine = { name = "docker", disable-host-mount = true } + ``` + +!!! tab examples "Environment variables" + + ```yaml + # use podman instead of docker + CIBW_CONTAINER_ENGINE: podman + + # pass command line options to 'docker create' + CIBW_CONTAINER_ENGINE: "docker; create_args: --gpus all" + + # disable the /host mount + CIBW_CONTAINER_ENGINE: "docker; disable_host_mount: true" + ``` + + + +### `dependency-versions` {: #dependency-versions env-var toml} + +> Control the versions of the tools cibuildwheel uses + +Options: `pinned` `latest` `packages: SPECIFIER...` `` Default: `pinned` -If `CIBW_DEPENDENCY_VERSIONS` is `pinned`, cibuildwheel uses versions of tools +If `dependency-versions` is `pinned`, cibuildwheel uses versions of tools like `pip`, `setuptools`, `virtualenv` that were pinned with that release of cibuildwheel. This represents a known-good set of dependencies, and is recommended for build repeatability. @@ -985,26 +1194,48 @@ fixes that can't wait for a new cibuildwheel release. To control the versions of dependencies yourself, you can supply a [pip constraints](https://pip.pypa.io/en/stable/user_guide/#constraints-files) file -here and it will be used instead. +here and it will be used instead. Alternatively, you can list constraint +specifiers inline with the `packages: SPECIFIER...` syntax. !!! note If you need different dependencies for each python version, provide them in the same folder with a `-pythonXY` suffix. e.g. if your - `CIBW_DEPENDENCY_VERSIONS=./constraints.txt`, cibuildwheel will use - `./constraints-python37.txt` on Python 3.7, or fallback to + `dependency-versions="./constraints.txt"`, cibuildwheel will use + `./constraints-python38.txt` on Python 3.8, or fallback to `./constraints.txt` if that's not found. Platform-specific environment variables are also available:
-`CIBW_DEPENDENCY_VERSIONS_MACOS` | `CIBW_DEPENDENCY_VERSIONS_WINDOWS` +`CIBW_DEPENDENCY_VERSIONS_MACOS` | `CIBW_DEPENDENCY_VERSIONS_WINDOWS` | `CIBW_DEPENDENCY_VERSIONS_IOS` | `CIBW_DEPENDENCY_VERSIONS_PYODIDE` !!! note This option does not affect the tools used on the Linux build - those versions are bundled with the manylinux/musllinux image that cibuildwheel uses. To change - dependency versions on Linux, use the [CIBW_MANYLINUX_* / CIBW_MUSLLINUX_*](#linux-image) + dependency versions on Linux, use the [`manylinux-*` / `musllinux-*`](#linux-image) options. #### Examples +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # Use tools versions that are bundled with cibuildwheel (this is the default) + dependency-versions = "pinned" + + # Use the latest versions available on PyPI + dependency-versions = "latest" + + # Use your own pip constraints file + dependency-versions = { file = "./constraints.txt" } + + # Specify requirements inline + dependency-versions = { packages = ["auditwheel==6.2.0"] } + + [tool.cibuildwheel.pyodide] + # Choose a specific pyodide-build version + dependency-versions = { packages = ["pyodide-build==0.29.1"] } + ``` + !!! tab examples "Environment variables" ```yaml @@ -1014,90 +1245,150 @@ Platform-specific environment variables are also available:
# Use the latest versions available on PyPI CIBW_DEPENDENCY_VERSIONS: latest - # Use your own pip constraints file - CIBW_DEPENDENCY_VERSIONS: ./constraints.txt - ``` + # Use your own pip constraints file + CIBW_DEPENDENCY_VERSIONS: ./constraints.txt + + # Specify requirements inline + CIBW_DEPENDENCY_VERSIONS: "packages: auditwheel==6.2.0" + + # Choose a specific pyodide-build version + CIBW_DEPENDENCY_VERSIONS_PYODIDE: "packages: pyodide-build==0.29.1" + + # Use shell-style quoting around spaces package specifiers + CIBW_DEPENDENCY_VERSIONS: "packages: 'pip >=16.0.0, !=17'" + ``` + + + +### `pyodide-version` {: #pyodide-version toml env-var } + +> Specify the Pyodide version to use for `pyodide` platform builds + +This option allows you to specify a specific version of Pyodide to be used when building wheels for the `pyodide` platform. If unset, cibuildwheel will use a pinned Pyodide version. + +This option is particularly useful for: + +- Testing against specific Pyodide alpha or older releases. +- Ensuring reproducibility by targeting a known Pyodide version. + +The available Pyodide versions are determined by the version of `pyodide-build` being used. You can list the compatible versions using the command `pyodide xbuildenv search --all` as described in the [Pyodide platform documentation](platforms.md#pyodide-choosing-a-version). + +!!! tip + You can set the version of `pyodide-build` using the [`dependency-versions`](#dependency-versions) option. + +!!! warning + This option is considered experimental, and might be converted to a more general mechanism in a future minor cibuildwheel release. + +!!! warning + Make sure to scope it to one specific pyodide identifier with overrides if using the `pyodide-prerelease` enable. + +#### Examples !!! tab examples "pyproject.toml" ```toml - [tool.cibuildwheel] - # Use tools versions that are bundled with cibuildwheel (this is the default) - dependency-versions = "pinned" + [tool.cibuildwheel.pyodide] + # Build Pyodide wheels using Pyodide version 0.27.6 + pyodide-version = "0.27.6" - # Use the latest versions available on PyPI - dependency-versions = "latest" + [tool.cibuildwheel.pyodide] + # Build Pyodide wheels using a specific alpha release + pyodide-version = "0.28.0a2" + ``` - # Use your own pip constraints file - dependency-versions = "./constraints.txt" +!!! tab examples "Environment variables" + + ```yaml + # Build Pyodide wheels using Pyodide version 0.27.6 + CIBW_PYODIDE_VERSION: 0.27.6 + + # Build Pyodide wheels using a specific alpha release + CIBW_PYODIDE_VERSION: 0.28.0a2 ``` + ## Testing -### `CIBW_TEST_COMMAND` {: #test-command} -> Execute a shell command to test each built wheel +### `test-command` {: #test-command env-var toml} +> The command to test each built wheel Shell command to run tests after the build. The wheel will be installed -automatically and available for import from the tests. To ensure the wheel is -imported by your tests (instead of your source copy), **tests are not run from -your project directory**. Use the placeholders `{project}` and `{package}` when -specifying paths in your project. If this variable is not set, your wheel will -not be installed after building. - -- `{project}` is an absolute path to the project root - the working directory - where cibuildwheel was called. -- `{package}` is the path to the package being built - the `package_dir` - argument supplied to cibuildwheel on the command line. +automatically and available for import from the tests. If this variable is not +set, your wheel will not be installed after building. -The command is run in a shell, so you can write things like `cmd1 && cmd2`. +To ensure the wheel is imported by your tests (instead of your source copy), +**Tests are executed from a temporary directory**, outside of your source +tree. To access your test code, you have a couple of options: -Platform-specific environment variables are also available:
-`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` +- You can use the [`test-sources`](#test-sources) setting to copy specific + files from your source tree into the temporary directory. When using + test-sources, use relative paths in your test command, as if they were + relative to the project root. -#### Examples +- You can use the `{package}` or `{project}` placeholders in your + `test-command` to refer to the package being built or the project root, + respectively. -!!! tab examples "Environment variables" + - `{package}` is the path to the package being built - the `package_dir` + argument supplied to cibuildwheel on the command line. + - `{project}` is an absolute path to the project root - the working + directory where cibuildwheel was called. - ```yaml - # Run the project tests against the installed wheel using `nose` - CIBW_TEST_COMMAND: nosetests {project}/tests +On all platforms other than iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. - # Run the package tests using `pytest` - CIBW_TEST_COMMAND: pytest {package}/tests +On iOS, the value of the `test-command` setting must follow the format `python +-m MODULE [ARGS...]` - where MODULE is a Python module name, followed by +arguments that will be assigned to `sys.argv`. Other commands cannot be used. - # Trigger an install of the package, but run nothing of note - CIBW_TEST_COMMAND: "echo Wheel installed" +Platform-specific environment variables are also available:
+`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE` - # Multi-line example - join with && on all platforms - CIBW_TEST_COMMAND: > - pytest {package}/tests && - python {package}/test.py - ``` +#### Examples !!! tab examples "pyproject.toml" ```toml [tool.cibuildwheel] - # Run the project tests against the installed wheel using `nose` - test-command = "nosetests {project}/tests" - # Run the package tests using `pytest` - test-command = "pytest {package}/tests" + test-command = "pytest {project}/tests" # Trigger an install of the package, but run nothing of note test-command = "echo Wheel installed" # Multiline example test-command = [ - "pytest {package}/tests", - "python {package}/test.py", + "pytest {project}/tests", + "python {project}/test.py", ] + + # run tests on ios - when test-sources is set, use relative paths, not {project} or {package} + [tool.cibuildwheel.ios] + test-sources = ["tests"] + test-command = "python -m pytest ./tests" ``` In configuration files, you can use an array, and the items will be joined with `&&`. +!!! tab examples "Environment variables" + + ```yaml + # Run the package tests using `pytest` + CIBW_TEST_COMMAND: pytest {project}/tests + + # Trigger an install of the package, but run nothing of note + CIBW_TEST_COMMAND: "echo Wheel installed" + + # Multi-line example - join with && on all platforms + CIBW_TEST_COMMAND: > + pytest {project}/tests && + python {project}/test.py + + # run tests on ios - when test-sources is set, use relative paths, not {project} or {package} + CIBW_TEST_SOURCES_IOS: tests + CIBW_TEST_COMMAND_IOS: python -m pytest ./tests + ``` -### `CIBW_BEFORE_TEST` {: #before-test} +### `before-test` {: #before-test env-var toml} > Execute a shell command before testing each wheel A shell command to run in **each** test virtual environment, before your wheel is installed and tested. This is useful if you need to install a non-pip package, invoke pip with different environment variables, @@ -1108,30 +1399,10 @@ The active Python binary can be accessed using `python`, and pip with `pip`; cib The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
- `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` + `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` | `CIBW_BEFORE_TEST_IOS` | `CIBW_BEFORE_TEST_PYODIDE` #### Examples -!!! tab examples "Environment variables" - - ```yaml - # Install test dependencies with overwritten environment variables. - CIBW_BEFORE_TEST: CC=gcc CXX=g++ pip install -r requirements.txt - - # Chain commands using && - CIBW_BEFORE_TEST: rm -rf ./data/cache && mkdir -p ./data/cache - - # Install non pip python package - CIBW_BEFORE_TEST: > - cd some_dir && - ./configure && - make && - make install - - # Install python packages that are required to install test dependencies - CIBW_BEFORE_TEST: pip install cmake scikit-build - ``` - !!! tab examples "pyproject.toml" ```toml @@ -1161,57 +1432,124 @@ Platform-specific environment variables are also available:
In configuration files, you can use an array, and the items will be joined with `&&`. +!!! tab examples "Environment variables" -### `CIBW_TEST_REQUIRES` {: #test-requires} -> Install Python dependencies before running the tests + ```yaml + # Install test dependencies with overwritten environment variables. + CIBW_BEFORE_TEST: CC=gcc CXX=g++ pip install -r requirements.txt -Space-separated list of dependencies required for running the tests. + # Chain commands using && + CIBW_BEFORE_TEST: rm -rf ./data/cache && mkdir -p ./data/cache + + # Install non pip python package + CIBW_BEFORE_TEST: > + cd some_dir && + ./configure && + make && + make install + + # Install python packages that are required to install test dependencies + CIBW_BEFORE_TEST: pip install cmake scikit-build + ``` + + +### `test-sources` {: #test-sources env-var toml} +> Files and folders from the source tree that are copied into an isolated tree before running the tests + +A space-separated list of files and folders, relative to the root of the +project, required for running the tests. If specified, these files and folders +will be copied into a temporary folder, and that temporary folder will be used +as the working directory for running the test suite. + +The use of `test-sources` is *required* for iOS builds. This is because the +simulator does not have access to the project directory, as it is not stored on +the simulator device. On iOS, the files will be copied into the test application, +rather than a temporary folder. Platform-specific environment variables are also available:
-`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` +`CIBW_TEST_SOURCES_MACOS` | `CIBW_TEST_SOURCES_WINDOWS` | `CIBW_TEST_SOURCES_LINUX` | `CIBW_TEST_SOURCES_IOS` | `CIBW_TEST_SOURCES_PYODIDE` #### Examples +!!! tab examples "pyproject.toml" + + ```toml + # Copy the "tests" folder, plus "data/test-image.png" from the source folder to the test folder. + [tool.cibuildwheel] + test-sources = ["tests", "data/test-image.png"] + ``` + + In configuration files, you can use an array, and the items will be joined with a space. + !!! tab examples "Environment variables" ```yaml - # Install pytest before running CIBW_TEST_COMMAND - CIBW_TEST_REQUIRES: pytest - - # Install specific versions of test dependencies - CIBW_TEST_REQUIRES: nose==1.3.7 moto==0.4.31 + # Copy the "tests" folder, plus "data/test-image.png" from the source folder to the test folder. + CIBW_TEST_SOURCES: tests data/test-image.png ``` + +### `test-requires` {: #test-requires env-var toml} +> Install Python dependencies before running the tests + +Space-separated list of dependencies required for running the tests. + +Platform-specific environment variables are also available:
+`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` | `CIBW_TEST_REQUIRES_IOS` | `CIBW_TEST_REQUIRES_PYODIDE` + +#### Examples + !!! tab examples "pyproject.toml" ```toml - # Install pytest before running CIBW_TEST_COMMAND + # Install pytest before running test-command [tool.cibuildwheel] test-requires = "pytest" # Install specific versions of test dependencies [tool.cibuildwheel] - test-requires = ["nose==1.3.7", "moto==0.4.31"] + test-requires = ["pytest==8.2.2", "packaging==24.1"] ``` In configuration files, you can use an array, and the items will be joined with a space. +!!! tab examples "Environment variables" + + ```yaml + # Install pytest before running CIBW_TEST_COMMAND + CIBW_TEST_REQUIRES: pytest + + # Install specific versions of test dependencies + CIBW_TEST_REQUIRES: pytest==8.2.2 packaging==24.1 + ``` + + -### `CIBW_TEST_EXTRAS` {: #test-extras} +### `test-extras` {: #test-extras env-var toml} > Install your wheel for testing using `extras_require` List of -[extras_require](https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies) +[extras_require](https://setuptools.pypa.io/en/latest/userguide/dependency_management.html#declaring-required-dependency) options that should be included when installing the wheel prior to running the tests. This can be used to avoid having to redefine test dependencies in -`CIBW_TEST_REQUIRES` if they are already defined in `setup.py` or -`setup.cfg`. +`test-requires` if they are already defined in `pyproject.toml`, +`setup.cfg` or `setup.py`. Platform-specific environment variables are also available:
-`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` +`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` | `CIBW_TEST_EXTRAS_IOS` | `CIBW_TEST_EXTRAS_PYODIDE` #### Examples +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # Will cause the wheel to be installed with `pip install [test,qt]` + test-extras = ["test", "qt"] + ``` + + In configuration files, you can use an inline array, and the items will be joined with a comma. + !!! tab examples "Environment variables" ```yaml @@ -1221,20 +1559,46 @@ Platform-specific environment variables are also available:
Separate multiple items with a comma. + + +### `test-groups` {: #test-groups env-var toml} +> Specify test dependencies from your project's `dependency-groups` + +List of +[dependency-groups](https://peps.python.org/pep-0735) +that should be included when installing the wheel prior to running the +tests. This can be used to avoid having to redefine test dependencies in +`test-requires` if they are already defined in `pyproject.toml`. + +Platform-specific environment variables are also available:
+`CIBW_TEST_GROUPS_MACOS` | `CIBW_TEST_GROUPS_WINDOWS` | `CIBW_TEST_GROUPS_LINUX` | `CIBW_TEST_GROUPS_PYODIDE` + +#### Examples + !!! tab examples "pyproject.toml" ```toml [tool.cibuildwheel] - # Will cause the wheel to be installed with `pip install [test,qt]` - test-extras = ["test", "qt"] + # Will cause the wheel to be installed with these groups of dependencies + test-groups = ["test", "qt"] ``` - In configuration files, you can use an inline array, and the items will be joined with a comma. + In configuration files, you can use an inline array, and the items will be joined with a space. + -### `CIBW_TEST_SKIP` {: #test-skip} +!!! tab examples "Environment variables" + + ```yaml + # Will cause the wheel to be installed with these groups of dependencies + CIBW_TEST_GROUPS: "test qt" + ``` + + Separate multiple items with a space. + +### `test-skip` {: #test-skip env-var toml} > Skip running tests on some builds -This will skip testing on any identifiers that match the given skip patterns (see [`CIBW_SKIP`](#build-skip)). This can be used to mask out tests for wheels that have missing dependencies upstream that are slow or hard to build, or to skip slow tests on emulated architectures. +This will skip testing on any identifiers that match the given skip patterns (see [`skip`](#build-skip)). This can be used to mask out tests for wheels that have missing dependencies upstream that are slow or hard to build, or to skip slow tests on emulated architectures. With macOS `universal2` wheels, you can also skip the individual archs inside the wheel using an `:arch` suffix. For example, `cp39-macosx_universal2:x86_64` or `cp39-macosx_universal2:arm64`. @@ -1242,76 +1606,181 @@ This option is not supported in the overrides section in `pyproject.toml`. #### Examples -!!! tab examples "Environment variables" +!!! tab examples "pyproject.toml" - ```yaml + ```toml + [tool.cibuildwheel] # Will avoid testing on emulated architectures - CIBW_TEST_SKIP: "*-*linux_{aarch64,ppc64le,s390x}" + test-skip = "*-*linux_{aarch64,ppc64le,s390x,armv7l}" # Skip trying to test arm64 builds on Intel Macs - CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64" + test-skip = "*-macosx_arm64 *-macosx_universal2:arm64" ``` -!!! tab examples "pyproject.toml" +!!! tab examples "Environment variables" - ```toml - [tool.cibuildwheel] + ```yaml # Will avoid testing on emulated architectures - test-skip = "*-*linux_{aarch64,ppc64le,s390x}" + CIBW_TEST_SKIP: "*-*linux_{aarch64,ppc64le,s390x,armv7l}" # Skip trying to test arm64 builds on Intel Macs - test-skip = "*-macosx_arm64 *-macosx_universal2:arm64" + CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64" ``` -## Other -### `CIBW_BUILD_VERBOSITY` {: #build-verbosity} -> Increase/decrease the output of pip wheel +### `test-environment` {: #test-environment toml env-var } -An number from 1 to 3 to increase the level of verbosity (corresponding to invoking pip with `-v`, `-vv`, and `-vvv`), between -1 and -3 (`-q`, `-qq`, and `-qqq`), or just 0 (default verbosity). These flags are useful while debugging a build when the output of the actual build invoked by `pip wheel` is required. +> Set environment variables for the test environment + +A space-separated list of environment variables to set in the test environment. + +The syntax is the same as for [`environment`](#environment). Platform-specific environment variables are also available:
-`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` +`CIBW_TEST_ENVIRONMENT_MACOS` | `CIBW_TEST_ENVIRONMENT_WINDOWS` | `CIBW_TEST_ENVIRONMENT_LINUX` | `CIBW_TEST_ENVIRONMENT_IOS` | `CIBW_TEST_ENVIRONMENT_PYODIDE` #### Examples +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # Set the environment variable MY_ENV_VAR to "my_value" in the test environment + test-environment = { MY_ENV_VAR="my_value" } + + # Set PYTHONSAFEPATH in the test environment + test-environment = { PYTHONSAFEPATH="1" } + ``` + !!! tab examples "Environment variables" ```yaml - # Increase pip debugging output - CIBW_BUILD_VERBOSITY: 1 + # Set the environment variable MY_ENV_VAR to "my_value" in the test environment + CIBW_TEST_ENVIRONMENT: MY_ENV_VAR=my_value + + # Set PYTHONSAFEPATH in the test environment + CIBW_TEST_ENVIRONMENT: PYTHONSAFEPATH=1 ``` + +## Debugging + +### `debug-keep-container` {: #debug-keep-container env-var} +> Keep the container after running for debugging. + +Enable this flag to keep the container around for inspection after a build. This +option is provided for debugging purposes only. + +Default: Off (0). + +!!! caution + This option can only be set as environment variable on the host machine + +#### Examples + +```shell +export CIBW_DEBUG_KEEP_CONTAINER=TRUE +``` + +### `debug-traceback` {: #debug-traceback cmd-line env-var} +> Print full traceback when errors occur. + +Print a full traceback for the cibuildwheel process when errors occur. This +option is provided for debugging cibuildwheel. + +This option can also be set using the [command-line option](#command-line) `--debug-traceback`. + +#### Examples + +```shell +export CIBW_DEBUG_TRACEBACK=TRUE +``` + +### `build-verbosity` {: #build-verbosity env-var toml} +> Increase/decrease the output of the build + +This setting controls `-v`/`-q` flags to the build frontend. Since there is +no communication between the build backend and the build frontend, build +messages from the build backend will always be shown with `1`; higher levels +will not produce more logging about the build itself. Other levels only affect +the build frontend output, which is usually things like resolving and +downloading dependencies. The settings are: + +| | build | pip | desc | +|-------------|-------|--------|----------------------------------| +| -2 | N/A | `-qq` | even more quiet, where supported | +| -1 | N/A | `-q` | quiet mode, where supported | +| 0 (default) | | | default for build tool | +| 1 | | `-v` | print backend output | +| 2 | `-v` | `-vv` | print log messages e.g. resolving info | +| 3 | `-vv` | `-vvv` | print even more debug info | + +Settings that are not supported for a specific frontend will log a warning. +The default build frontend is `build`, which does show build backend output by +default. + +Platform-specific environment variables are also available:
+`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` | `CIBW_BUILD_VERBOSITY_IOS` | `CIBW_BUILD_VERBOSITY_PYODIDE` + +#### Examples + !!! tab examples "pyproject.toml" ```toml [tool.cibuildwheel] - # Increase pip debugging output + # Ensure that the build backend output is present build-verbosity = 1 ``` +!!! tab examples "Environment variables" + + ```yaml + # Ensure that the build backend output is present + CIBW_BUILD_VERBOSITY: 1 + ``` + -## Command line options {: #command-line} -```text + +## Command line {: #command-line} + +### Options + « subprocess_run("cibuildwheel", "--help") » -``` + +### Return codes + +cibuildwheel exits 0 on success, or >0 if an error occurs. + +Specific error codes are defined: + +- 2 means a configuration error +- 3 means no builds are selected (and [`--allow-empty`](#allow-empty) wasn't passed) +- 4 means you specified an option that has been deprecated. + ## Placeholders Some options support placeholders, like `{project}`, `{package}` or `{wheel}`, that are substituted by cibuildwheel before they are used. If, for some reason, you need to write the literal name of a placeholder, e.g. literally `{project}` in a command that would ordinarily substitute `{project}`, prefix it with a hash character - `#{project}`. This is only necessary in commands where the specific string between the curly brackets would be substituted - otherwise, strings not modified. @@ -1335,20 +1864,21 @@ Some options support placeholders, like `{project}`, `{package}` or `{wheel}`, t var headers = [] $('.rst-content h3') - .filter(function (i, el) { - return !!$(el).text().match(/(^([A-Z0-9, _*]| and )+)¶$/); - }) .each(function (i, el) { var optionName = $(el).text().replace('¶', ''); var description = $(el).next('blockquote').text() var header = $(el).prevAll('h2').first().text().replace('¶', '') var id = el.id; + if (optionName[0].match(/[A-Z]/)) { + // all the options are kebab-case, so this header isn't an option + return; + } + if (options[header] === undefined) { options[header] = []; headers.push(header); } - console.log(optionName, description, header); options[header].push({name: optionName, description, id}); }); @@ -1384,41 +1914,56 @@ Some options support placeholders, like `{project}`, `{package}` or `{wheel}`, t } } - // write the markdown table for the README - - var markdown = '' + // add the option tags to each heading + $('.rst-content h3') + .each(function (i, el) { + el.classList.add('option', 'clearfix'); + var optionName = $(el).text().replace('¶', ''); - markdown += '| | Option | Description |\n' - markdown += '|---|--------|-------------|\n' + var cmdLine = el.getAttribute('cmd-line'); + var envVar = el.getAttribute('env-var'); + var toml = el.getAttribute('toml'); - var prevHeader = null + if (!(cmdLine || envVar || toml)) { + return; + } - for (var i = 0; i < headers.length; i += 1) { - var header = headers[i]; - var headerOptions = options[header]; - for (var j = 0; j < headerOptions.length; j += 1) { - var option = headerOptions[j]; + var badgesEl = $('
') + .appendTo(el); - if (j == 0) { - markdown += '| **'+header+'** ' - } else { - markdown += '| ' + // fill default value + if (cmdLine == "cmd-line") { + cmdLine = '--'+optionName; + } + if (envVar == "env-var") { + envVar = optionName + .split(', ') + .map(opt => 'CIBW_'+opt.toUpperCase().replace(/-/g, '_')) + .join(', '); + } + if (toml == "toml") { + toml = optionName } - var optionNames = option.name.trim().split(', ') - var url = 'https://cibuildwheel.readthedocs.io/en/stable/options/#'+option.id; - var namesMarkdown = $.map(optionNames, function(n) { - return '[`'+n+'`]('+url+') ' - }).join('
') - - markdown += '| '+namesMarkdown+' ' - markdown += '| '+option.description.trim()+' ' - markdown += '|\n' - } - } + if (toml) { + badgesEl.append(' '+toml+''); + } + if (cmdLine) { + badgesEl.append(' '+cmdLine+''); + } + if (envVar) { + badgesEl.append(' '+envVar+''); + } + }); - console.log('readme options markdown\n', markdown) + $('.toctree-l3') + .each(function (i, el) { + var tocEntryName = $(el).text() + var isOption = tocEntryName[0].match(/^[a-z]/); + if (isOption) { + $(el).addClass('option'); + } + }); }); - -[TOML]: https://toml.io + diff --git a/docs/platforms.md b/docs/platforms.md new file mode 100644 index 000000000..84a8cb502 --- /dev/null +++ b/docs/platforms.md @@ -0,0 +1,246 @@ +--- +title: Platforms +--- +# Platforms + +## Linux {: #linux} + +### System requirements + +If you've got [Docker](https://www.docker.com/get-started/) installed on your development machine, you can run a Linux build. + +!!! tip + You can run the Linux build on any platform. Even Windows can run + Linux containers these days, but there are a few hoops to jump + through. Check [this document](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/quick-start-windows-10-linux) + for more info. + +Because the builds are happening in manylinux Docker containers, they're perfectly reproducible. + +The only side effect to your system will be docker images being pulled. + +### Build containers {: #linux-containers} + +Linux wheels are built in [`manylinux`/`musllinux` containers](https://github.com/pypa/manylinux) to provide binary compatible wheels on Linux, according to [PEP 600](https://www.python.org/dev/peps/pep-0600/) / [PEP 656](https://www.python.org/dev/peps/pep-0656/). Because of this, when building with `cibuildwheel` on Linux, a few things should be taken into account: + +- Programs and libraries are not installed on the CI runner host, but rather should be installed inside the container - using `yum` for `manylinux2014`, `apt-get` for `manylinux_2_31`, `dnf` for `manylinux_2_28` and `apk` for `musllinux_1_1` or `musllinux_1_2`, or manually. The same goes for environment variables that are potentially needed to customize the wheel building. + + `cibuildwheel` supports this by providing the [`environment`](options.md#environment) and [`before-all`](options.md#before-all) options to setup the build environment inside the running container. + +- The project directory is copied into the container as `/project`, the output directory for the wheels to be copied out is `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that `/host` is not available on CircleCI and GitLab CI due to their Docker policies. + +- Alternative Docker images can be specified with the `manylinux-*-image`/`musllinux-*-image` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#linux-image) for more details. + +## macOS {: #macos} + +### System requirements + +You need to have native build tools installed. Use `xcode-select --install` to install the Xcode command line tools. + +Because the builds are happening without full isolation, there might be some differences compared to CI builds (Xcode version, OS version, local files, ...) that might prevent you from finding an issue only seen in CI. + +In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: /Users/Matt/Library/Caches/cibuildwheel`). You can override the cache folder using the `CIBW_CACHE_PATH` environment variable. + +!!! warning + cibuildwheel uses official python.org macOS installers for CPython but those can only be installed globally. + + In order not to mess with your system, cibuildwheel won't install those if they are missing. Instead, it will error out with a message to let you install the missing CPython: + + ```console + Error: CPython 3.9 is not installed. + cibuildwheel will not perform system-wide installs when running outside of CI. + To build locally, install CPython 3.9 on this machine, or, disable this version of Python using CIBW_SKIP=cp39-macosx_* + + Download link: https://www.python.org/ftp/python/3.9.8/python-3.9.8-macosx10.9.pkg + ``` + +### macOS Version Compatibility + +macOS allows you to specify a "deployment target" version that will ensure backwards compatibility with older versions of macOS. For most projects, the way to do this is to set the `MACOSX_DEPLOYMENT_TARGET` environment variable. + +macOS builds will honor the `MACOSX_DEPLOYMENT_TARGET` environment variable to control the minimum supported macOS version for generated wheels. The lowest value you can set `MACOSX_DEPLOYMENT_TARGET` is as follows: + +| Arch | Python version range | Minimum target | +|-------|----------------------|----------------| +| Intel | CPython 3.8-3.11 | 10.9 | +| Intel | CPython 3.12+ | 10.13 | +| AS | CPython or PyPy | 11 | +| Intel | PyPy 3.8 | 10.13 | +| Intel | PyPy 3.9+ | 10.15 | + +If you set the value lower, cibuildwheel will cap it to the lowest supported value for each target as needed. + +!!! note + For Rust-based extensions, `Rustc` requires `MACOSX_DEPLOYMENT_TARGET` to be at + least 10.12. However, `cibuildwheel` defaults to 10.9 for + **Intel / CPython 3.8-3.11** builds. Users must manually set + `MACOSX_DEPLOYMENT_TARGET` to 10.12 or higher when building Rust extensions. + +### macOS architectures + +`cibuildwheel` supports both native builds and cross-compiling between `arm64` (Apple Silicon) and `x86_64` (Intel) architectures, including the cross-compatible `universal2` format. By default, macOS builds will build a single architecture wheel, using the build machine's architecture. + +If you need to support both `x86_64` and Apple Silicon, you can use the [`macos.archs`](options.md#archs) setting to specify the architectures you want to build, or the value `universal2` to build a multi-architecture wheel. cibuildwheel _will_ test `x86_64` wheels (or the `x86_64` slice of a `universal2` wheel) when running on Apple Silicon hardware using Rosetta 2 emulation, but it is *not* possible to test Apple Silicon wheels on `x86_64` hardware. + +#### Overview of Mac architectures + +##### `x86_64` + +The traditional wheel for Apple, loads on Intel machines, and on +Apple Silicon when running Python under Rosetta 2 emulation. + +Due to a change in naming, Pip 20.3+ (or an installer using packaging 20.5+) +is required to install a binary wheel on macOS Big Sur. + +##### `arm64` + +The native wheel for macOS on Apple Silicon. + +##### `universal2` + +This wheel contains both architectures, causing it to be up to twice the +size (data files do not get doubled, only compiled code). + +The dual-architecture `universal2` has a few benefits, but a key benefit +to a universal wheel is that a user can bundle these wheels into an +application and ship a single binary. + +However, if you have a large library, then you might prefer to ship +the two single-arch wheels instead - `x86_64` and `arm64`. In rare cases, +you might want to build all three, but in that case, pip will not download +the universal wheels, because it prefers the most specific wheel +available. + +#### What to provide? + +Opinions vary on which of arch-specific or `universal2` wheels are best - some packagers prefer `universal2` because it's one wheel for all Mac users, so simpler, and easier to build into apps for downstream users. However, because they contain code for both architectures, their file size is larger, meaning they consume more disk space and bandwidth, and are harder to build for some projects. + +See [GitHub issue 1333](https://github.com/pypa/cibuildwheel/issues/1333) for more discussion. + +#### How? + +It's easiest to build `x86_64` wheels on `x86_64` runners, and `arm64` wheels on `arm64` runners. + +On GitHub Actions, `macos-14` runners are `arm64`, and `macos-13` runners are `x86_64`. So all you need to do is ensure both are in your build matrix. + +#### Cross-compiling + +If your CI provider doesn't offer arm64 runners yet, or you want to create `universal2`, you'll have to cross-compile. Cross-compilation can be enabled by adding extra archs to the [`CIBW_ARCHS_MACOS` option](options.md#archs) - e.g. `CIBW_ARCHS_MACOS="x86_64 universal2"`. Cross-compilation is provided by Xcode toolchain v12.2+. + +Regarding testing, + +- On an arm64 runner, it is possible to test `x86_64` wheels and both parts of a `universal2` wheel using Rosetta 2 emulation. +- On an `x86_64` runner, arm64 code can be compiled but it can't be tested. `cibuildwheel` will raise a warning to notify you of this - these warnings can be silenced by skipping testing on these platforms: `test-skip = ["*_arm64", "*_universal2:arm64"]`. + +!!! note + If your project uses **Poetry** as a build backend, cross-compiling on macOS [does not currently work](https://github.com/python-poetry/poetry/issues/7107). In some cases arm64 wheels can be built but their tags will be incorrect, with the platform tag showing `x86_64` instead of `arm64`. + + As a workaround, the tag can be fixed before running delocate to repair the wheel. The [`wheel tags`](https://wheel.readthedocs.io/en/stable/reference/wheel_tags.html) command is ideal for this. See [this workflow](https://gist.github.com/anderssonjohan/49f07e33fc5cb2420515a8ac76dc0c95#file-build-pendulum-wheels-yml-L39-L53) for an example usage of `wheel tags`. + + +## Windows {: #windows} + +### System requirements + +You must have native build tools (i.e., Visual Studio) installed. + +Because the builds are happening without full isolation, there might be some differences compared to CI builds (Visual Studio version, OS version, local files, ...) that might prevent you from finding an issue only seen in CI. + +In order to speed-up builds, cibuildwheel will cache the tools it needs to be reused for future builds. The folder used for caching is system/user dependent and is reported in the printed preamble of each run (e.g. `Cache folder: C:\Users\Matt\AppData\Local\pypa\cibuildwheel\Cache`). You can override the cache folder using the ``CIBW_CACHE_PATH`` environment variable. + +### Windows ARM64 builds {: #windows-arm64} + +`cibuildwheel` supports cross-compiling `ARM64` wheels on all Windows runners, but a native `ARM64` runner is required for testing. On non-native runners, tests for `ARM64` wheels will be automatically skipped with a warning. Add `"*-win_arm64"` to your `test-skip` setting to suppress the warning. + +Cross-compilation on Windows relies on a supported build backend. Supported backends use an environment variable to specify their target platform (the one they are compiling native modules for, as opposed to the one they are running on), which is set in [cibuildwheel's windows.py](https://github.com/pypa/cibuildwheel/blob/main/cibuildwheel/platforms/windows.py) before building. Currently, `setuptools>=65.4.1` and `setuptools_rust` are the only supported backends. + +By default, `ARM64` is not enabled when running on non-`ARM64` runners. Use [`CIBW_ARCHS`](options.md#archs) to select it. + +## Pyodide/WebAssembly {: #pyodide} + +Pyodide is offered as an experimental feature in cibuildwheel. + +### System requirements + +Pyodide builds require a Linux or macOS machine. + +### Specifying a pyodide build + +You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). + +### Choosing a Pyodide version {: #pyodide-choosing-a-version} + +It is also possible to target a specific Pyodide version by setting the [`pyodide-version`](options.md#pyodide-version) option to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json), which can be viewed more easily by installing `pyodide-build` from PyPI and using `pyodide xbuildenv search --all` to see a compatibility table. + +If there are pre-releases available for a newer Python version, the `pyodide-prerelease` [`enable`](options.md#enable) can be used to include pre-release versions. + +### Running tests + +Currently, it's recommended to run tests using a `python -m` entrypoint, rather than a command line entrypoint, or a shell script. This is because custom entrypoints have some issues in the Pyodide virtual environment. For example, `pytest` may not work as a command line entrypoint, but will work as a `python -m pytest` entrypoint. + +## iOS {: #ios} + +### System requirements + +You must be building on a macOS machine, with Xcode installed. The Xcode installation must have an iOS SDK available, with all license agreements agreed to by the user. To check if an iOS SDK is available, open the Xcode settings panel, and check the Platforms tab. This will also ensure that license agreements have been acknowledged. + +Building iOS wheels also requires a working macOS Python installation. See the notes on [macOS builds](#macos) for details about configuration of the macOS environment. + +### Specifying an iOS build + +iOS is effectively 2 platforms - physical devices, and simulators. While the API for these two platforms are identical, the ABI is not compatible, even when dealing with a device and simulator with the same CPU architecture. For this reason, the architecture specification for iOS builds includes *both* the CPU architecture *and* the ABI that is being targeted. There are three possible values for architecture on iOS; the values match those used by `sys.implementation._multiarch` when running on iOS (with hyphens replaced with underscores, matching wheel filename normalization): + +* `arm64_iphoneos` (for physical iOS devices); +* `arm64_iphonesimulator` (for iOS simulators running on Apple Silicon macOS machines); and +* `x64_64_iphonesimulator` (for iOS simulators running on Intel macOS machines). + +By default, cibuildwheel will build all wheels for the CPU architecture of the build machine. You can build all wheels for all architectures by specifying `--archs all`. + +If you need to specify different compilation flags or other properties on a per-ABI or per-CPU basis, you can use [configuration overrides](configuration.md#overrides) with a `select` clause that targets the specific ABI or architecture. For example, consider the following example: + +```toml +[tool.cibuildwheel.ios] +test-sources = ["tests"] +test-requires = ["pytest"] + +[[tool.cibuildwheel.overrides]] +select = "*_iphoneos" +environment.PATH = "/path/to/special/device/details:..." + +[[tool.cibuildwheel.overrides]] +select = "*-ios_arm64_*" +inherit.test-requires = "append" +test-requires = ["arm64-testing-helper"] +``` + +This configuration would: + + * Specify a `test-sources` and `test-requires` for all iOS targets; + * Add a `PATH` setting that will be used on physical iOS devices; and + * Add `arm64-testing-helper` to the test environment for all ARM64 iOS devices (whether simulator or device). + +### iOS version compatibility + +iOS builds will honor the `IPHONEOS_DEPLOYMENT_TARGET` environment variable to set the minimum supported API version for generated wheels. This will default to `13.0` if the environment variable isn't set. + +### Cross platform builds + +iOS builds are *cross platform builds*, as it not possible to run compilers and other build tools "on device". The pre-compiled iOS binaries used to support iOS builds include tooling that can convert any virtual environment into a cross platform virtual environment - that is, an environment that can run binaries on the build machine (macOS), but, if asked, will respond as if it is an iOS machine. This allows `pip`, `build`, and other build tools to perform iOS-appropriate behaviour. + +### Build frontend support + +iOS builds support both the `pip` and `build` build frontends. In principle, support for `uv` with the `build[uv]` frontend should be possible, but `uv` [doesn't currently have support for cross-platform builds](https://github.com/astral-sh/uv/issues/7957), and [doesn't have support for iOS (or Android) tags](https://github.com/astral-sh/uv/issues/8029). + +### Build environment + +The environment used to run builds does not inherit the full user environment - in particular, `PATH` is deliberately re-written. This is because UNIX C tooling doesn't do a great job differentiating between "macOS ARM64" and "iOS ARM64" binaries. If (for example) Homebrew is on the path when compilation commands are invoked, it's easy for a macOS version of a library to be linked into the iOS binary, rendering it unusable on iOS. To prevent this, iOS builds always force `PATH` to a "known minimal" path, that includes only the bare system utilities, and the iOS compiler toolchain. + +If your project requires additional tools to build (such as `cmake`, `ninja`, or `rustc`), those tools must be explicitly declared as cross-build tools using [`xbuild-tools`](options.md#xbuild-tools). *Any* tool used by the build process must be included in the `xbuild-tools` list, not just tools that cibuildwheel will invoke directly. For example, if your build script invokes `cmake`, and the `cmake` script invokes `magick` to perform some image transformations, both `cmake` and `magick` must be included in your cross-build tools list. + +### Tests + +If tests have been configured, the test suite will be executed on the simulator matching the architecture of the build machine - that is, if you're building on an ARM64 macOS machine, the ARM64 wheel will be tested on an ARM64 simulator. It is not possible to use cibuildwheel to test wheels on other simulators, or on physical devices. + +The iOS test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. In addition, the project must use [`test-sources`](options.md#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the simulator device will not have access to the local project directory. + +The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. diff --git a/docs/setup.md b/docs/setup.md index 63f78aa12..03a13491b 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -1,351 +1,68 @@ --- -title: 'Setup' +title: 'Getting started' --- -# Run cibuildwheel locally (optional) {: #local} +# Getting started -Before getting to CI setup, it can be convenient to test cibuildwheel -locally to quickly iterate and track down issues without even touching CI. +Before getting to [CI setup](ci-services.md), it can be convenient to test cibuildwheel locally to quickly iterate and track down issues without having to commit each change, push, and then check CI logs. Install cibuildwheel and run a build like this: -!!! tab "Linux" +```sh +# run using uv +uvx cibuildwheel - Using [pipx](https://github.com/pypa/pipx): - ```sh - pipx run cibuildwheel --platform linux - ``` - - Or, - ```sh - pip install cibuildwheel - cibuildwheel --platform linux - ``` - -!!! tab "macOS" - - Using [pipx](https://github.com/pypa/pipx): - ```sh - pipx run cibuildwheel --platform macos - ``` - - Or, - ```sh - pip install cibuildwheel - cibuildwheel --platform macos - ``` - - -!!! tab "Windows" - - Using [pipx](https://github.com/pypa/pipx): - ```bat - pipx run cibuildwheel --platform windows - ``` - - Or, - ```bat - pip install cibuildwheel - cibuildwheel --platform windows - ``` - -You should see the builds taking place. You can experiment with options using environment variables or pyproject.toml. - -!!! tab "Environment variables" - - cibuildwheel will read config from the environment. Syntax varies, depending on your shell: - - > POSIX shell (Linux/macOS) - - ```sh - # run a command to set up the build system - export CIBW_BEFORE_ALL='apt install libpng-dev' - - cibuildwheel --platform linux - ``` +# or pipx +pipx run cibuildwheel - > CMD (Windows) +# or, install it first +pip install cibuildwheel +cibuildwheel +``` - ```bat - set CIBW_BEFORE_ALL='apt install libpng-dev' +!!!tip + You can pass the `--platform linux` option to cibuildwheel to build Linux wheels, even if you're not on Linux. On most machines, the easiest builds to try are the Linux builds. You don't need any software installed except a Docker daemon, such as [Docker Desktop](https://www.docker.com/get-started/). Each platform that cibuildwheel supports has its own system requirements and platform-specific behaviors. See the [platforms page](platforms.md) for details. - cibuildwheel --platform linux - ``` +You should see the builds taking place. You can experiment with [options](options.md) using pyproject.toml or environment variables. !!! tab "pyproject.toml" - If you write your options into [`pyproject.toml`](options.md#configuration-file), you can work on your options locally, and they'll be automatically picked up when running in CI. + If you write your options into [`pyproject.toml`](configuration.md#configuration-file), you can work on your options locally, and they'll be automatically picked up when running in CI. > pyproject.toml ``` [tool.cibuildwheel] - before-all = "apt install libpng-dev" + before-all = "uname -a" ``` Then invoke cibuildwheel, like: ```console - cibuildwheel --platform linux - ``` - -## Linux builds - -If you've got [Docker](https://www.docker.com/products/docker-desktop) installed on -your development machine, you can run a Linux build. - -!!! tip - You can run the Linux build on any platform. Even Windows can run - Linux containers these days, but there are a few hoops to jump - through. Check [this document](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/quick-start-windows-10-linux) - for more info. - -Because the builds are happening in manylinux Docker containers, -they're perfectly reproducible. - -The only side effect to your system will be docker images being pulled. - -## macOS / Windows builds - -Pre-requisite: you need to have native build tools installed. - -Because the builds are happening without full isolation, there might be some -differences compared to CI builds (Xcode version, Visual Studio version, -OS version, local files, ...) that might prevent you from finding an issue only -seen in CI. - -In order to speed-up builds, cibuildwheel will cache the tools it needs to be -reused for future builds. The folder used for caching is system/user dependent and is -reported in the printed preamble of each run (e.g. "Cache folder: /Users/Matt/Library/Caches/cibuildwheel"). - -You can override the cache folder using the ``CIBW_CACHE_PATH`` environment variable. - -!!! warning - cibuildwheel uses official python.org macOS installers for CPython but - those can only be installed globally. - - In order not to mess with your system, cibuildwheel won't install those if they are - missing. Instead, it will error out with a message to let you install the missing - CPython: - - ```console - Error: CPython 3.6 is not installed. - cibuildwheel will not perform system-wide installs when running outside of CI. - To build locally, install CPython 3.6 on this machine, or, disable this version of Python using CIBW_SKIP=cp36-macosx_* - - Download link: https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg - ``` - - -# Configure a CI service - -## GitHub Actions [linux/mac/windows] {: #github-actions} - -To build Linux, Mac, and Windows wheels using GitHub Actions, create a `.github/workflows/build_wheels.yml` file in your repo. - -!!! tab "Action" - For GitHub Actions, `cibuildwheel` provides an action you can use. This is - concise and enables easier auto updating via GitHub's Dependabot; see - [Automatic updates](faq.md#automatic-updates). - - > .github/workflows/build_wheels.yml - - ```yaml - {% include "../examples/github-minimal.yml" %} + cibuildwheel ``` +!!! tab "Environment variables" - You can use `env:` with the action just like you would with `run:`; you can - also use `with:` to set the command line options: `package-dir: .` and - `output-dir: wheelhouse` (those values are the defaults). - -!!! tab "pipx" - The GitHub Actions runners have pipx installed, so you can easily build in - just one line. This is internally how the action works; the main benefit of - the action form is easy updates via GitHub's Dependabot. - - > .github/workflows/build_wheels.yml - - ```yaml - name: Build - - on: [push, pull_request] - - jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04, windows-2019, macos-10.15] + cibuildwheel will read config from the environment. Syntax varies, depending on your shell: - steps: - - uses: actions/checkout@v2 + > POSIX shell (Linux/macOS) - - name: Build wheels - run: pipx run cibuildwheel==2.3.1 + ```sh + # run a command to set up the build system + export CIBW_BEFORE_ALL='uname -a' - - uses: actions/upload-artifact@v2 - with: - path: ./wheelhouse/*.whl + cibuildwheel ``` -!!! tab "Generic" - This is the most generic form using setup-python and pip; it looks the most - like the other CI examples. If you want to avoid having setup that takes - advantage of GitHub Actions features or pipx being preinstalled, this might - appeal to you. - - > .github/workflows/build_wheels.yml - - ```yaml - name: Build - - on: [push, pull_request] - - jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04, windows-2019, macos-10.15] - - steps: - - uses: actions/checkout@v2 - - # Used to host cibuildwheel - - uses: actions/setup-python@v2 - - - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.3.1 + > CMD (Windows) - - name: Build wheels - run: python -m cibuildwheel --output-dir wheelhouse + ```bat + set CIBW_BEFORE_ALL='uname -a' - - uses: actions/upload-artifact@v2 - with: - path: ./wheelhouse/*.whl + cibuildwheel ``` - -Commit this file, and push to GitHub - either to your default branch, or to a PR branch. The build should start automatically. - -For more info on this file, check out the [docs](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions). - -[`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) extends this minimal example with a demonstration of how to automatically upload the built wheels to PyPI. - - -## Azure Pipelines [linux/mac/windows] {: #azure-pipelines} - -To build Linux, Mac, and Windows wheels on Azure Pipelines, create a `azure-pipelines.yml` file in your repo. - -> azure-pipelines.yml - -```yaml -{% include "../examples/azure-pipelines-minimal.yml" %} -``` - -Commit this file, enable building of your repo on Azure Pipelines, and push. - -Wheels will be stored for you and available through the Pipelines interface. For more info on this file, check out the [docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema). - -## Travis CI [linux/windows] {: #travis-ci} - -To build Linux and Windows wheels on Travis CI, create a `.travis.yml` file in your repo. - -> .travis.yml - -```yaml -{% include "../examples/travis-ci-minimal.yml" %} -``` - -Commit this file, enable building of your repo on Travis CI, and push. - -Then setup a deployment method by following the [Travis CI deployment docs](https://docs.travis-ci.com/user/deployment/), or see [Delivering to PyPI](deliver-to-pypi.md). For more info on `.travis.yml`, check out the [docs](https://docs.travis-ci.com/). - -[`examples/travis-ci-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/travis-ci-deploy.yml) extends this minimal example with a demonstration of how to automatically upload the built wheels to PyPI. - -## AppVeyor [linux/mac/windows] {: #appveyor} - -To build Linux, Mac, and Windows wheels on AppVeyor, create an `appveyor.yml` file in your repo. - -> appveyor.yml - -```yaml -{% include "../examples/appveyor-minimal.yml" %} -``` - -Commit this file, enable building of your repo on AppVeyor, and push. - -AppVeyor will store the built wheels for you - you can access them from the project console. Alternatively, you may want to store them in the same place as the Travis CI build. See [AppVeyor deployment docs](https://www.appveyor.com/docs/deployment/) for more info, or see [Delivering to PyPI](deliver-to-pypi.md) below. - -For more info on this config file, check out the [docs](https://www.appveyor.com/docs/). - -## CircleCI [linux/mac] {: #circleci} - -To build Linux and Mac wheels on CircleCI, create a `.circleci/config.yml` file in your repo, - -> .circleci/config.yml - -```yaml -{% include "../examples/circleci-minimal.yml" %} -``` - -Commit this file, enable building of your repo on CircleCI, and push. - -!!! note - CircleCI doesn't enable free macOS containers for open source by default, but you can ask for access. See [here](https://circleci.com/docs/2.0/oss/#overview) for more information. - -CircleCI will store the built wheels for you - you can access them from the project console. Check out the CircleCI [docs](https://circleci.com/docs/2.0/configuration-reference/#section=configuration) for more info on this config file. - -## Gitlab CI [linux] {: #gitlab-ci} - -To build Linux wheels on Gitlab CI, create a `.gitlab-ci.yml` file in your repo, - -> .gitlab-ci.yml - -```yaml -{% include "../examples/gitlab-minimal.yml" %} -``` - -Commit this file, and push to Gitlab. The pipeline should start automatically. - -Gitlab will store the built wheels for you - you can access them from the Pipelines view. Check out the Gitlab [docs](https://docs.gitlab.com/ee/ci/yaml/) for more info on this config file. - -> ⚠️ Got an error? Check the [FAQ](faq.md). - -# Next steps - -Once you've got the wheel building successfully, you might want to set up [testing](options.md#test-command) or [automatic releases to PyPI](deliver-to-pypi.md#automatic-method). - - +- Once you've got a build working locally, you can move on to [setting up a CI service](ci-services.md). +- View the [full options reference](options.md) to see what cibuildwheel can do. +- Check out the [FAQ](faq.md) for common questions. diff --git a/docs/theme_overrides/js/theme.js b/docs/theme_overrides/js/theme.js new file mode 100644 index 000000000..05689756c --- /dev/null +++ b/docs/theme_overrides/js/theme.js @@ -0,0 +1,288 @@ +// imported from +// https://github.com/readthedocs/sphinx_rtd_theme/blob/bc73ad84068b4ee7d8ddae7e51f5b767e8fd377b/src/theme.js +// modified to suit our documentation structure, to not scroll the nav sidebar +// when clicked - this is disorienting + +var jQuery = window.jQuery; + +// Sphinx theme nav state +function ThemeNav () { + + var nav = { + navBar: null, + win: null, + winScroll: false, + winResize: false, + linkScroll: false, + winPosition: 0, + winHeight: null, + docHeight: null, + isRunning: false + }; + + nav.enable = function (withStickyNav) { + var self = this; + + // TODO this can likely be removed once the theme javascript is broken + // out from the RTD assets. This just ensures old projects that are + // calling `enable()` get the sticky menu on by default. All other cals + // to `enable` should include an argument for enabling the sticky menu. + if (typeof(withStickyNav) == 'undefined') { + withStickyNav = true; + } + + if (self.isRunning) { + // Only allow enabling nav logic once + return; + } + + self.isRunning = true; + jQuery(function ($) { + self.init($); + + self.reset(); + self.win.on('hashchange', self.reset); + + if (withStickyNav) { + // Set scroll monitor + self.win.on('scroll', function () { + if (!self.linkScroll) { + if (!self.winScroll) { + self.winScroll = true; + requestAnimationFrame(function() { self.onScroll(); }); + } + } + }); + } + + // Set resize monitor + self.win.on('resize', function () { + if (!self.winResize) { + self.winResize = true; + requestAnimationFrame(function() { self.onResize(); }); + } + }); + + self.onResize(); + }); + + }; + + // TODO remove this with a split in theme and Read the Docs JS logic as + // well, it's only here to support 0.3.0 installs of our theme. + nav.enableSticky = function() { + this.enable(true); + }; + + nav.init = function ($) { + var doc = $(document), + self = this; + + this.navBar = $('div.wy-side-scroll:first'); + this.win = $(window); + + // Set up javascript UX bits + $(document) + // Shift nav in mobile when clicking the menu. + .on('click', "[data-toggle='wy-nav-top']", function() { + $("[data-toggle='wy-nav-shift']").toggleClass("shift"); + $("[data-toggle='rst-versions']").toggleClass("shift"); + }) + + // Nav menu link click operations + .on('click', ".wy-menu-vertical .current ul li a", function() { + var target = $(this); + // Close menu when you click a link. + $("[data-toggle='wy-nav-shift']").removeClass("shift"); + $("[data-toggle='rst-versions']").toggleClass("shift"); + // Handle dynamic display of l3 and l4 nav lists + self.toggleCurrent(target); + self.hashChange(); + }) + .on('click', "[data-toggle='rst-current-version']", function() { + $("[data-toggle='rst-versions']").toggleClass("shift-up"); + }) + + // Make tables responsive + $("table.docutils:not(.field-list,.footnote,.citation)") + .wrap("
"); + + // Add extra class to responsive tables that contain + // footnotes or citations so that we can target them for styling + $("table.docutils.footnote") + .wrap("
"); + $("table.docutils.citation") + .wrap("
"); + + // Add expand links to all parents of nested ul + $('.wy-menu-vertical ul').not('.simple').siblings('a').each(function () { + var link = $(this); + expand = + $(''); + expand.on('click', function (ev) { + self.toggleCurrent(link); + ev.stopPropagation(); + return false; + }); + link.prepend(expand); + }); + + // EDIT by joerick + // + // workaround a bug with the site in safari. safari navigates to the + // anchor before the above code has run, specifically wrap with + // .wy-table-responsive activates css rules that change the size of + // tables, making anchor points move around. + console.log('Document ready, checking for anchor in URL'); + if (window.location.hash) { + const anchorEl = document.querySelector(window.location.hash); + anchorEl.getBoundingClientRect(); // Force layout to ensure scrollIntoView works correctly + if (anchorEl) { + console.log('Anchor element:', anchorEl); + anchorEl.scrollIntoView({ behavior: 'instant', block: 'start' }); + } + } + // end edit by joerick + }; + + nav.reset = function () { + // Get anchor from URL and open up nested nav + var anchor = encodeURI(window.location.hash) || '#'; + + try { + var vmenu = $('.wy-menu-vertical'); + var link = vmenu.find('[href="' + anchor + '"]'); + if (link.length === 0) { + // this link was not found in the sidebar. + // Find associated id element, then its closest section + // in the document and try with that one. + var id_elt = $('.document [id="' + anchor.substring(1) + '"]'); + var closest_section = id_elt.closest('div.section'); + link = vmenu.find('[href="#' + closest_section.attr("id") + '"]'); + if (link.length === 0) { + // still not found in the sidebar. fall back to main section + link = vmenu.find('[href="#"]'); + } + } + // If we found a matching link then reset current and re-apply + // otherwise retain the existing match + if (link.length > 0) { + $('.wy-menu-vertical .current') + .removeClass('current') + .attr('aria-expanded','false'); + link.addClass('current') + .attr('aria-expanded','true'); + link.closest('li.toctree-l1') + .parent() + .addClass('current') + .attr('aria-expanded','true'); + for (let i = 1; i <= 10; i++) { + link.closest('li.toctree-l' + i) + .addClass('current') + .attr('aria-expanded','true'); + } + + // EDIT by joerick + console.log('no scroll') + // link[0].scrollIntoView(); + } + } + catch (err) { + console.log("Error expanding nav for anchor", err); + } + + }; + + nav.onScroll = function () { + this.winScroll = false; + var newWinPosition = this.win.scrollTop(), + winBottom = newWinPosition + this.winHeight, + navPosition = this.navBar.scrollTop(), + newNavPosition = navPosition + (newWinPosition - this.winPosition); + if (newWinPosition < 0 || winBottom > this.docHeight) { + return; + } + this.navBar.scrollTop(newNavPosition); + this.winPosition = newWinPosition; + }; + + nav.onResize = function () { + this.winResize = false; + this.winHeight = this.win.height(); + this.docHeight = $(document).height(); + }; + + nav.hashChange = function () { + this.linkScroll = true; + this.win.one('hashchange', function () { + this.linkScroll = false; + }); + }; + + nav.toggleCurrent = function (elem) { + var parent_li = elem.closest('li'); + parent_li + .siblings('li.current') + .removeClass('current') + .attr('aria-expanded','false'); + parent_li + .siblings() + .find('li.current') + .removeClass('current') + .attr('aria-expanded','false'); + var children = parent_li.find('> ul li'); + // Don't toggle terminal elements. + if (children.length) { + children + .removeClass('current') + .attr('aria-expanded','false'); + parent_li + .toggleClass('current') + .attr('aria-expanded', function(i, old) { + return old == 'true' ? 'false' : 'true'; + }); + } + } + + return nav; +}; + +const ThemeNavInstance = ThemeNav(); + + window.SphinxRtdTheme = { + Navigation: ThemeNavInstance, + // TODO remove this once static assets are split up between the theme + // and Read the Docs. For now, this patches 0.3.0 to be backwards + // compatible with a pre-0.3.0 layout.html + StickyNav: ThemeNavInstance, + }; + + +// requestAnimationFrame polyfill by Erik Möller. fixes from Paul Irish and Tino Zijdel +// https://gist.github.com/paulirish/1579671 +// MIT license + +(function() { + var lastTime = 0; + var vendors = ['ms', 'moz', 'webkit', 'o']; + for(var x = 0; x < vendors.length && !window.requestAnimationFrame; ++x) { + window.requestAnimationFrame = window[vendors[x]+'RequestAnimationFrame']; + window.cancelAnimationFrame = window[vendors[x]+'CancelAnimationFrame'] + || window[vendors[x]+'CancelRequestAnimationFrame']; + } + + if (!window.requestAnimationFrame) + window.requestAnimationFrame = function(callback, element) { + var currTime = new Date().getTime(); + var timeToCall = Math.max(0, 16 - (currTime - lastTime)); + var id = window.setTimeout(function() { callback(currTime + timeToCall); }, + timeToCall); + lastTime = currTime + timeToCall; + return id; + }; + + if (!window.cancelAnimationFrame) + window.cancelAnimationFrame = function(id) { + clearTimeout(id); + }; +}()); diff --git a/docs/working-examples.md b/docs/working-examples.md index 68a6650b3..146ef39d8 100644 --- a/docs/working-examples.md +++ b/docs/working-examples.md @@ -9,273 +9,235 @@ title: Working examples | Name | CI | OS | Notes | |-----------------------------------|----|----|:------| | [scikit-learn][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The machine learning library. A complex but clean config using many of cibuildwheel's features to build a large project with Cython and C++ extensions. | -| [Tornado][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed. | | [pytorch-fairseq][] | ![github icon][] | ![apple icon][] ![linux icon][] | Facebook AI Research Sequence-to-Sequence Toolkit written in Python. | +| [duckdb][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | DuckDB is an analytical in-process SQL database management system | +| [NumPy][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The fundamental package for scientific computing with Python. | +| [Tornado][] | ![github icon][] | ![linux icon][] ![apple icon][] ![windows icon][] | Tornado is a Python web framework and asynchronous networking library. Uses stable ABI for a small C extension. | +| [NCNN][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | ncnn is a high-performance neural network inference framework optimized for the mobile platform | | [Matplotlib][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The venerable Matplotlib, a Python library with C++ portions | -| [MyPy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | MyPyC, the compiled component of MyPy. | -| [pydantic][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Data parsing and validation using Python type hints | +| [MyPy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | The compiled version of MyPy using MyPyC. | +| [Prophet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Tool for producing high quality forecasts for time series data that has multiple seasonality with linear or non-linear growth. | +| [Kivy][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Open source UI framework written in Python, running on Windows, Linux, macOS, Android and iOS | +| [Triton][] | ![github icon][] | ![linux icon][] | Self hosted runners | +| [MemRay][] | ![github icon][] | ![linux icon][] | Memray is a memory profiler for Python | | [uvloop][] | ![github icon][] | ![apple icon][] ![linux icon][] | Ultra fast asyncio event loop. | | [psutil][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Cross-platform lib for process and system monitoring in Python | -| [vaex][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Out-of-Core hybrid Apache Arrow/NumPy DataFrame for Python, ML, visualization and exploration of big tabular data at a billion rows per second 🚀 | | [Google Benchmark][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A microbenchmark support library | +| [vaex][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Out-of-Core hybrid Apache Arrow/NumPy DataFrame for Python, ML, visualization and exploration of big tabular data at a billion rows per second 🚀 | +| [Apache Beam][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Apache Beam is a unified programming model for Batch and Streaming data processing. | +| [PyGame][] | ![github icon][] | ![apple icon][] ![linux icon][] | 🐍🎮 pygame (the library) is a Free and Open Source python programming language library for making multimedia applications like games built on top of the excellent SDL library. C, Python, Native, OpenGL. | | [asyncpg][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A fast PostgreSQL Database Client Library for Python/asyncio. | -| [Apache Beam][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Apache Beam is a unified programming model for Batch and Streaming | +| [cmake][] | ![github icon][] ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Multitagged binary builds for all supported platforms, using cibw 2 config configuration. | +| [pyinstrument][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Python profiler with a C extension. No external dependencies. | | [scikit-image][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Image processing library. Uses cibuildwheel to build and test a project that uses Cython with platform-native code. | +| [PyOxidizer][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A modern Python application packaging and distribution tool | | [twisted-iocpsupport][] | ![github icon][] | ![windows icon][] | A submodule of Twisted that hooks into native C APIs using Cython. | -| [cmake][] | ![github icon][] ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Multitagged binary builds for all supported platforms, using cibw 2 config configuration. | -| [duckdb][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | DuckDB is an in-process SQL OLAP Database Management System | -| [websockets][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Library for building WebSocket servers and clients. Mostly written in Python, with a small C 'speedups' extension module. | | [cvxpy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A Python-embedded modeling language for convex optimization problems. | -| [Triton][] | ![github icon][] | ![linux icon][] | Self hosted runners | -| [PyOxidizer][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A modern Python application packaging and distribution tool | -| [OpenSpiel][] | ![github icon][] | ![apple icon][] ![linux icon][] | OpenSpiel is a collection of environments and algorithms for research in general reinforcement learning and search/planning in games. | +| [pedalboard][] | ![github icon][] | ![windows icon][] ![linux icon][] ![apple icon][] | A Python library for working with audio data and audio plugins by wrapping the [JUCE](https://github.com/juce-framework/JUCE/) C++ framework. Uses cibuildwheel to deploy on as many operating systems and Python versions as possible with only one dependency (any NumPy). | +| [websockets][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Library for building WebSocket servers and clients. Mostly written in Python, with a small C 'speedups' extension module. | | [River][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | 🌊 Online machine learning in Python | -| [pyzmq][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Python bindings for zeromq, the networking library. Uses Cython and CFFI. | -| [vispy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Main repository for Vispy | | [aiortc][] | ![github icon][] | ![apple icon][] ![linux icon][] | WebRTC and ORTC implementation for Python using asyncio. | -| [Confluent client for Kafka][] | ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | setup in `tools/wheels/build-wheels.bat` | +| [OpenSpiel][] | ![github icon][] | ![apple icon][] ![linux icon][] | OpenSpiel is a collection of environments and algorithms for research in general reinforcement learning and search/planning in games. | +| [UltraJSON][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Ultra fast JSON decoder and encoder written in C with Python bindings | +| [Dependency Injector][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Dependency injection framework for Python, uses Windows TravisCI | +| [pyzmq][] | ![github icon][] ![circleci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Python bindings for zeromq, the networking library. Uses Cython on CPython and CFFI on PyPy. ARM wheels for linux are built natively on CircleCI. | +| [CTranslate2][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes libraries from the [Intel oneAPI toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) and CUDA kernels compiled for multiple GPU architectures. | +| [Implicit][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes GPU support for linux wheels | | [tinyobjloader][] | ![azurepipelines icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Tiny but powerful single file wavefront obj loader | +| [vispy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Main repository for Vispy | | [coverage.py][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The coverage tool for Python | | [PyCryptodome][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A self-contained cryptographic library for Python | -| [Dependency Injector][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Dependency injection framework for Python, uses Windows TravisCI | +| [Line Profiler][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Line-by-line profiling for Python | +| [PyAV][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Pythonic bindings for FFmpeg's libraries. | | [PyYAML][] | ![github icon][] | ![apple icon][] | Canonical source repository for PyYAML | -| [numexpr][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Fast numerical array expression evaluator for Python, NumPy, PyTables, pandas, bcolz and more | +| [pikepdf][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python library for reading and writing PDF, powered by QPDF | +| [numexpr][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Fast numerical array expression evaluator for Python, NumPy, Pandas, PyTables and more | +| [Wrapt][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python module for decorators, wrappers and monkey patching. | | [h5py][] | ![azurepipelines icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | HDF5 for Python -- The h5py package is a Pythonic interface to the HDF5 binary data format. | -| [PyAV][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Pythonic bindings for FFmpeg's libraries. | +| [envd][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A machine learning development environment build tool | +| [Psycopg 3][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A modern implementation of a PostgreSQL adapter for Python | | [OpenColorIO][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A color management framework for visual effects and animation. | -| [PyTables][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python package to manage extremely large amounts of data | -| [Line Profiler][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Line-by-line profiling for Python | -| [OpenTimelineIO][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Open Source API and interchange format for editorial timeline information. | | [ruptures][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Extensive Cython + NumPy [pyproject.toml](https://github.com/deepcharles/ruptures/blob/master/pyproject.toml) example. | -| [pikepdf][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python library for reading and writing PDF, powered by qpdf | | [aioquic][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | QUIC and HTTP/3 implementation in Python | -| [DeepForest][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | An Efficient, Scalable and Optimized Python Framework for Deep Forest (2021.2.1) | +| [SimpleJSON][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | simplejson is a simple, fast, extensible JSON encoder/decoder for Python | +| [OpenTimelineIO][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Open Source API and interchange format for editorial timeline information. | +| [PyTables][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python package to manage extremely large amounts of data | | [google neuroglancer][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | WebGL-based viewer for volumetric data | -| [AutoPy][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes a Windows Travis build. | | [Parselmouth][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python interface to the Praat software package, using pybind11, C++17 and CMake, with the core Praat static library built only once and shared between wheels. | -| [Psycopg 3][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A modern implementation of a PostgreSQL adapter for Python | +| [DeepForest][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | An Efficient, Scalable and Optimized Python Framework for Deep Forest (2021.2.1) | +| [AutoPy][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes a Windows Travis build. | | [H3-py][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Python bindings for H3, a hierarchical hexagonal geospatial indexing system | -| [markupsafe][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Safely add untrusted strings to HTML/XML markup. | -| [Rtree][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Rtree: spatial index for Python GIS ¶ | -| [python-rapidjson][] | ![travisci icon][] ![gitlab icon][] ![appveyor icon][] | ![windows icon][] ![linux icon][] | Python wrapper around rapidjson | -| [python-snappy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Python bindings for the snappy google library | +| [time-machine][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Time mocking library using only the CPython C API. | +| [mosec][] | ![github icon][] | ![linux icon][] ![apple icon][] | A machine learning model serving framework powered by Rust | +| [Picologging][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A high-performance logging library for Python. | | [pybind11 cmake_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Example pybind11 module built with a CMake-based build system | +| [markupsafe][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Safely add untrusted strings to HTML/XML markup. | +| [Rtree][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Rtree: spatial index for Python GIS | | [KDEpy][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Kernel Density Estimation in Python | +| [dd-trace-py][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Uses custom alternate arch emulation on GitHub | | [tgcalls][] | ![github icon][] | ![apple icon][] ![windows icon][] | Python `pybind11` binding to Telegram's WebRTC library with third party dependencies like `OpenSSL`, `MozJPEG`, `FFmpeg`, etc. | | [pybind11 python_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Example pybind11 module built with a Python-based build system | -| [dd-trace-py][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Uses custom alternate arch emulation on GitHub | -| [cyvcf2][] | ![github icon][] | ![apple icon][] ![linux icon][] | cython + htslib == fast VCF and BCF processing | +| [python-rapidjson][] | ![travisci icon][] ![gitlab icon][] | ![windows icon][] ![linux icon][] | Python wrapper around rapidjson | | [sourmash][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Quickly search, compare, and analyze genomic and metagenomic data sets. | -| [time-machine][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Time mocking library using only the CPython C API. | -| [CTranslate2][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes libraries from the [Intel oneAPI toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) and CUDA kernels compiled for multiple GPU architectures. | -| [matrixprofile][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python 3 library making time series data mining tasks, utilizing matrix profile algorithms, accessible to everyone. | +| [abess][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A fast best-subset selection library. It uses cibuildwheel to build a large project with C++ extensions. | +| [python-snappy][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Python bindings for the snappy google library | +| [cyvcf2][] | ![github icon][] | ![apple icon][] ![linux icon][] | cython + htslib == fast VCF and BCF processing | | [jq.py][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Python bindings for jq | -| [iminuit][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Jupyter-friendly Python interface for C++ MINUIT2 | +| [matrixprofile][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python 3 library making time series data mining tasks, utilizing matrix profile algorithms, accessible to everyone. | | [Tokenizer][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Fast and customizable text tokenization library with BPE and SentencePiece support | +| [iminuit][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Jupyter-friendly Python interface for C++ MINUIT2 | +| [Confluent client for Kafka][] | ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | setup in `tools/wheels/build-wheels.bat` | +| [pillow-heif][] | ![github icon][] ![cirrusci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Bindings to libheif library with third party dependencies. Fully automated CI for tests and publishing including Apple Silicon builds. | +| [keyvi][] | ![github icon][] | ![linux icon][] ![apple icon][] | FST based key value index highly optimized for size and lookup performance, utilizes ccache action for improved runtime | | [PyGLM][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Fast OpenGL Mathematics (GLM) for Python | -| [bx-python][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | A library that includes Cython extensions. | -| [boost-histogram][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Supports full range of wheels, including PyPy and alternate archs. | -| [iDynTree][] | ![github icon][] | ![linux icon][] | Uses manylinux_2_24 | | [TgCrypto][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes a Windows Travis build. | +| [iDynTree][] | ![github icon][] | ![linux icon][] | Uses manylinux_2_24 | +| [streaming-form-data][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Streaming parser for multipart/form-data written in Cython | +| [power-grid-model][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Python/C++ library for distribution power system analysis | | [pybase64][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Fast Base64 encoding/decoding in Python | -| [etebase-py][] | ![travisci icon][] | ![linux icon][] | Python bindings to a Rust library using `setuptools-rust`, and `sccache` for improved speed. | -| [fathon][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | python package for DFA (Detrended Fluctuation Analysis) and related algorithms | +| [bx-python][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | A library that includes Cython extensions. | +| [boost-histogram][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Supports full range of wheels, including PyPy and alternate archs. | | [Imagecodecs (fork)][] | ![azurepipelines icon][] | ![apple icon][] ![linux icon][] | Over 20 external dependencies in compiled libraries, custom docker image, `libomp`, `openblas` and `install_name_tool` for macOS. | -| [numpythia][] | ![github icon][] | ![apple icon][] ![linux icon][] | The interface between PYTHIA and NumPy | -| [pyjet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The interface between FastJet and NumPy | -| [polaroid][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Full range of wheels for setuptools rust, with auto release and PyPI deploy. | +| [Python-WebRTC][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | a Python extension that provides bindings to WebRTC M92 | +| [fathon][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | python package for DFA (Detrended Fluctuation Analysis) and related algorithms | +| [pybind11 scikit_build_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | An example combining scikit-build and pybind11 | +| [Arbor][] | ![github icon][] | ![apple icon][] ![linux icon][] | Arbor is a multi-compartment neuron simulation library; compatible with next-generation accelerators; best-practices applied to research software; focused on community-driven development. Includes a [small script](https://github.com/arbor-sim/arbor/blob/master/scripts/patchwheel.py) patching `rpath` in bundled libraries. | | [clang-format][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Scikit-build wrapper around LLVM's CMake, all platforms, generic wheels. | +| [polaroid][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Full range of wheels for setuptools rust, with auto release and PyPI deploy. | | [ninja][] | ![github icon][] ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Multitagged binary builds for all supported platforms, using cibw 2 config configuration. | -| [pybind11 scikit_build_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | An example combining scikit-build and pybind11 | +| [etebase-py][] | ![travisci icon][] | ![linux icon][] | Python bindings to a Rust library using `setuptools-rust`, and `sccache` for improved speed. | +| [cf-units][] | ![github icon][] | ![apple icon][] ![linux icon][] | Units of measure as required by the Climate and Forecast (CF) Metadata Conventions | +| [numpythia][] | ![github icon][] | ![apple icon][] ![linux icon][] | The interface between PYTHIA and NumPy | +| [pyjet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The interface between FastJet and NumPy | +| [ril][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A python binding to Rust Imaging library using maturin and Pyo3, utilizes Github Action cache to improve speed. Builds abi3 wheels. | +| [SiPM][] | ![github icon][] | ![apple icon][] ![linux icon][] | High performance library for SiPM detectors simulation using C++17, OpenMP and AVX2 intrinsics. | +| [aalink][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Async Python interface for Ableton Link. | | [GSD][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Cython and NumPy project with 64-bit wheels. | -| [pyinstrument_cext][] | ![travisci icon][] ![appveyor icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A simple C extension, without external dependencies | -| [pillow-heif][] | ![github icon][] | ![apple icon][] ![linux icon][] | Python CFFI binding to libheif library with third party dependencies like `libde265`, `x265`, `libaom` with test & publishing on PyPi. | -| [xmlstarlet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Python 3.6+ CFFI bindings with true MSVC build. | | [CorrectionLib][] | ![github icon][] | ![apple icon][] ![linux icon][] | Structured JSON powered correction library for HEP, designed for the CMS experiment at CERN. | -| [SiPM][] | ![github icon][] | ![apple icon][] ![linux icon][] | High performance library for SiPM detectors simulation using C++17, OpenMP and AVX2 intrinsics. | +| [xmlstarlet][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Python 3.6+ CFFI bindings with true MSVC build. | +| [werpy][] | ![github icon][] | ![windows icon][] ![linux icon][] ![apple icon][] | An ultra-fast python package using optimized dynamic programming to compute the Word Error Rate (WER). | +| [pybind11 cross build example][] | ![github icon][] ![gitlab icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Same as pybind11 cmake_example but used to demo Linux ARM + Windows + macOS builds on GitLab | [scikit-learn]: https://github.com/scikit-learn/scikit-learn +[pytorch-fairseq]: https://github.com/facebookresearch/fairseq +[duckdb]: https://github.com/duckdb/duckdb +[NumPy]: https://github.com/numpy/numpy [Tornado]: https://github.com/tornadoweb/tornado -[pytorch-fairseq]: https://github.com/pytorch/fairseq +[NCNN]: https://github.com/Tencent/ncnn [Matplotlib]: https://github.com/matplotlib/matplotlib [MyPy]: https://github.com/mypyc/mypy_mypyc-wheels -[pydantic]: https://github.com/samuelcolvin/pydantic +[Prophet]: https://github.com/facebook/prophet +[Kivy]: https://github.com/kivy/kivy +[Triton]: https://github.com/openai/triton +[MemRay]: https://github.com/bloomberg/memray [uvloop]: https://github.com/MagicStack/uvloop [psutil]: https://github.com/giampaolo/psutil -[vaex]: https://github.com/vaexio/vaex [Google Benchmark]: https://github.com/google/benchmark -[asyncpg]: https://github.com/MagicStack/asyncpg +[vaex]: https://github.com/vaexio/vaex [Apache Beam]: https://github.com/apache/beam +[PyGame]: https://github.com/pygame/pygame +[asyncpg]: https://github.com/MagicStack/asyncpg +[cmake]: https://github.com/scikit-build/cmake-python-distributions +[pyinstrument]: https://github.com/joerick/pyinstrument [scikit-image]: https://github.com/scikit-image/scikit-image +[PyOxidizer]: https://github.com/indygreg/PyOxidizer [twisted-iocpsupport]: https://github.com/twisted/twisted-iocpsupport -[cmake]: https://github.com/scikit-build/cmake-python-distributions -[duckdb]: https://github.com/duckdb/duckdb -[websockets]: https://github.com/aaugustin/websockets [cvxpy]: https://github.com/cvxpy/cvxpy -[Triton]: https://github.com/openai/triton -[PyOxidizer]: https://github.com/indygreg/PyOxidizer -[OpenSpiel]: https://github.com/deepmind/open_spiel +[pedalboard]: https://github.com/spotify/pedalboard +[websockets]: https://github.com/python-websockets/websockets [River]: https://github.com/online-ml/river -[pyzmq]: https://github.com/zeromq/pyzmq -[vispy]: https://github.com/vispy/vispy [aiortc]: https://github.com/aiortc/aiortc -[Confluent client for Kafka]: https://github.com/confluentinc/confluent-kafka-python +[OpenSpiel]: https://github.com/google-deepmind/open_spiel +[UltraJSON]: https://github.com/ultrajson/ultrajson +[Dependency Injector]: https://github.com/ets-labs/python-dependency-injector +[pyzmq]: https://github.com/zeromq/pyzmq +[CTranslate2]: https://github.com/OpenNMT/CTranslate2 +[Implicit]: https://github.com/benfred/implicit [tinyobjloader]: https://github.com/tinyobjloader/tinyobjloader +[vispy]: https://github.com/vispy/vispy [coverage.py]: https://github.com/nedbat/coveragepy [PyCryptodome]: https://github.com/Legrandin/pycryptodome -[Dependency Injector]: https://github.com/ets-labs/python-dependency-injector +[Line Profiler]: https://github.com/pyutils/line_profiler +[PyAV]: https://github.com/PyAV-Org/PyAV [PyYAML]: https://github.com/yaml/pyyaml +[pikepdf]: https://github.com/pikepdf/pikepdf [numexpr]: https://github.com/pydata/numexpr +[Wrapt]: https://github.com/GrahamDumpleton/wrapt [h5py]: https://github.com/h5py/h5py -[PyAV]: https://github.com/PyAV-Org/PyAV +[envd]: https://github.com/tensorchord/envd +[Psycopg 3]: https://github.com/psycopg/psycopg [OpenColorIO]: https://github.com/AcademySoftwareFoundation/OpenColorIO -[PyTables]: https://github.com/PyTables/PyTables -[Line Profiler]: https://github.com/pyutils/line_profiler -[OpenTimelineIO]: https://github.com/PixarAnimationStudios/OpenTimelineIO [ruptures]: https://github.com/deepcharles/ruptures -[pikepdf]: https://github.com/pikepdf/pikepdf [aioquic]: https://github.com/aiortc/aioquic -[DeepForest]: https://github.com/LAMDA-NJU/Deep-Forest +[SimpleJSON]: https://github.com/simplejson/simplejson +[OpenTimelineIO]: https://github.com/AcademySoftwareFoundation/OpenTimelineIO +[PyTables]: https://github.com/PyTables/PyTables [google neuroglancer]: https://github.com/google/neuroglancer -[AutoPy]: https://github.com/autopilot-rs/autopy [Parselmouth]: https://github.com/YannickJadoul/Parselmouth -[Psycopg 3]: https://github.com/psycopg/psycopg +[DeepForest]: https://github.com/LAMDA-NJU/Deep-Forest +[AutoPy]: https://github.com/autopilot-rs/autopy [H3-py]: https://github.com/uber/h3-py +[time-machine]: https://github.com/adamchainz/time-machine +[mosec]: https://github.com/mosecorg/mosec +[Picologging]: https://github.com/microsoft/picologging +[pybind11 cmake_example]: https://github.com/pybind/cmake_example [markupsafe]: https://github.com/pallets/markupsafe [Rtree]: https://github.com/Toblerity/rtree -[python-rapidjson]: https://github.com/python-rapidjson/python-rapidjson -[python-snappy]: https://github.com/andrix/python-snappy -[pybind11 cmake_example]: https://github.com/pybind/cmake_example [KDEpy]: https://github.com/tommyod/KDEpy +[dd-trace-py]: https://github.com/DataDog/dd-trace-py [tgcalls]: https://github.com/MarshalX/tgcalls [pybind11 python_example]: https://github.com/pybind/python_example -[dd-trace-py]: https://github.com/DataDog/dd-trace-py +[python-rapidjson]: https://github.com/python-rapidjson/python-rapidjson +[sourmash]: https://github.com/sourmash-bio/sourmash +[abess]: https://github.com/abess-team/abess +[python-snappy]: https://github.com/intake/python-snappy [cyvcf2]: https://github.com/brentp/cyvcf2 -[sourmash]: https://github.com/dib-lab/sourmash -[time-machine]: https://github.com/adamchainz/time-machine -[CTranslate2]: https://github.com/OpenNMT/CTranslate2 -[matrixprofile]: https://github.com/matrix-profile-foundation/matrixprofile [jq.py]: https://github.com/mwilliamson/jq.py -[iminuit]: https://github.com/scikit-hep/iminuit +[matrixprofile]: https://github.com/matrix-profile-foundation/matrixprofile [Tokenizer]: https://github.com/OpenNMT/Tokenizer +[iminuit]: https://github.com/scikit-hep/iminuit +[Confluent client for Kafka]: https://github.com/confluentinc/confluent-kafka-python +[pillow-heif]: https://github.com/bigcat88/pillow_heif +[keyvi]: https://github.com/KeyviDev/keyvi [PyGLM]: https://github.com/Zuzu-Typ/PyGLM -[bx-python]: https://github.com/bxlab/bx-python -[boost-histogram]: https://github.com/scikit-hep/boost-histogram -[iDynTree]: https://github.com/robotology/idyntree [TgCrypto]: https://github.com/pyrogram/tgcrypto +[iDynTree]: https://github.com/robotology/idyntree +[streaming-form-data]: https://github.com/siddhantgoel/streaming-form-data +[power-grid-model]: https://github.com/PowerGridModel/power-grid-model [pybase64]: https://github.com/mayeut/pybase64 -[etebase-py]: https://github.com/etesync/etebase-py -[fathon]: https://github.com/stfbnc/fathon +[bx-python]: https://github.com/bxlab/bx-python +[boost-histogram]: https://github.com/scikit-hep/boost-histogram [Imagecodecs (fork)]: https://github.com/czaki/imagecodecs_build -[numpythia]: https://github.com/scikit-hep/numpythia -[pyjet]: https://github.com/scikit-hep/pyjet -[polaroid]: https://github.com/daggy1234/polaroid +[Python-WebRTC]: https://github.com/MarshalX/python-webrtc +[fathon]: https://github.com/stfbnc/fathon +[pybind11 scikit_build_example]: https://github.com/pybind/scikit_build_example +[Arbor]: https://github.com/arbor-sim/arbor [clang-format]: https://github.com/ssciwr/clang-format-wheel +[polaroid]: https://github.com/daggy1234/polaroid [ninja]: https://github.com/scikit-build/ninja-python-distributions -[pybind11 scikit_build_example]: https://github.com/pybind/scikit_build_example +[etebase-py]: https://github.com/etesync/etebase-py +[cf-units]: https://github.com/SciTools/cf-units +[numpythia]: https://github.com/scikit-hep/numpythia +[pyjet]: https://github.com/scikit-hep/pyjet +[ril]: https://github.com/Cryptex-github/ril-py +[SiPM]: https://github.com/EdoPro98/SimSiPM +[aalink]: https://github.com/artfwo/aalink [GSD]: https://github.com/glotzerlab/gsd -[pyinstrument_cext]: https://github.com/joerick/pyinstrument_cext -[pillow-heif]: https://github.com/bigcat88/pillow_heif -[xmlstarlet]: https://github.com/dimitern/xmlstarlet [CorrectionLib]: https://github.com/cms-nanoAOD/correctionlib -[SiPM]: https://github.com/EdoPro98/SimSiPM +[xmlstarlet]: https://github.com/dimitern/xmlstarlet +[werpy]: https://github.com/analyticsinmotion/werpy +[pybind11 cross build example]: https://github.com/wbarnha/pybind_cmake_example_crossbuild -[appveyor icon]: data/readme_icons/appveyor.svg [github icon]: data/readme_icons/github.svg [azurepipelines icon]: data/readme_icons/azurepipelines.svg [circleci icon]: data/readme_icons/circleci.svg [gitlab icon]: data/readme_icons/gitlab.svg [travisci icon]: data/readme_icons/travisci.svg +[cirrusci icon]: data/readme_icons/cirrusci.svg [windows icon]: data/readme_icons/windows.svg [apple icon]: data/readme_icons/apple.svg [linux icon]: data/readme_icons/linux.svg - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - > Add your repo here! Let us know on [GitHub Discussions](https://github.com/pypa/cibuildwheel/discussions/485), or send a PR, adding your information to `docs/data/projects.yml`. diff --git a/examples/appveyor-minimal.yml b/examples/appveyor-minimal.yml deleted file mode 100644 index 33f438182..000000000 --- a/examples/appveyor-minimal.yml +++ /dev/null @@ -1,21 +0,0 @@ -environment: - matrix: - - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu - APPVEYOR_JOB_NAME: "python37-x64-ubuntu" - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - APPVEYOR_JOB_NAME: "python37-x64-vs2015" - - APPVEYOR_BUILD_WORKER_IMAGE: macos-mojave - APPVEYOR_JOB_NAME: "python37-x64-macos-mojave" - -stack: python 3.7 - -init: -- cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% - -install: python -m pip install cibuildwheel==2.3.1 - -build_script: python -m cibuildwheel --output-dir wheelhouse - -artifacts: - - path: "wheelhouse\\*.whl" - name: Wheels diff --git a/examples/azure-pipelines-minimal.yml b/examples/azure-pipelines-minimal.yml index dcfed02c9..e73864f8e 100644 --- a/examples/azure-pipelines-minimal.yml +++ b/examples/azure-pipelines-minimal.yml @@ -1,12 +1,12 @@ jobs: - job: linux - pool: {vmImage: 'Ubuntu-16.04'} + pool: {vmImage: 'ubuntu-latest'} steps: - task: UsePythonVersion@0 - bash: | set -o errexit python3 -m pip install --upgrade pip - pip3 install cibuildwheel==2.3.1 + pip3 install cibuildwheel==3.0.0 displayName: Install dependencies - bash: cibuildwheel --output-dir wheelhouse . displayName: Build wheels @@ -14,13 +14,13 @@ jobs: inputs: {pathtoPublish: 'wheelhouse'} - job: macos - pool: {vmImage: 'macOS-10.15'} + pool: {vmImage: 'macOS-latest'} steps: - task: UsePythonVersion@0 - bash: | set -o errexit python3 -m pip install --upgrade pip - python3 -m pip install cibuildwheel==2.3.1 + python3 -m pip install cibuildwheel==3.0.0 displayName: Install dependencies - bash: cibuildwheel --output-dir wheelhouse . displayName: Build wheels @@ -28,13 +28,13 @@ jobs: inputs: {pathtoPublish: wheelhouse} - job: windows - pool: {vmImage: 'vs2017-win2016'} + pool: {vmImage: 'windows-latest'} steps: - task: UsePythonVersion@0 - bash: | set -o errexit python -m pip install --upgrade pip - pip install cibuildwheel==2.3.1 + pip install cibuildwheel==3.0.0 displayName: Install dependencies - bash: cibuildwheel --output-dir wheelhouse . displayName: Build wheels diff --git a/examples/circleci-minimal.yml b/examples/circleci-minimal.yml index 7de81a34f..816a58135 100644 --- a/examples/circleci-minimal.yml +++ b/examples/circleci-minimal.yml @@ -4,28 +4,47 @@ jobs: linux-wheels: working_directory: ~/linux-wheels docker: - - image: circleci/python:3.6 + - image: cimg/python:3.12 steps: - checkout - setup_remote_docker - run: name: Build the Linux wheels. command: | - pip3 install --user cibuildwheel==2.3.1 + python3 -m pip install --user cibuildwheel==3.0.0 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ + linux-aarch64-wheels: + working_directory: ~/linux-aarch64-wheels + machine: + image: default + # resource_class is what tells CircleCI to use an ARM worker for native arm builds + # https://circleci.com/product/features/resource-classes/ + resource_class: arm.medium + steps: + - checkout + - run: + name: Build the Linux aarch64 wheels. + command: | + python3 -m pip install --user cibuildwheel==3.0.0 + python3 -m cibuildwheel --output-dir wheelhouse + - store_artifacts: + path: wheelhouse/ + osx-wheels: working_directory: ~/osx-wheels macos: - xcode: 12.5.1 + xcode: 15.4.0 + resource_class: macos.m1.medium.gen1 steps: - checkout - run: name: Build the OS X wheels. command: | - pip3 install cibuildwheel==2.3.1 + sudo softwareupdate --install-rosetta --agree-to-license # for python<=3.8 or x86_64/universal2 tests + pip3 install cibuildwheel==3.0.0 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ @@ -35,4 +54,5 @@ workflows: all-tests: jobs: - linux-wheels + - linux-aarch64-wheels - osx-wheels diff --git a/examples/cirrus-ci-intel-mac.yml b/examples/cirrus-ci-intel-mac.yml new file mode 100644 index 000000000..d59ecdad1 --- /dev/null +++ b/examples/cirrus-ci-intel-mac.yml @@ -0,0 +1,21 @@ +build_and_store_wheels: &BUILD_AND_STORE_WHEELS + install_cibuildwheel_script: + - python -m pip install cibuildwheel==3.0.0 + run_cibuildwheel_script: + - cibuildwheel + wheels_artifacts: + path: "wheelhouse/*" + + +macos_task: + name: Build macOS x86_64 and arm64 wheels. + macos_instance: + image: ghcr.io/cirruslabs/macos-runner:sonoma + env: + VENV_ROOT: ${HOME}/venv-cibuildwheel + PATH: ${VENV_ROOT}/bin:${PATH} + CIBW_ARCHS_MACOS: x86_64 arm64 + install_pre_requirements_script: + - brew install python@3.12 + - python3.12 -m venv ${VENV_ROOT} + <<: *BUILD_AND_STORE_WHEELS diff --git a/examples/cirrus-ci-minimal.yml b/examples/cirrus-ci-minimal.yml new file mode 100644 index 000000000..0f552a15a --- /dev/null +++ b/examples/cirrus-ci-minimal.yml @@ -0,0 +1,74 @@ +build_and_store_wheels: &BUILD_AND_STORE_WHEELS + install_cibuildwheel_script: + - python -m pip install cibuildwheel==3.0.0 + run_cibuildwheel_script: + - cibuildwheel + wheels_artifacts: + path: "wheelhouse/*" + + +linux_x86_task: + name: Build Linux x86 wheels. + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 4 + memory: 4G + env: + VENV_ROOT: ${HOME}/venv-cibuildwheel + PATH: ${VENV_ROOT}/bin:${PATH} + install_pre_requirements_script: + - add-apt-repository -y ppa:deadsnakes/ppa + - apt-get update + - apt-get install -y python3.12-venv + - python3.12 -m venv ${VENV_ROOT} + <<: *BUILD_AND_STORE_WHEELS + +linux_aarch64_task: + name: Build Linux aarch64 wheels. + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 4 + memory: 4G + env: + VENV_ROOT: ${HOME}/venv-cibuildwheel + PATH: ${VENV_ROOT}/bin:${PATH} + install_pre_requirements_script: + - add-apt-repository -y ppa:deadsnakes/ppa + - apt-get update + - apt-get install -y python3.12-venv + - python3.12 -m venv ${VENV_ROOT} + <<: *BUILD_AND_STORE_WHEELS + +windows_x86_task: + name: Build Windows x86 wheels. + windows_container: + image: cirrusci/windowsservercore:visualstudio2022 + cpu: 4 + memory: 4G + + install_pre_requirements_script: + - certutil -generateSSTFromWU roots.sst + - certutil -addstore -f root roots.sst + - del roots.sst + - choco install -y --no-progress python3 --version 3.12.4 + - refreshenv + - powershell -Command "$cleanPath = $env:PATH -replace ';+$', ''; Add-Content -Path $env:CIRRUS_ENV -Value ('PATH=' + $cleanPath)" + + <<: *BUILD_AND_STORE_WHEELS + +macos_arm64_task: + name: Build macOS arm64 wheels. + macos_instance: + image: ghcr.io/cirruslabs/macos-runner:sequoia + env: + VENV_ROOT: ${HOME}/venv-cibuildwheel + PATH: ${VENV_ROOT}/bin:${PATH} + install_pre_requirements_script: + - brew install python@3.12 + - python3.12 -m venv ${VENV_ROOT} + <<: *BUILD_AND_STORE_WHEELS diff --git a/examples/github-apple-silicon.yml b/examples/github-apple-silicon.yml deleted file mode 100644 index 9bb927b30..000000000 --- a/examples/github-apple-silicon.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Build - -on: [push, pull_request] - -jobs: - build_wheels_macos: - name: Build wheels on macos-10.15 - runs-on: macos-10.15 - steps: - - uses: actions/checkout@v2 - - - name: Build wheels - uses: pypa/cibuildwheel@v2.3.1 - env: - CIBW_ARCHS_MACOS: x86_64 universal2 - - - uses: actions/upload-artifact@v2 - with: - path: ./wheelhouse/*.whl diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 23180e140..596ac716c 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -1,61 +1,92 @@ name: Build and upload to PyPI -# Build on every branch push, tag push, and pull request change: -on: [push, pull_request] -# Alternatively, to publish when a (published) GitHub Release is created, use the following: -# on: -# push: -# pull_request: -# release: -# types: -# - published +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + release: + types: + - published jobs: build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: Build wheels for ${{ matrix.os }} + runs-on: ${{ matrix.runs-on }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-10.15] + include: + - os: linux-intel + runs-on: ubuntu-latest + - os: linux-arm + runs-on: ubuntu-24.04-arm + - os: windows-intel + runs-on: windows-latest + - os: windows-arm + runs-on: windows-11-arm + - os: macos-intel + # macos-13 was the last x86_64 runner + runs-on: macos-13 + - os: macos-arm + # macos-14+ (including latest) are ARM64 runners + runs-on: macos-latest + - os: ios + runs-on: macos-latest + platform: ios + - os: pyodide + runs-on: ubuntu-latest + platform: pyodide steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build wheels - uses: pypa/cibuildwheel@v2.3.1 + uses: pypa/cibuildwheel@v3.0.0 + env: + CIBW_PLATFORM: ${{ matrix.platform || 'auto' }} + # Can also be configured directly, using `with:` + # with: + # package-dir: . + # output-dir: wheelhouse + # config-file: "{package}/pyproject.toml" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl build_sdist: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-sdist path: dist/*.tar.gz upload_pypi: needs: [build_wheels, build_sdist] runs-on: ubuntu-latest - # upload to PyPI on every tag starting with 'v' - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') - # alternatively, to publish when a GitHub Release is created, use the following rule: - # if: github.event_name == 'release' && github.event.action == 'published' + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: - name: artifact + # unpacks all CIBW artifacts into dist/ + pattern: cibw-* path: dist + merge-multiple: true - - uses: pypa/gh-action-pypi-publish@v1.4.2 + - uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: ${{ secrets.pypi_password }} - # To test: repository_url: https://test.pypi.org/legacy/ + # To test: repository-url: https://test.pypi.org/legacy/ diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index fdcb775b0..4bc1fd5b8 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -8,17 +8,23 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-10.15] + # macos-13 is an intel runner, macos-14 is apple silicon + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-14] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build wheels - uses: pypa/cibuildwheel@v2.3.1 - # to supply options, put them in 'env', like: + uses: pypa/cibuildwheel@v3.0.0 # env: # CIBW_SOME_OPTION: value + # ... + # with: + # package-dir: . + # output-dir: wheelhouse + # config-file: "{package}/pyproject.toml" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl diff --git a/examples/github-pipx.yml b/examples/github-pipx.yml new file mode 100644 index 000000000..258b562e2 --- /dev/null +++ b/examples/github-pipx.yml @@ -0,0 +1,23 @@ +name: Build + +on: [push, pull_request] + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + # macos-13 is an intel runner, macos-14 is apple silicon + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-14] + + steps: + - uses: actions/checkout@v4 + + - name: Build wheels + run: pipx run cibuildwheel==3.0.0 + + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl diff --git a/examples/github-with-qemu.yml b/examples/github-with-qemu.yml index ad5c493d5..7b376bc15 100644 --- a/examples/github-with-qemu.yml +++ b/examples/github-with-qemu.yml @@ -8,29 +8,27 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-10.15] + # macos-13 is an intel runner, macos-14 is apple silicon + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-14] steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' + - uses: actions/checkout@v4 - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v1 + if: runner.os == 'Linux' && runner.arch == 'X64' + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.3.1 + uses: pypa/cibuildwheel@v3.0.0 env: - # configure cibuildwheel to build native archs ('auto'), and some - # emulated ones - CIBW_ARCHS_LINUX: auto aarch64 ppc64le s390x + # configure cibuildwheel on Linux to build native archs ('auto'), + # and to split the remaining architectures between the x86_64 and + # ARM runners + CIBW_ARCHS_LINUX: ${{ runner.arch == 'X64' && 'auto ppc64le s390x' || 'auto' }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl diff --git a/examples/gitlab-minimal.yml b/examples/gitlab-minimal.yml index 447a4ad2f..1a8e054a3 100644 --- a/examples/gitlab-minimal.yml +++ b/examples/gitlab-minimal.yml @@ -1,5 +1,5 @@ linux: - image: python:3.8 + image: python:3.12 # make a docker daemon available for cibuildwheel to use services: - name: docker:dind @@ -12,8 +12,34 @@ linux: DOCKER_TLS_CERTDIR: "" script: - curl -sSL https://get.docker.com/ | sh - - python -m pip install cibuildwheel==2.3.1 + - python -m pip install cibuildwheel==3.0.0 - cibuildwheel --output-dir wheelhouse artifacts: paths: - wheelhouse/ + +windows: + image: mcr.microsoft.com/windows/servercore:1809 + before_script: + - choco install python -y --allow-downgrade --version 3.12.4 + - choco install git.install -y + - py -m pip install cibuildwheel==3.0.0 + script: + - py -m cibuildwheel --output-dir wheelhouse --platform windows + artifacts: + paths: + - wheelhouse/ + tags: + - saas-windows-medium-amd64 + +macos: + image: macos-14-xcode-15 + before_script: + - python3 -m pip install cibuildwheel==3.0.0 + script: + - python3 -m cibuildwheel --output-dir wheelhouse + artifacts: + paths: + - wheelhouse/ + tags: + - saas-macos-medium-m1 diff --git a/examples/gitlab-with-qemu.yml b/examples/gitlab-with-qemu.yml new file mode 100644 index 000000000..db86bb583 --- /dev/null +++ b/examples/gitlab-with-qemu.yml @@ -0,0 +1,22 @@ +linux: + image: python:3.12 + # make a docker daemon available for cibuildwheel to use + services: + - name: docker:dind + entrypoint: ["env", "-u", "DOCKER_HOST"] + command: ["dockerd-entrypoint.sh"] + variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_DRIVER: overlay2 + # See https://github.com/docker-library/docker/pull/166 + DOCKER_TLS_CERTDIR: "" + script: + - curl -sSL https://get.docker.com/ | sh + # Warning: This is extremely slow, be careful with how many wheels you build + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - python -m pip install cibuildwheel==3.0.0 + # Assuming your CI runner's default architecture is x86_64... + - cibuildwheel --output-dir wheelhouse --platform linux --archs aarch64 + artifacts: + paths: + - wheelhouse/ diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index 104dbafeb..fbf762793 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -2,8 +2,9 @@ # commit, but will only push to PyPI on tagged commits. os: linux -dist: focal +dist: jammy language: python +python: "3.12" jobs: include: @@ -13,13 +14,13 @@ jobs: - os: windows language: shell before_install: - - choco upgrade python -y --version 3.8.6 - - export PATH="/c/Python38:/c/Python38/Scripts:$PATH" + - choco upgrade python -y --version 3.12.4 + - export PATH="/c/Python312:/c/Python312/Scripts:$PATH" # make sure it's on PATH as 'python3' - - ln -s /c/Python38/python.exe /c/Python38/python3.exe + - ln -s /c/Python312/python.exe /c/Python312/python3.exe install: - - python3 -m pip install cibuildwheel==2.3.1 + - python3 -m pip install cibuildwheel==3.0.0 script: # build the wheels, put them into './dist' diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index 42b921562..796253f35 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -1,6 +1,7 @@ os: linux -dist: focal +dist: jammy language: python +python: "3.12" jobs: include: @@ -19,13 +20,13 @@ jobs: - os: windows language: shell before_install: - - choco upgrade python -y --version 3.8.6 - - export PATH="/c/Python38:/c/Python38/Scripts:$PATH" + - choco upgrade python -y --version 3.12.4 + - export PATH="/c/Python312:/c/Python312/Scripts:$PATH" # make sure it's on PATH as 'python3' - - ln -s /c/Python38/python.exe /c/Python38/python3.exe + - ln -s /c/Python312/python.exe /c/Python312/python3.exe install: - - python3 -m pip install cibuildwheel==2.3.1 + - python3 -m pip install cibuildwheel==3.0.0 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index 0ef18760e..fc0dffa59 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -6,20 +6,17 @@ # distribution is also created. os: linux -dist: focal +dist: jammy language: python -python: - - 3.6 - - 3.7 - - 3.8 +python: "3.12" before_install: - | if [[ "$TRAVIS_OS_NAME" = windows ]]; then - choco upgrade python -y --version 3.8.6 - export PATH="/c/Python38:/c/Python38/Scripts:$PATH" + choco upgrade python -y --version 3.12.4 + export PATH="/c/Python312:/c/Python312/Scripts:$PATH" # make sure it's on PATH as 'python3' - ln -s /c/Python38/python.exe /c/Python38/python3.exe + ln -s /c/Python312/python.exe /c/Python312/python3.exe fi install: @@ -55,7 +52,7 @@ jobs: - stage: deploy name: Build and deploy Linux wheels services: docker - install: python3 -m pip install cibuildwheel==2.3.1 twine + install: python3 -m pip install cibuildwheel==3.0.0 twine script: python3 -m cibuildwheel --output-dir wheelhouse after_success: python3 -m twine upload --skip-existing wheelhouse/*.whl # Deploy on windows @@ -63,7 +60,7 @@ jobs: name: Build and deploy Windows wheels os: windows language: shell - install: python3 -m pip install cibuildwheel==2.3.1 twine + install: python3 -m pip install cibuildwheel==3.0.0 twine script: python3 -m cibuildwheel --output-dir wheelhouse after_success: python3 -m twine upload --skip-existing wheelhouse/*.whl diff --git a/mkdocs.yml b/mkdocs.yml index 688556748..2f38025b5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,6 +5,9 @@ theme: highlightjs: true hljs_languages: - yaml + collapse_navigation: false + navigation_depth: 3 + custom_dir: docs/theme_overrides repo_url: https://github.com/pypa/cibuildwheel edit_uri: edit/main/docs/ @@ -14,16 +17,23 @@ extra_css: extra_javascript: - extra.js + nav: - Home: index.md - - setup.md - - options.md - - deliver-to-pypi.md - - cpp_standards.md - - faq.md - - working-examples.md - - contributing.md - - changelog.md + - Guide: + - setup.md + - ci-services.md + - deliver-to-pypi.md + - faq.md + - cpp_standards.md + - Reference: + - options.md + - configuration.md + - platforms.md + - working-examples.md + - About: + - contributing.md + - changelog.md markdown_extensions: - md_in_html @@ -45,3 +55,5 @@ plugins: module_name: docs/main j2_variable_start_string: « j2_variable_end_string: » + on_error_fail: true + on_undefined: strict diff --git a/noxfile.py b/noxfile.py old mode 100644 new mode 100755 index bee10da53..d475c60e3 --- a/noxfile.py +++ b/noxfile.py @@ -1,100 +1,189 @@ +#!/usr/bin/env -S uv run + +# /// script +# dependencies = ["nox>=2025.2.9"] +# /// + +""" +cibuildwheel's nox support + +Tags: + + lint: All linting jobs + update: All update jobs + +See sessions with `nox -l` +""" + +import os import shutil import sys from pathlib import Path import nox -nox.options.sessions = ["lint", "check_manifest", "tests"] - -PYTHON_ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +nox.needs_version = ">=2025.2.9" +nox.options.default_venv_backend = "uv|virtualenv" DIR = Path(__file__).parent.resolve() +@nox.session(tags=["lint"]) +def lint(session: nox.Session) -> None: + """ + Run the linter. + """ + session.install("pre-commit") + session.run("pre-commit", "run", "--all-files", *session.posargs) + + +@nox.session(tags=["lint"]) +def pylint(session: nox.Session) -> None: + """ + Run pylint. + """ + + session.install("pylint>=3.2", "-e.") + session.run("pylint", "cibuildwheel", *session.posargs) + + @nox.session def tests(session: nox.Session) -> None: """ Run the unit and regular tests. """ - unit_test_args = ["--run-docker"] if sys.platform.startswith("linux") else [] - session.install("-e", ".[test]") + pyproject = nox.project.load_toml() + session.install("-e.", *nox.project.dependency_groups(pyproject, "test")) if session.posargs: session.run("pytest", *session.posargs) else: + unit_test_args = ["--run-docker"] if sys.platform.startswith("linux") else [] session.run("pytest", "unit_test", *unit_test_args) session.run("pytest", "test", "-x", "--durations", "0", "--timeout=2400", "test") -@nox.session -def lint(session: nox.Session) -> None: +@nox.session(default=False, tags=["update"]) +def update_constraints(session: nox.Session) -> None: """ - Run the linter. + Update the dependencies inplace. """ - session.install("pre-commit") - session.run("pre-commit", "run", "--all-files", *session.posargs) + session.install("-e.", "click") -@nox.session -def check_manifest(session: nox.Session) -> None: - """ - Ensure all needed files are included in the manifest. - """ + resources = Path("cibuildwheel/resources") - session.install("check-manifest") - session.run("check-manifest", *session.posargs) + if session.venv_backend != "uv": + session.install("uv>=0.1.23") + # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to + # regenerate the constraints files + env = os.environ.copy() + env["UV_CUSTOM_COMPILE_COMMAND"] = f"nox -s {session.name}" -@nox.session(python=PYTHON_ALL_VERSIONS) -def update_constraints(session: nox.Session) -> None: - """ - Update the dependencies inplace. - """ - session.install("requests", "pip-tools") - session.run("python", "bin/update_dependencies.py") + for minor_version in range(8, 15): + python_version = f"3.{minor_version}" + output_file = resources / f"constraints-python{python_version.replace('.', '')}.txt" + session.run( + "uv", + "pip", + "compile", + f"--python-version={python_version}", + "--upgrade", + resources / "constraints.in", + f"--output-file={output_file}", + env=env, + ) + shutil.copyfile( + resources / "constraints-python314.txt", + resources / "constraints.txt", + ) -@nox.session + build_platforms = nox.project.load_toml(resources / "build-platforms.toml") + pyodides = build_platforms["pyodide"]["python_configurations"] + for pyodide in pyodides: + python_version = ".".join(pyodide["version"].split(".")[:2]) + pyodide_version = pyodide["default_pyodide_version"] + + tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in" + + session.run( + "python", + "bin/generate_pyodide_constraints.py", + "--output-file", + tmp_file, + pyodide_version, + ) + + output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt" + session.run( + "uv", + "pip", + "compile", + f"--python-version={python_version}", + "--upgrade", + tmp_file, + f"--output-file={output_file}", + env=env, + ) + + +@nox.session(default=False, tags=["update"]) def update_pins(session: nox.Session) -> None: """ - Update the python, docker and virtualenv pins version inplace. + Update the python, docker, virtualenv, node, and python-build-standalone + version pins inplace. """ - session.install("-e", ".[bin]") + pyproject = nox.project.load_toml() + session.install("-e.", *nox.project.dependency_groups(pyproject, "bin")) session.run("python", "bin/update_pythons.py", "--force") session.run("python", "bin/update_docker.py") session.run("python", "bin/update_virtualenv.py", "--force") + session.run("python", "bin/update_nodejs.py", "--force") + session.run("python", "bin/update_python_build_standalone.py") -@nox.session +@nox.session(default=False, reuse_venv=True, tags=["update"]) def update_proj(session: nox.Session) -> None: """ Update the README inplace. """ - session.install("-e", ".[bin]") - session.run( - "python", + session.install_and_run_script( "bin/projects.py", "docs/data/projects.yml", *session.posargs, ) -@nox.session -def docs(session: nox.Session) -> None: +@nox.session(default=False, reuse_venv=True, tags=["update"]) +def generate_schema(session: nox.Session) -> None: """ - Build the docs. + Generate the cibuildwheel.schema.json file. """ - session.install("-e", ".[docs]") + output = session.install_and_run_script("bin/generate_schema.py", silent=True) + assert isinstance(output, str) + DIR.joinpath("cibuildwheel/resources/cibuildwheel.schema.json").write_text(output) - if session.posargs: - if "serve" in session.posargs: - session.run("mkdocs", "serve") - else: - session.error("Unrecognized args, use 'serve'") - else: - session.run("mkdocs", "build") +@nox.session(default=False, reuse_venv=True) +def bump_version(session: nox.Session) -> None: + """ + Bump cibuildwheel's version. Interactive. + """ + session.install_and_run_script("bin/bump_version.py") -@nox.session + +@nox.session(default=False) +def docs(session: nox.Session) -> None: + """ + Build the docs. Will serve unless --non-interactive + """ + pyproject = nox.project.load_toml() + session.install("-e.", *nox.project.dependency_groups(pyproject, "docs")) + session.run("mkdocs", "serve" if session.interactive else "build", "--strict", *session.posargs) + + +@nox.session(default=False) def build(session: nox.Session) -> None: """ Build an SDist and wheel. @@ -110,3 +199,7 @@ def build(session: nox.Session) -> None: session.install("build") session.run("python", "-m", "build") + + +if __name__ == "__main__": + nox.main() diff --git a/pyproject.toml b/pyproject.toml index ab4ed941f..0b1ef884d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,33 +1,114 @@ [build-system] -requires = [ - "setuptools>=42", - "wheel" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "cibuildwheel" +version = "3.0.0" +description = "Build Python wheels on CI with minimal configuration." +readme = "README.md" +license = "BSD-2-Clause" +license-files = ["LICENSE"] +requires-python = ">=3.11" +authors = [ + { name = "Joe Rickerby", email = "joerick@mac.com" }, +] +keywords = [ + "ci", + "linux", + "macos", + "packaging", + "pypi", + "travis", + "wheel", + "windows", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Build Tools", +] +dependencies = [ + "bashlex!=0.13", + "bracex", + "certifi", + "dependency-groups>=1.2", + "filelock", + "packaging>=20.9", + "platformdirs" ] -build-backend = "setuptools.build_meta" +[project.optional-dependencies] +uv = ["uv"] +[project.scripts] +cibuildwheel = "cibuildwheel.__main__:main" -[tool.black] -line-length = 100 -target-version = ['py36', 'py37', 'py38', 'py39'] +[project.entry-points."validate_pyproject.tool_schema"] +cibuildwheel = "cibuildwheel.schema:get_schema" +[project.urls] +Changelog = "https://github.com/pypa/cibuildwheel#changelog" +Documentation = "https://cibuildwheel.pypa.io/" +Homepage = "https://github.com/pypa/cibuildwheel" -[tool.isort] -profile = "black" -multi_line_output = 3 +[dependency-groups] +bin = [ + "click", + "packaging>=21.0", + "pip-tools", + "pygithub", + "pyyaml", + "requests", + "rich>=9.6", +] +docs = [ + "jinja2>=3.1.2", + "mkdocs-include-markdown-plugin==6.2.2", + "mkdocs-macros-plugin", + "mkdocs==1.6.1", + "pymdown-extensions", + "rich", +] +test = [ + "build", + "filelock", + "jinja2", + "pytest-timeout", + "pytest-xdist", + "pytest>=6", + "setuptools", + "tomli_w", + "validate-pyproject", +] +dev = [ + {include-group = "bin"}, + {include-group = "docs"}, + {include-group = "test"}, +] [tool.pytest.ini_options] -minversion = 6.0 +minversion = "6.0" +addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] junit_family = "xunit2" -testpaths = [ - "test", - "unit_test", +xfail_strict = true +filterwarnings = ["error"] +log_cli_level = "INFO" +markers = [ + "serial: tests that must *not* be run in parallel (deselect with '-m \"not serial\"')", ] - [tool.mypy] -python_version = 3.7 +python_version = "3.11" files = [ "cibuildwheel/*.py", "test/**/*.py", @@ -36,48 +117,129 @@ files = [ "noxfile.py", ] warn_unused_configs = true -warn_redundant_casts = true -no_implicit_reexport = true -strict_equality = true -warn_unused_ignores = true -check_untyped_defs = true -disallow_subclassing_any = true -disallow_any_generics = true -warn_return_any = true -no_implicit_optional = true +strict = true +disallow_untyped_defs = false + +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] +warn_unreachable = false [[tool.mypy.overrides]] module = "cibuildwheel.*" disallow_untyped_defs = true -disallow_untyped_calls = true -disallow_incomplete_defs = true -disallow_untyped_decorators = true [[tool.mypy.overrides]] module = [ - "setuptools", - "pytest", # ignored in pre-commit to speed up check + "setuptools._distutils", # needed even if only directly import setuptools._distutils.util + "setuptools._distutils.util", "bashlex", + "bashlex.*", "importlib_resources", - "ghapi.*", ] ignore_missing_imports = true -[tool.pycln] -all = true -[tool.check-manifest] +[tool.pylint] +py-version = "3.11" +jobs = "0" +fail-on = ["E", "F"] +fail-under = "9.8" +reports.output-format = "colorized" +messages_control.enable = [ + "useless-suppression", +] +messages_control.disable = [ + "abstract-class-instantiated", # filelock triggers this + "duplicate-code", + "fixme", + "invalid-name", + "line-too-long", + "missing-class-docstring", + "missing-function-docstring", + "missing-module-docstring", + "no-else-break", + "no-else-return", + "protected-access", + "too-few-public-methods", + "too-many-arguments", + "too-many-branches", + "too-many-instance-attributes", + "too-many-lines", + "too-many-locals", + "too-many-nested-blocks", + "too-many-return-statements", + "too-many-statements", + "unsubscriptable-object", + "wrong-import-position", + "unused-argument", # Handled by Ruff + "broad-exception-raised", # Could be improved eventually + "consider-using-in", # MyPy can't narrow "in" +] + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +extend-select = [ + "B", # flake8-bugbear + "I", # isort + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "G", # flake8-logging-format + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "RUF", # Ruff-specific + "SIM", # flake8-simplify + "TID251", # flake8-tidy-imports.banned-api + "UP", # pyupgrade + "YTT", # flake8-2020 + "EXE", # flake8-executable + "PYI", # flake8-pyi + "PERF101", "PERF102", "PERF401", "PERF402", "PERF403", # A selection of perflint codes +] ignore = [ - ".*", - ".circleci/**", - "test/**", - "unit_test/**", - "docs/**", - "examples/**", - "bin/**", - "*.yml", - "CI.md", # TODO: can change test/test_ssl and remove this - "requirements-dev.txt", - "noxfile.py", + "PLR", # Design related pylint codes + "RET504", "RET505", "RET508", # else after control flow + "PT007", # Lists of tuples in Pytest + "PYI025", # Set as AbstractSet + "ISC001", # Conflicts with formatter + "EXE003", # Ruff doesn't like uv? + "PTH123", # open -> Path.open +] +flake8-unused-arguments.ignore-variadic-names = true + +[tool.ruff.lint.flake8-tidy-imports.banned-api] +"typing.Mapping".msg = "Use collections.abc.Mapping instead." +"typing.Callable".msg = "Use collections.abc.Callable instead." +"typing.Iterator".msg = "Use collections.abc.Iterator instead." +"typing.Sequence".msg = "Use collections.abc.Sequence instead." +"typing.Set".msg = "Use collections.abc.Set instead." + +[tool.ruff.lint.per-file-ignores] +"unit_test/*" = ["PLC1901"] +"bin/*" = ["TID251"] +"cibuildwheel/resources/install_certifi.py" = ["PTH"] + +[tool.repo-review] +ignore = ["PC170", "PP303"] + +[tool.check-wheel-contents] +ignore = ["W002"] # constraints-*.txt are allowed to be duplicates of one another + +[tool.codespell] +ignore-words-list = [ + "sur", + "assertin", +] +skip = [ + '^docs/working-examples\.md', + 'htmlcov', + 'all_known_setup.yaml', ] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 18dc383fb..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1 +0,0 @@ --e .[dev,docs] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ad21db4ca..000000000 --- a/setup.cfg +++ /dev/null @@ -1,71 +0,0 @@ -[metadata] -name = cibuildwheel -version = 2.3.1 -description = Build Python wheels on CI with minimal configuration. -long_description = file: README.md -long_description_content_type = text/markdown -url = https://github.com/pypa/cibuildwheel -author = Joe Rickerby -author_email = joerick@mac.com -license = BSD-2-Clause -license_file = LICENSE -classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Developers - License :: OSI Approved :: BSD License - Natural Language :: English - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: Implementation :: CPython - Topic :: Software Development :: Build Tools -keywords = ci,wheel,packaging,pypi,travis,appveyor,macos,linux,windows -project_urls = - Changelog=https://github.com/pypa/cibuildwheel#changelog - Documentation=https://cibuildwheel.readthedocs.io/ - -[options] -packages = find: -install_requires = - bashlex!=0.13 - bracex - certifi - filelock - packaging - platformdirs - tomli - dataclasses;python_version < '3.7' - typing-extensions>=3.10.0.0;python_version < '3.8' -python_requires = >=3.6 -include_package_data = True -zip_safe = False - -[options.packages.find] -include = - cibuildwheel - -[options.entry_points] -console_scripts = - cibuildwheel = cibuildwheel.__main__:main - -[options.package_data] -cibuildwheel = resources/* - -[flake8] -ignore = E501,W503,E741,E226,B950,E203 -select = C,E,F,W,B,B9 -application-import-names = cibuildwheel -exclude = - cibuildwheel/resources/, - dist/, - build/, - .git/, - env/, - env2/, - env??/, - .venv/, - site/ diff --git a/setup.py b/setup.py deleted file mode 100644 index 70fd3b0e6..000000000 --- a/setup.py +++ /dev/null @@ -1,47 +0,0 @@ -from setuptools import setup - -extras = { - "docs": [ - "mkdocs-include-markdown-plugin==2.8.0", - "mkdocs==1.0.4", - "pymdown-extensions", - "mkdocs-macros-plugin", - ], - "test": [ - "jinja2", - "pytest>=6", - "pytest-timeout", - "pytest-xdist", - ], - "bin": [ - "click", - "ghapi", - "pip-tools", - "pygithub", - "pyyaml", - "requests", - "rich>=9.6", - "packaging>=21.0", - ], - "mypy": [ - "mypy>=0.901", - "types-jinja2", - "types-certifi", - "types-toml", - "types-jinja2", - "types-pyyaml", - "types-click", - "types-requests", - "types-toml", - ], -} - -extras["dev"] = [ - *extras["mypy"], - *extras["test"], - *extras["bin"], -] - -extras["all"] = sum(extras.values(), []) - -setup(extras_require=extras) diff --git a/test/conftest.py b/test/conftest.py index d8ff2f626..88d6175b6 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,30 +1,191 @@ -from typing import Dict +import json +import os +import subprocess +from collections.abc import Generator import pytest +from filelock import FileLock +from cibuildwheel.architecture import Architecture +from cibuildwheel.ci import detect_ci_provider +from cibuildwheel.options import CommandLineArguments, Options +from cibuildwheel.selector import EnableGroup +from cibuildwheel.typing import PLATFORMS +from cibuildwheel.venv import find_uv -def pytest_addoption(parser) -> None: +from .utils import DEFAULT_CIBW_ENABLE, EMULATED_ARCHS, get_platform + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--run-emulation", + action="http://github.com/store", + default=None, + help="run emulation tests", + choices=("all", *EMULATED_ARCHS), + ) + parser.addoption("--run-podman", action="http://github.com/store_true", default=False, help="run podman tests") + parser.addoption( + "--run-cp38-universal2", + action="http://github.com/store_true", + default=False, + help="macOS cp38 uses the universal2 installer", + ) + parser.addoption( + "--enable", + action="http://github.com/store", + default=None, + help="Set the CIBW_ENABLE environment variable for all tests.", + ) parser.addoption( - "--run-emulation", action="http://github.com/store_true", default=False, help="run emulation tests" + "--platform", + action="http://github.com/store", + default=None, + help="Set the CIBW_PLATFORM environment variable for all tests.", ) def pytest_configure(config): - config.addinivalue_line("markers", "emulation: mark test requiring qemu binfmt_misc to run") + flag_enable = config.getoption("--enable") + flag_platform = config.getoption("--platform") + + if flag_enable is not None and "CIBW_ENABLE" in os.environ: + msg = ( + "Both --enable pytest option and CIBW_ENABLE environment variable are set. " + "Please specify only one." + ) + raise pytest.UsageError(msg) + if flag_platform is not None and "CIBW_PLATFORM" in os.environ: + msg = ( + "Both --platform pytest option and CIBW_PLATFORM environment variable are set. " + "Please specify only one." + ) + raise pytest.UsageError(msg) + + if flag_enable is not None: + EnableGroup.parse_option_value(flag_enable) + os.environ["CIBW_ENABLE"] = flag_enable + if flag_enable is None and "CIBW_ENABLE" not in os.environ: + # Set default value for CIBW_ENABLE + os.environ["CIBW_ENABLE"] = DEFAULT_CIBW_ENABLE + + if flag_platform is not None: + assert flag_platform in PLATFORMS, f"Invalid platform: {flag_platform}" + os.environ["CIBW_PLATFORM"] = flag_platform -def pytest_collection_modifyitems(config, items) -> None: - if config.getoption("--run-emulation"): - # --run-emulation given in cli: do not skip emulation tests +def docker_warmup(request: pytest.FixtureRequest) -> None: + machine = request.config.getoption("--run-emulation", default=None) + if machine is None: + archs = {arch.value for arch in Architecture.auto_archs("linux")} + elif machine == "all": + archs = set(EMULATED_ARCHS) + else: + archs = {machine} + + # Only include architectures where there are missing pre-installed interpreters + archs &= {"x86_64", "i686", "aarch64"} + if not archs: return - skip_emulation = pytest.mark.skip(reason="need --run-emulation option to run") - for item in items: - if "emulation" in item.keywords: - item.add_marker(skip_emulation) + options = Options( + platform="linux", + command_line_arguments=CommandLineArguments.defaults(), + env={}, + defaults=True, + ) + build_options = options.build_options(None) + assert build_options.manylinux_images is not None + assert build_options.musllinux_images is not None + images = [build_options.manylinux_images[arch] for arch in archs] + [ + build_options.musllinux_images[arch] for arch in archs + ] + command = ( + "manylinux-interpreters ensure-all &&" + "cpython3.13 -m pip download -d /tmp setuptools wheel pytest" + ) + for image in images: + container_id = subprocess.run( + ["docker", "create", image, "bash", "-c", command], + text=True, + check=True, + stdout=subprocess.PIPE, + ).stdout.strip() + try: + subprocess.run(["docker", "start", container_id], check=True, stdout=subprocess.DEVNULL) + exit_code = subprocess.run( + ["docker", "wait", container_id], text=True, check=True, stdout=subprocess.PIPE + ).stdout.strip() + assert exit_code == "0" + subprocess.run( + ["docker", "commit", container_id, image], check=True, stdout=subprocess.DEVNULL + ) + finally: + subprocess.run(["docker", "rm", container_id], check=True, stdout=subprocess.DEVNULL) + + +@pytest.fixture(scope="session", autouse=True) +def docker_warmup_fixture( + request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str +) -> None: + # if we're in CI testing linux, let's warm-up docker images + if detect_ci_provider() is None or get_platform() != "linux": + return None + if request.config.getoption("--run-emulation", default=None) is not None: + # emulation tests only run one test in CI, caching the image only slows down the test + return None + + if worker_id == "master": + # not executing with multiple workers + # it might be unsafe to write to tmp_path_factory.getbasetemp().parent + return docker_warmup(request) + + # get the temp directory shared by all workers + root_tmp_dir = tmp_path_factory.getbasetemp().parent + + fn = root_tmp_dir / "warmup.done" + with FileLock(str(fn) + ".lock"): + if not fn.is_file(): + docker_warmup(request) + fn.write_text("done") + return None + + +@pytest.fixture(params=["pip", "build"]) +def build_frontend_env_nouv(request: pytest.FixtureRequest) -> dict[str, str]: + frontend = request.param + if get_platform() == "pyodide" and frontend == "pip": + pytest.skip("Can't use pip as build frontend for pyodide platform") + + return {"CIBW_BUILD_FRONTEND": frontend} + + +@pytest.fixture +def build_frontend_env(build_frontend_env_nouv: dict[str, str]) -> dict[str, str]: + frontend = build_frontend_env_nouv["CIBW_BUILD_FRONTEND"] + if frontend != "build" or get_platform() == "pyodide" or find_uv() is None: + return build_frontend_env_nouv + + return {"CIBW_BUILD_FRONTEND": "build[uv]"} + + +@pytest.fixture +def docker_cleanup() -> Generator[None, None, None]: + def get_images() -> set[str]: + if detect_ci_provider() is None or get_platform() != "linux": + return set() + images = subprocess.run( + ["docker", "image", "ls", "--format", "{{json .ID}}"], + text=True, + check=True, + stdout=subprocess.PIPE, + ).stdout + return {json.loads(image.strip()) for image in images.splitlines() if image.strip()} -@pytest.fixture( - params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"] -) -def build_frontend_env(request) -> Dict[str, str]: - return request.param # type: ignore[no-any-return] + images_before = get_images() + try: + yield + finally: + images_after = get_images() + for image in images_after - images_before: + subprocess.run(["docker", "rmi", image], check=False) diff --git a/test/test_0_basic.py b/test/test_0_basic.py index ec72483a3..2524d6286 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -1,8 +1,10 @@ import textwrap +import packaging.utils import pytest from cibuildwheel.logger import Logger +from cibuildwheel.selector import EnableGroup from . import test_projects, utils @@ -18,7 +20,16 @@ ) -def test(tmp_path, build_frontend_env): +@pytest.mark.serial +def test_dummy_serial(): + """A no-op test to ensure that at least one serial test is always found. + + Without this no-op test, CI fails on CircleCI because no serial tests are + found, and pytest errors if a test suite finds no tests. + """ + + +def test(tmp_path, build_frontend_env, capfd): project_dir = tmp_path / "project" basic_project.generate(project_dir) @@ -27,7 +38,17 @@ def test(tmp_path, build_frontend_env): # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") - assert set(actual_wheels) == set(expected_wheels) + actual_wheels_normalized = {packaging.utils.parse_wheel_filename(w) for w in actual_wheels} + expected_wheels_normalized = {packaging.utils.parse_wheel_filename(w) for w in expected_wheels} + assert actual_wheels_normalized == expected_wheels_normalized + + enable_groups = utils.get_enable_groups() + if EnableGroup.GraalPy not in enable_groups: + # Verify pip warning not shown + captured = capfd.readouterr() + for stream in (captured.err, captured.out): + assert "WARNING: Running pip as the 'root' user can result" not in stream + assert "A new release of pip available" not in stream @pytest.mark.skip(reason="to keep test output clean") @@ -46,16 +67,43 @@ def test_sample_build(tmp_path, capfd): logger.step_end() -def test_build_identifiers(tmp_path): +@pytest.mark.parametrize( + "enable_setting", ["", "cpython-prerelease", "pypy", "cpython-freethreading"] +) +def test_build_identifiers(tmp_path, enable_setting, monkeypatch): project_dir = tmp_path / "project" basic_project.generate(project_dir) + monkeypatch.setenv("CIBW_ENABLE", enable_setting) + # check that the number of expected wheels matches the number of build # identifiers expected_wheels = utils.expected_wheels("spam", "0.1.0") - build_identifiers = utils.cibuildwheel_get_build_identifiers( - project_dir, prerelease_pythons=True + build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir) + assert len(expected_wheels) == len(build_identifiers), ( + f"{expected_wheels} vs {build_identifiers}" + ) + + +@pytest.mark.parametrize( + ("add_args", "env_allow_empty"), + [ + (["--allow-empty"], {}), + (["--allow-empty"], {"CIBW_ALLOW_EMPTY": "0"}), + (None, {"CIBW_ALLOW_EMPTY": "1"}), + ], +) +def test_allow_empty(tmp_path, add_args, env_allow_empty): + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + # Sanity check - --allow-empty should cause a no-op build to complete + # without error + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={"CIBW_SKIP": "*", **env_allow_empty}, + add_args=add_args, ) - assert len(expected_wheels) == len( - build_identifiers - ), f"{expected_wheels} vs {build_identifiers}" + + # check that nothing was built + assert len(actual_wheels) == 0 diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py new file mode 100644 index 000000000..2a849f3a2 --- /dev/null +++ b/test/test_abi_variants.py @@ -0,0 +1,210 @@ +import textwrap + +from . import test_projects, utils + +pyproject_toml = r""" +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" +""" + +limited_api_project = test_projects.new_c_project( + setup_py_add=textwrap.dedent( + r""" + import sysconfig + + IS_CPYTHON = sys.implementation.name == "cpython" + Py_GIL_DISABLED = sysconfig.get_config_var("Py_GIL_DISABLED") + CAN_USE_ABI3 = IS_CPYTHON and not Py_GIL_DISABLED + setup_options = {} + extension_kwargs = {} + if CAN_USE_ABI3 and sys.version_info[:2] >= (3, 10): + extension_kwargs["define_macros"] = [("Py_LIMITED_API", "0x030A0000")] + extension_kwargs["py_limited_api"] = True + setup_options = {"bdist_wheel": {"py_limited_api": "cp310"}} + """ + ), + setup_py_extension_args_add="**extension_kwargs", + setup_py_setup_args_add="options=setup_options", +) + +limited_api_project.files["pyproject.toml"] = pyproject_toml + + +def test_abi3(tmp_path): + project_dir = tmp_path / "project" + limited_api_project.generate(project_dir) + + # build the wheels + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + # free_threaded, GraalPy, and PyPy do not have a Py_LIMITED_API equivalent, just build one of those + # also limit the number of builds for test performance reasons + "CIBW_BUILD": "cp39-* cp310-* pp310-* gp311_242-* cp312-* cp313t-*", + "CIBW_ENABLE": "all", + }, + ) + + # check that the expected wheels are produced + if utils.get_platform() == "pyodide": + # there's only 1 possible configuration for pyodide, cp312. It builds + # a wheel that is tagged abi3, compatible back to 3.10 + expected_wheels = utils.expected_wheels( + "spam", + "0.1.0", + python_abi_tags=["cp310-abi3"], + ) + else: + expected_wheels = utils.expected_wheels( + "spam", + "0.1.0", + python_abi_tags=[ + "cp39-cp39", + "cp310-abi3", # <-- ABI3, works with 3.10 and 3.12 + "cp313-cp313t", + "pp310-pypy310_pp73", + "graalpy311-graalpy242_311_native", + ], + ) + + assert set(actual_wheels) == set(expected_wheels) + + +ctypes_project = test_projects.TestProject() +ctypes_project.files["setup.py"] = textwrap.dedent( + """ + from setuptools import setup, Extension + + from distutils.command.build_ext import build_ext as _build_ext + class CTypesExtension(Extension): pass + class build_ext(_build_ext): + def build_extension(self, ext): + self._ctypes = isinstance(ext, CTypesExtension) + return super().build_extension(ext) + + def get_export_symbols(self, ext): + if self._ctypes: + return ext.export_symbols + return super().get_export_symbols(ext) + + def get_ext_filename(self, ext_name): + if self._ctypes: + return ext_name + '.so' + return super().get_ext_filename(ext_name) + + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel + class bdist_wheel_abi_none(_bdist_wheel): + def finalize_options(self): + _bdist_wheel.finalize_options(self) + self.root_is_pure = False + + def get_tag(self): + python, abi, plat = _bdist_wheel.get_tag(self) + return "py3", "none", plat + + setup( + name="ctypesexample", + version="1.0.0", + package_dir = {"": "src"}, + py_modules = ["ctypesexample.summing"], + ext_modules=[ + CTypesExtension( + "ctypesexample.csumlib", + ["src/ctypesexample/csumlib.c"], + ), + ], + cmdclass={'build_ext': build_ext, 'bdist_wheel': bdist_wheel_abi_none}, + ) + """ +) +ctypes_project.files["src/ctypesexample/csumlib.c"] = textwrap.dedent( + """ + #ifdef _WIN32 + #define LIBRARY_API __declspec(dllexport) + #else + #define LIBRARY_API + #endif + + #include + + + LIBRARY_API double *add_vec3(double *a, double *b) + { + double *res = malloc(sizeof(double) * 3); + + for (int i = 0; i < 3; ++i) + { + res[i] = a[i] + b[i]; + } + + return res; + } + """ +) +ctypes_project.files["src/ctypesexample/summing.py"] = textwrap.dedent( + """ + import ctypes + import pathlib + + # path of the shared library + libfile = pathlib.Path(__file__).parent / "csumlib.so" + csumlib = ctypes.CDLL(str(libfile)) + + type_vec3 = ctypes.POINTER(ctypes.c_double * 3) + + csumlib.add_vec3.restype = type_vec3 + csumlib.add_vec3.argtypes = [type_vec3, type_vec3] + def add(a: list, b: list) -> list: + a_p = (ctypes.c_double * 3)(*a) + b_p = (ctypes.c_double * 3)(*b) + r_p = csumlib.add_vec3(a_p,b_p) + + return [l for l in r_p.contents] + """ +) + +ctypes_project.files["test/add_test.py"] = textwrap.dedent( + """ + import ctypesexample.summing + + def test(): + a = [1, 2, 3] + b = [4, 5, 6] + assert ctypesexample.summing.add(a, b) == [5, 7, 9] + """ +) + +ctypes_project.files["pyproject.toml"] = pyproject_toml + + +def test_abi_none(tmp_path, capfd): + project_dir = tmp_path / "project" + ctypes_project.generate(project_dir) + + # build the wheels + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_TEST_REQUIRES": "pytest", + "CIBW_TEST_COMMAND": f"{utils.invoke_pytest()} {{project}}/test", + # limit the number of builds for test performance reasons + "CIBW_BUILD": "cp38-* cp{}{}-* cp313t-* pp310-*".format(*utils.SINGLE_PYTHON_VERSION), + "CIBW_ENABLE": "all", + }, + ) + + expected_wheels = utils.expected_wheels("ctypesexample", "1.0.0", python_abi_tags=["py3-none"]) + # check that the expected wheels are produced + assert set(actual_wheels) == set(expected_wheels) + + captured = capfd.readouterr() + + if utils.get_platform() == "pyodide": + # pyodide builds a different platform tag for each python version, so + # wheels are not reused + assert "Found previously built wheel" not in captured.out + else: + # check that each wheel was built once, and reused + assert "Building wheel..." in captured.out + assert "Found previously built wheel" in captured.out diff --git a/test/test_before_all.py b/test/test_before_all.py index 9c8722d72..f0c89972b 100644 --- a/test/test_before_all.py +++ b/test/test_before_all.py @@ -8,13 +8,22 @@ project_with_before_build_asserts = test_projects.new_c_project( setup_py_add=textwrap.dedent( r""" - # assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step - # is the same one as is currently running. + import os + with open("text_info.txt") as f: stored_text = f.read() - print("## stored text: " + stored_text) assert stored_text == "sample text 123" + + # assert that the Python version as written to python_prefix.txt in the CIBW_BEFORE_ALL step + # is not the same one as is currently running. + with open('python_prefix.txt') as f: + stored_prefix = f.read() + print('stored_prefix', stored_prefix) + print('sys.prefix', sys.prefix) + # Works around path-comparison bugs caused by short-paths on Windows e.g. + # vssadm~1 instead of vssadministrator + assert not os.path.samefile(stored_prefix, sys.prefix) """ ) ) @@ -27,22 +36,28 @@ def test(tmp_path): with (project_dir / "text_info.txt").open(mode="w") as ff: print("dummy text", file=ff) - # build the wheels - before_all_command = '''python -c "import os;open('{project}/text_info.txt', 'w').write('sample text '+os.environ.get('TEST_VAL', ''))"''' + # write python version information to a temporary file, this is checked in + # setup.py + # + # note, before_all runs in whatever the host environment is, `python` + # might be any version of python (even Python 2 on Travis ci!), so this is + # written to be broadly compatible + before_all_command = ( + """python -c "import os, sys; f = open('{project}/text_info.txt', 'w'); f.write('sample text '+os.environ.get('TEST_VAL', '')); f.close()" && """ + '''python -c "import sys; f = open('{project}/python_prefix.txt', 'w'); f.write(sys.prefix); f.close()"''' + ) actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ - # write python version information to a temporary file, this is - # checked in setup.py "CIBW_BEFORE_ALL": before_all_command, - "CIBW_BEFORE_ALL_LINUX": f'{before_all_command} && python -c "import sys; assert sys.version_info >= (3, 6)"', + "CIBW_BEFORE_ALL_LINUX": f'{before_all_command} && python -c "import sys; assert sys.version_info >= (3, 8)"', "CIBW_ENVIRONMENT": "TEST_VAL='123'", }, + single_python=True, ) # also check that we got the right wheels - (project_dir / "text_info.txt").unlink() - expected_wheels = utils.expected_wheels("spam", "0.1.0") + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) assert set(actual_wheels) == set(expected_wheels) @@ -70,7 +85,8 @@ def test_cwd(tmp_path): "CIBW_BEFORE_ALL": f'''python -c "import os; assert os.getcwd() == {str(project_dir)!r}"''', "CIBW_BEFORE_ALL_LINUX": '''python -c "import os; assert os.getcwd() == '/project'"''', }, + single_python=True, ) - expected_wheels = utils.expected_wheels("spam", "0.1.0") + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_before_build.py b/test/test_before_build.py index 00906acc1..767bb06cc 100644 --- a/test/test_before_build.py +++ b/test/test_before_build.py @@ -5,33 +5,32 @@ from . import test_projects, utils +# pyodide does not support building without isolation, need to check the base_prefix +SYS_PREFIX = f"sys.{'base_' if utils.get_platform() == 'pyodide' else ''}prefix" + + project_with_before_build_asserts = test_projects.new_c_project( setup_py_add=textwrap.dedent( - r""" + rf""" import os - # assert that the Python version as written to pythonversion.txt in the CIBW_BEFORE_BUILD step + # assert that the Python version as written to pythonversion_bb.txt in the CIBW_BEFORE_BUILD step # is the same one as is currently running. - version_file = 'c:\\pythonversion.txt' if sys.platform == 'win32' else '/tmp/pythonversion.txt' - with open(version_file) as f: + with open('pythonversion_bb.txt') as f: stored_version = f.read() print('stored_version', stored_version) print('sys.version', sys.version) assert stored_version == sys.version - # check that the executable also was written - executable_file = 'c:\\pythonexecutable.txt' if sys.platform == 'win32' else '/tmp/pythonexecutable.txt' - with open(executable_file) as f: - stored_executable = f.read() - print('stored_executable', stored_executable) - print('sys.executable', sys.executable) + # check that the prefix also was written + with open('pythonprefix_bb.txt') as f: + stored_prefix = f.read() + print('stored_prefix', stored_prefix) + print('{SYS_PREFIX}', {SYS_PREFIX}) + # Works around path-comparison bugs caused by short-paths on Windows e.g. + # vssadm~1 instead of vssadministrator - # windows/mac are case insensitive - stored_path = os.path.realpath(stored_executable).lower() - current_path = os.path.realpath(sys.executable).lower() - - # TODO: This is not valid in an virtual environment - assert stored_path == current_path, '{0} != {1}'.format(stored_path, current_path) + assert os.path.samefile(stored_prefix, {SYS_PREFIX}) """ ) ) @@ -42,9 +41,13 @@ def test(tmp_path): project_with_before_build_asserts.generate(project_dir) before_build = ( - """python -c "import sys; open('{output_dir}pythonversion.txt', 'w').write(sys.version)" && """ - '''python -c "import sys; open('{output_dir}pythonexecutable.txt', 'w').write(sys.executable)"''' + """python -c "import pathlib, sys; pathlib.Path('{project}/pythonversion_bb.txt').write_text(sys.version)" && """ + f'''python -c "import pathlib, sys; pathlib.Path('{{project}}/pythonprefix_bb.txt').write_text({SYS_PREFIX})"''' ) + frontend = "build" + if utils.get_platform() != "pyodide": + before_build = f"python -m pip install setuptools && {before_build}" + frontend = f"{frontend};args: --no-isolation" # build the wheels actual_wheels = utils.cibuildwheel_run( @@ -52,8 +55,8 @@ def test(tmp_path): add_env={ # write python version information to a temporary file, this is # checked in setup.py - "CIBW_BEFORE_BUILD": before_build.format(output_dir="/tmp/"), - "CIBW_BEFORE_BUILD_WINDOWS": before_build.format(output_dir=r"c:\\"), + "CIBW_BEFORE_BUILD": before_build, + "CIBW_BUILD_FRONTEND": frontend, }, ) @@ -86,7 +89,8 @@ def test_cwd(tmp_path): "CIBW_BEFORE_BUILD": f'''python -c "import os; assert os.getcwd() == {str(project_dir)!r}"''', "CIBW_BEFORE_BUILD_LINUX": '''python -c "import os; assert os.getcwd() == '/project'"''', }, + single_python=True, ) - expected_wheels = utils.expected_wheels("spam", "0.1.0") + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_before_test.py b/test/test_before_test.py index 535131aa0..3c722d089 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -1,59 +1,72 @@ from . import test_projects, utils before_test_project = test_projects.new_c_project() -before_test_project.files[ - "test/spam_test.py" -] = r""" +before_test_project.files["test/spam_test.py"] = r""" import sys import os +from pathlib import Path from unittest import TestCase +PROJECT_DIR = Path(__file__).parent.parent.resolve() + class TestBeforeTest(TestCase): def test_version(self): - # assert that the Python version as written to pythonversion.txt in the CIBW_BEFORE_TEST step + # assert that the Python version as written to pythonversion_bt.txt in the CIBW_BEFORE_TEST step # is the same one as is currently running. # because of use symlinks in MacOS run this test is also need - version_file = 'c:\\pythonversion.txt' if sys.platform == 'win32' else '/tmp/pythonversion.txt' - with open(version_file) as f: - stored_version = f.read() + stored_version = PROJECT_DIR.joinpath('pythonversion_bt.txt').read_text() print('stored_version', stored_version) print('sys.version', sys.version) assert stored_version == sys.version def test_prefix(self): # check that the prefix also was written - prefix_file = 'c:\\pythonprefix.txt' if sys.platform == 'win32' else '/tmp/pythonprefix.txt' - with open(prefix_file) as f: - stored_prefix = f.read() + stored_prefix = PROJECT_DIR.joinpath('pythonprefix_bt.txt').read_text() print('stored_prefix', stored_prefix) print('sys.prefix', sys.prefix) # Works around path-comparison bugs caused by short-paths on Windows e.g. # vssadm~1 instead of vssadministrator - assert os.stat(stored_prefix) == os.stat(sys.prefix) + assert os.path.samefile(stored_prefix, sys.prefix) """ -def test(tmp_path): +def test(tmp_path, build_frontend_env): project_dir = tmp_path / "project" before_test_project.generate(project_dir) test_project_dir = project_dir / "dependency" test_projects.new_c_project().generate(test_project_dir) + before_test_steps = [ + '''python -c "import pathlib, sys; pathlib.Path('{project}/pythonversion_bt.txt').write_text(sys.version)"''', + '''python -c "import pathlib, sys; pathlib.Path('{project}/pythonprefix_bt.txt').write_text(sys.prefix)"''', + ] + + if utils.get_platform() == "pyodide": + before_test_steps.extend( + ["pyodide build {project}/dependency", "pip install --find-links dist/ spam"] + ) + elif build_frontend_env["CIBW_BUILD_FRONTEND"] in {"pip", "build"}: + before_test_steps.append("python -m pip install {project}/dependency") + + before_test = " && ".join(before_test_steps) + # build the wheels actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ # write python version information to a temporary file, this is # checked in setup.py - "CIBW_BEFORE_TEST": """python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency""", - "CIBW_BEFORE_TEST_WINDOWS": """python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency""", + "CIBW_BEFORE_TEST": before_test, "CIBW_TEST_REQUIRES": "pytest", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": "false || pytest {project}/test", - "CIBW_TEST_COMMAND_WINDOWS": "pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", + # pytest fails on GraalPy 24.2.0 on Windows so we skip it there + # until https://github.com/oracle/graalpython/issues/490 is fixed + "CIBW_TEST_COMMAND_WINDOWS": "where graalpy || pytest {project}/test", + **build_frontend_env, }, ) diff --git a/test/test_build_frontend_args.py b/test/test_build_frontend_args.py new file mode 100644 index 000000000..83c0a1775 --- /dev/null +++ b/test/test_build_frontend_args.py @@ -0,0 +1,40 @@ +import subprocess + +import pytest + +from . import utils +from .test_projects.c import new_c_project + + +@pytest.mark.parametrize( + "frontend_name", + [ + pytest.param("pip", marks=utils.skip_if_pyodide("No pip for pyodide")), + "build", + ], +) +def test_build_frontend_args(tmp_path, capfd, frontend_name): + project = new_c_project() + project_dir = tmp_path / "project" + project.generate(project_dir) + + # the build will fail because the frontend is called with '-h' - it prints the help message + add_env = {"CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h"} + if utils.get_platform() == "pyodide": + add_env["TERM"] = "dumb" # disable color / style + add_env["NO_COLOR"] = "1" + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run(project_dir, add_env=add_env, single_python=True) + + captured = capfd.readouterr() + print(captured.out) + + # check that the help message was printed + if frontend_name == "pip": + assert "Usage:" in captured.out + assert "Wheel Options:" in captured.out + elif utils.get_platform() == "pyodide": + assert "Usage: pyodide build" in captured.out + else: + assert "usage:" in captured.out + assert "A simple, correct Python build frontend." in captured.out diff --git a/test/test_build_skip.py b/test/test_build_skip.py index 0d285d822..ad7d6f9de 100644 --- a/test/test_build_skip.py +++ b/test/test_build_skip.py @@ -4,12 +4,12 @@ project_with_skip_asserts = test_projects.new_c_project( setup_py_add=textwrap.dedent( - r""" - # explode if run on PyPyor Python 3.7 (these should be skipped) + rf""" if sys.implementation.name != "cpython": raise Exception("Only CPython shall be built") - if sys.version_info[0:2] == (3, 7): - raise Exception("CPython 3.7 should be skipped") + expected_version = {"({}, {})".format(*utils.SINGLE_PYTHON_VERSION)} + if sys.version_info[0:2] != expected_version: + raise Exception("CPython {{}}.{{}} should be skipped".format(*sys.version_info[0:2])) """ ) ) @@ -19,17 +19,19 @@ def test(tmp_path): project_dir = tmp_path / "project" project_with_skip_asserts.generate(project_dir) + skip = " ".join( + f"cp3{minor}-*" for minor in range(6, 30) if (3, minor) != utils.SINGLE_PYTHON_VERSION + ) + # build the wheels actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ "CIBW_BUILD": "cp3*-*", - "CIBW_SKIP": "cp37-*", + "CIBW_SKIP": f"*t-* {skip}", }, ) - # check that we got the right wheels. There should be no PyPy or 3.7. - expected_wheels = [ - w for w in utils.expected_wheels("spam", "0.1.0") if ("-cp3" in w) and ("-cp37" not in w) - ] + # check that we got the right wheels. There should be a single version of CPython. + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_container_engine.py b/test/test_container_engine.py new file mode 100644 index 000000000..510916f06 --- /dev/null +++ b/test/test_container_engine.py @@ -0,0 +1,62 @@ +import pytest + +from . import test_projects, utils + +basic_project = test_projects.new_c_project() + + +def test_podman(tmp_path, capfd, request): + if utils.get_platform() != "linux": + pytest.skip("the test is only relevant to the linux build") + + if not request.config.getoption("--run-podman"): + pytest.skip("needs --run-podman option to run") + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + # build some musllinux and manylinux wheels (ensuring that we use two containers) + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_ARCHS": "native", + "CIBW_BEFORE_ALL": "echo 'test log statement from before-all'", + "CIBW_CONTAINER_ENGINE": "podman", + }, + single_python=True, + ) + + # check that the expected wheels are produced + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True, single_arch=True) + assert set(actual_wheels) == set(expected_wheels) + + # check that stdout is bring passed-though from container correctly + captured = capfd.readouterr() + assert "test log statement from before-all" in captured.out + + +def test_create_args(tmp_path, capfd): + if utils.get_platform() != "linux": + pytest.skip("the test is only relevant to the linux build") + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + # build a manylinux wheel, using create_args to set an environment variable + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_SKIP": "*-musllinux_*", + "CIBW_BEFORE_ALL": "echo TEST_CREATE_ARGS is set to $TEST_CREATE_ARGS", + "CIBW_CONTAINER_ENGINE": "docker; create_args: --env=TEST_CREATE_ARGS=itworks", + }, + single_python=True, + ) + + expected_wheels = utils.expected_wheels( + "spam", "0.1.0", musllinux_versions=[], single_python=True + ) + assert set(actual_wheels) == set(expected_wheels) + + captured = capfd.readouterr() + assert "TEST_CREATE_ARGS is set to itworks" in captured.out diff --git a/test/test_docker_images.py b/test/test_container_images.py similarity index 73% rename from test/test_docker_images.py rename to test/test_container_images.py index 8318a64ea..02263c649 100644 --- a/test/test_docker_images.py +++ b/test/test_container_images.py @@ -11,7 +11,7 @@ import os # check that we're running in the correct docker image as specified in the - # environment options CIBW_MANYLINUX1_*_IMAGE + # environment options CIBW_MANYLINUX_*_IMAGE if "linux" in sys.platform and not os.path.exists("/dockcross"): raise Exception( "/dockcross directory not found. Is this test running in the correct docker image?" @@ -21,8 +21,9 @@ ) +@pytest.mark.usefixtures("docker_cleanup") def test(tmp_path): - if utils.platform != "linux": + if utils.get_platform() != "linux": pytest.skip("the test is only relevant to the linux build") if platform.machine() not in ["x86_64", "i686"]: pytest.skip( @@ -37,14 +38,17 @@ def test(tmp_path): add_env={ "CIBW_MANYLINUX_X86_64_IMAGE": "dockcross/manylinux2014-x64", "CIBW_MANYLINUX_I686_IMAGE": "dockcross/manylinux2014-x86", - "CIBW_BUILD": "cp3{6,7,8,9}-manylinux*", + "CIBW_BUILD": "cp3{8,9}-manylinux*", }, ) # also check that we got the right wheels built + manylinux_versions = ["manylinux_2_5", "manylinux1", "manylinux_2_17", "manylinux2014"] expected_wheels = [ w - for w in utils.expected_wheels("spam", "0.1.0", musllinux_versions=[]) - if "-cp36-" in w or "-cp37-" in w or "-cp38-" in w or "-cp39-" in w + for w in utils.expected_wheels( + "spam", "0.1.0", manylinux_versions=manylinux_versions, musllinux_versions=[] + ) + if "-cp38-" in w or "-cp39-" in w ] assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_cpp_standards.py b/test/test_cpp_standards.py index a5385007f..fb7a0a84e 100644 --- a/test/test_cpp_standards.py +++ b/test/test_cpp_standards.py @@ -1,7 +1,4 @@ -import os - import jinja2 -import pytest from . import utils from .test_projects import TestProject @@ -54,78 +51,59 @@ cpp_test_project.files["setup.py"] = jinja2.Template(setup_py_template) cpp_test_project.files["spam.cpp"] = jinja2.Template(spam_cpp_template) -cpp11_project = cpp_test_project.copy() -cpp11_project.template_context["extra_compile_args"] = ( - ["/std:c++11"] if utils.platform == "windows" else ["-std=c++11"] -) -cpp11_project.template_context["spam_cpp_top_level_add"] = "#include " - def test_cpp11(tmp_path): # This test checks that the C++11 standard is supported project_dir = tmp_path / "project" - + cpp11_project = cpp_test_project.copy() + cpp11_project.template_context["extra_compile_args"] = ( + ["/std:c++11"] if utils.get_platform() == "windows" else ["-std=c++11"] + ) + cpp11_project.template_context["spam_cpp_top_level_add"] = "#include " cpp11_project.generate(project_dir) - actual_wheels = utils.cibuildwheel_run(project_dir) - expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0")] + actual_wheels = utils.cibuildwheel_run(project_dir, single_python=True) + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) assert set(actual_wheels) == set(expected_wheels) -cpp14_project = cpp_test_project.copy() -cpp14_project.template_context["extra_compile_args"] = ( - ["/std:c++14"] if utils.platform == "windows" else ["-std=c++14"] -) -cpp14_project.template_context["spam_cpp_top_level_add"] = "int a = 100'000;" - - def test_cpp14(tmp_path): # This test checks that the C++14 standard is supported project_dir = tmp_path / "project" - + cpp14_project = cpp_test_project.copy() + cpp14_project.template_context["extra_compile_args"] = ( + ["/std:c++14"] if utils.get_platform() == "windows" else ["-std=c++14"] + ) + cpp14_project.template_context["spam_cpp_top_level_add"] = "int a = 100'000;" cpp14_project.generate(project_dir) - actual_wheels = utils.cibuildwheel_run(project_dir) - expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0")] + actual_wheels = utils.cibuildwheel_run(project_dir, single_python=True) + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) assert set(actual_wheels) == set(expected_wheels) -cpp17_project = cpp_test_project.copy() - -cpp17_project.template_context["extra_compile_args"] = [ - "/std:c++17" if utils.platform == "windows" else "-std=c++17" -] -cpp17_project.template_context[ - "spam_cpp_top_level_add" -] = r""" -#include -auto a = std::pair(5.0, false); -""" - - def test_cpp17(tmp_path): # This test checks that the C++17 standard is supported project_dir = tmp_path / "project" - + cpp17_project = cpp_test_project.copy() + cpp17_project.template_context["extra_compile_args"] = [ + "/std:c++17" if utils.get_platform() == "windows" else "-std=c++17" + ] + cpp17_project.template_context["spam_cpp_top_level_add"] = r""" + #include + auto a = std::pair(5.0, false); + """ cpp17_project.generate(project_dir) - if os.environ.get("APPVEYOR_BUILD_WORKER_IMAGE", "") == "Visual Studio 2015": - pytest.skip("Visual Studio 2015 does not support C++17") - - # Pypy's distutils sets the default compiler to 'msvc9compiler', which - # is too old to support cpp17. - add_env = {"CIBW_SKIP": "pp??-*"} - - if utils.platform == "macos": + add_env = {} + if utils.get_platform() == "macos": add_env["MACOSX_DEPLOYMENT_TARGET"] = "10.13" - actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env) - expected_wheels = [ - w - for w in utils.expected_wheels("spam", "0.1.0", macosx_deployment_target="10.13") - if "-pp" not in w - ] + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env, single_python=True) + expected_wheels = utils.expected_wheels( + "spam", "0.1.0", macosx_deployment_target="10.13", single_python=True + ) assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_custom_repair_wheel.py b/test/test_custom_repair_wheel.py new file mode 100644 index 000000000..5af012653 --- /dev/null +++ b/test/test_custom_repair_wheel.py @@ -0,0 +1,58 @@ +import subprocess +from contextlib import nullcontext as does_not_raise + +import pytest + +from test import test_projects + +from . import utils + +basic_project = test_projects.new_c_project() +basic_project.files["repair.py"] = """ +import shutil +import sys +from pathlib import Path + +wheel = Path(sys.argv[1]) +dest_dir = Path(sys.argv[2]) +platform = wheel.stem.split("-")[-1] +if platform.startswith("pyodide"): + # for the sake of this test, munge the pyodide platforms into one, it's + # not valid, but it does activate the uniqueness check + platform = "pyodide" + +name = f"spam-0.1.0-py2-none-{platform}.whl" +dest = dest_dir / name +dest_dir.mkdir(parents=True, exist_ok=True) +dest.unlink(missing_ok=True) +shutil.copy(wheel, dest) +""" + + +def test(tmp_path, capfd): + # this test checks that a generated wheel name shall be unique in a given cibuildwheel run + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + num_builds = len(utils.cibuildwheel_get_build_identifiers(project_dir)) + expectation = ( + pytest.raises(subprocess.CalledProcessError) if num_builds > 1 else does_not_raise() + ) + + with expectation as exc_info: + result = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_REPAIR_WHEEL_COMMAND": "python repair.py {wheel} {dest_dir}", + }, + ) + + captured = capfd.readouterr() + if num_builds > 1: + assert exc_info is not None + assert "Build failed because a wheel named" in captured.err + assert exc_info.value.returncode == 6 + else: + # We only produced one wheel (perhaps Pyodide) + # check that it has the right name + assert result[0].startswith("spam-0.1.0-py2-none-") diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index 90854a866..5aa84a506 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -1,9 +1,11 @@ +import platform import re import textwrap +from pathlib import Path import pytest -import cibuildwheel.util +from cibuildwheel.util import resources from . import test_projects, utils @@ -12,9 +14,10 @@ r""" import subprocess import os + import sys versions_output_text = subprocess.check_output( - ['pip', 'freeze', '--all', '-qq'], + [sys.executable, '-m', 'pip', 'freeze', '--all', '-qq'], universal_newlines=True, ) versions = versions_output_text.strip().splitlines() @@ -24,58 +27,57 @@ print('Gathered versions', versions) - for package_name in ['pip', 'setuptools', 'wheel']: - env_name = 'EXPECTED_{}_VERSION'.format(package_name.upper()) - expected_version = os.environ[env_name] + expected_version = os.environ['EXPECTED_PIP_VERSION'] - assert '{}=={}'.format(package_name, expected_version) in versions, ( - 'error: {} version should equal {}'.format(package_name, expected_version) - ) + assert f'pip=={expected_version}' in versions, ( + f'error: pip version should equal {expected_version}' + ) """ ) ) +project_with_expected_version_checks.files["pyproject.toml"] = r""" +[build-system] +requires = ["setuptools", "pip"] +build-backend = "setuptools.build_meta" +""" VERSION_REGEX = r"([\w-]+)==([^\s]+)" -def get_versions_from_constraint_file(constraint_file): - constraint_file_text = constraint_file.read_text(encoding="utf8") +def get_versions_from_constraint_file(constraint_file: Path) -> dict[str, str]: + constraint_file_text = constraint_file.read_text(encoding="utf-8") - return { - package: version for package, version in re.findall(VERSION_REGEX, constraint_file_text) - } + return dict(re.findall(VERSION_REGEX, constraint_file_text)) -@pytest.mark.parametrize("python_version", ["3.6", "3.8", "3.9"]) -def test_pinned_versions(tmp_path, python_version, build_frontend_env): - if utils.platform == "linux": +@pytest.mark.parametrize("python_version", ["3.8", "3.12"]) +def test_pinned_versions(tmp_path, python_version, build_frontend_env_nouv): + if utils.get_platform() == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") + if python_version != "3.12" and utils.get_platform() == "pyodide": + pytest.skip(f"pyodide does not support Python {python_version}") + if ( + python_version == "3.8" + and utils.get_platform() == "windows" + and platform.machine() == "ARM64" + ): + pytest.skip(f"Windows ARM64 does not support Python {python_version}") project_dir = tmp_path / "project" project_with_expected_version_checks.generate(project_dir) + version_no_dot = python_version.replace(".", "") build_environment = {} - - if python_version == "3.6": - constraint_filename = "constraints-python36.txt" - build_pattern = "[cp]p36-*" - elif python_version == "3.7": - constraint_filename = "constraints-python37.txt" - build_pattern = "[cp]p37-*" - elif python_version == "3.8": - constraint_filename = "constraints-python38.txt" - build_pattern = "[cp]p38-*" + build_pattern = f"[cp]p{version_no_dot}-*" + if utils.get_platform() == "pyodide": + constraint_filename = f"constraints-pyodide{version_no_dot}.txt" else: - constraint_filename = "constraints.txt" - build_pattern = "[cp]p39-*" - - constraint_file = cibuildwheel.util.resources_dir / constraint_filename + constraint_filename = f"constraints-python{version_no_dot}.txt" + constraint_file = resources.PATH / constraint_filename constraint_versions = get_versions_from_constraint_file(constraint_file) - for package in ["pip", "setuptools", "wheel", "virtualenv"]: - env_name = f"EXPECTED_{package.upper()}_VERSION" - build_environment[env_name] = constraint_versions[package] + build_environment["EXPECTED_PIP_VERSION"] = constraint_versions["pip"] cibw_environment_option = " ".join(f"{k}={v}" for k, v in build_environment.items()) @@ -85,60 +87,65 @@ def test_pinned_versions(tmp_path, python_version, build_frontend_env): add_env={ "CIBW_BUILD": build_pattern, "CIBW_ENVIRONMENT": cibw_environment_option, - **build_frontend_env, + **build_frontend_env_nouv, }, ) # also check that we got the right wheels - if python_version == "3.6": - expected_wheels = [ - w for w in utils.expected_wheels("spam", "0.1.0") if "-cp36" in w or "-pp36" in w - ] - elif python_version == "3.8": - expected_wheels = [ - w for w in utils.expected_wheels("spam", "0.1.0") if "-cp38" in w or "-pp38" in w - ] - elif python_version == "3.9": - expected_wheels = [ - w for w in utils.expected_wheels("spam", "0.1.0") if "-cp39" in w or "-pp39" in w - ] - else: - raise ValueError("unhandled python version") + expected_wheels = [ + w + for w in utils.expected_wheels("spam", "0.1.0") + if f"-cp{version_no_dot}" in w or f"-pp{version_no_dot}" in w + ] assert set(actual_wheels) == set(expected_wheels) -def test_dependency_constraints_file(tmp_path, build_frontend_env): - if utils.platform == "linux": +@pytest.mark.parametrize("method", ["inline", "file"]) +def test_dependency_constraints(method, tmp_path, build_frontend_env_nouv): + if utils.get_platform() == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") project_dir = tmp_path / "project" project_with_expected_version_checks.generate(project_dir) tool_versions = { - "pip": "20.0.2", - "setuptools": "53.0.0", - "wheel": "0.34.2", - "virtualenv": "20.11.2", + "pip": "23.1.2", + "delocate": "0.10.3", } - constraints_file = tmp_path / "constraints file.txt" - constraints_file.write_text( - textwrap.dedent( - """ - pip=={pip} - setuptools=={setuptools} - wheel=={wheel} - virtualenv=={virtualenv} - importlib-metadata<3,>=0.12; python_version < "3.8" - """.format( - **tool_versions + if method == "file": + constraints_file = tmp_path / "constraints file.txt" + constraints_file.write_text( + textwrap.dedent( + """ + pip=={pip} + delocate=={delocate} + """.format(**tool_versions) ) ) - ) + dependency_version_option = str(constraints_file) + elif method == "inline": + dependency_version_option = "packages: " + " ".join( + f"{k}=={v}" for k, v in tool_versions.items() + ) + else: + msg = f"Unknown method: {method}" + raise ValueError(msg) build_environment = {} + if ( + utils.get_platform() == "windows" + and method == "file" + and build_frontend_env_nouv["CIBW_BUILD_FRONTEND"] == "build" + ): + # GraalPy fails to discover its standard library when a venv is created + # from a virtualenv seeded executable. See + # https://github.com/oracle/graalpython/issues/491 and remove this once + # fixed upstream. + build_frontend_env_nouv["CIBW_SKIP"] = "gp*" + for package_name, version in tool_versions.items(): env_name = f"EXPECTED_{package_name.upper()}_VERSION" build_environment[env_name] = version @@ -150,12 +157,21 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env): project_dir, add_env={ "CIBW_ENVIRONMENT": cibw_environment_option, - "CIBW_DEPENDENCY_VERSIONS": str(constraints_file), - **build_frontend_env, + "CIBW_DEPENDENCY_VERSIONS": dependency_version_option, + **build_frontend_env_nouv, }, ) # also check that we got the right wheels expected_wheels = utils.expected_wheels("spam", "0.1.0") + if ( + utils.get_platform() == "windows" + and method == "file" + and build_frontend_env_nouv["CIBW_BUILD_FRONTEND"] == "build" + ): + # See reference to https://github.com/oracle/graalpython/issues/491 + # above + expected_wheels = [w for w in expected_wheels if "graalpy" not in w] + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_emulation.py b/test/test_emulation.py index dd76ab36c..4a749cfa3 100644 --- a/test/test_emulation.py +++ b/test/test_emulation.py @@ -1,3 +1,4 @@ +import itertools import subprocess import pytest @@ -6,19 +7,23 @@ project_with_a_test = test_projects.new_c_project() -project_with_a_test.files[ - "test/spam_test.py" -] = r""" +project_with_a_test.files["test/spam_test.py"] = r""" import spam def test_spam(): - assert spam.system('python -c "exit(0)"') == 0 - assert spam.system('python -c "exit(1)"') != 0 + assert spam.filter("spam") == 0 + assert spam.filter("ham") != 0 """ -@pytest.mark.emulation -def test(tmp_path): +def test(tmp_path, request): + archs = request.config.getoption("--run-emulation") + if archs is None: + pytest.skip("needs --run-emulation option to run") + + if archs == "all": + archs = " ".join(utils.EMULATED_ARCHS) + project_dir = tmp_path / "project" project_with_a_test.generate(project_dir) @@ -28,21 +33,27 @@ def test(tmp_path): add_env={ "CIBW_TEST_REQUIRES": "pytest", "CIBW_TEST_COMMAND": "pytest {project}/test", - "CIBW_ARCHS": "aarch64 ppc64le s390x", + "CIBW_ARCHS": archs, + # TODO remove me once proper support is added + "CIBW_MANYLINUX_RISCV64_IMAGE": "ghcr.io/mayeut/manylinux_2_35:2025.05.11-1", + "CIBW_SKIP": "*-musllinux_riscv64", }, ) # also check that we got the right wheels - expected_wheels = ( - utils.expected_wheels("spam", "0.1.0", machine_arch="aarch64") - + utils.expected_wheels("spam", "0.1.0", machine_arch="ppc64le") - + utils.expected_wheels("spam", "0.1.0", machine_arch="s390x") + expected_wheels = list( + itertools.chain.from_iterable( + utils.expected_wheels("spam", "0.1.0", machine_arch=arch, single_arch=True) + for arch in archs.split(" ") + ) ) + # TODO remove me once proper support is added + expected_wheels = [wheel for wheel in expected_wheels if "musllinux_1_2_riscv64" not in wheel] assert set(actual_wheels) == set(expected_wheels) def test_setting_arch_on_other_platforms(tmp_path, capfd): - if utils.platform == "linux": + if utils.get_platform() == "linux": pytest.skip("this test checks the behaviour on platforms other than linux") project_dir = tmp_path / "project" diff --git a/test/test_environment.py b/test/test_environment.py index 6ad8ff9f4..30bab81f5 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -1,5 +1,6 @@ import os import subprocess +import sys import textwrap import pytest @@ -33,6 +34,7 @@ def test(tmp_path): + python_echo = f"'{sys.executable}' -c \"import sys; print(*sys.argv[1:])\"" project_dir = tmp_path / "project" project_with_environment_asserts.generate(project_dir) @@ -43,12 +45,13 @@ def test(tmp_path): project_dir, add_env={ "CIBW_ENVIRONMENT": """CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH=$PATH:/opt/cibw_test_path""", - "CIBW_ENVIRONMENT_WINDOWS": '''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH="$PATH;/opt/cibw_test_path"''', + "CIBW_ENVIRONMENT_WINDOWS": f'''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$({python_echo} 'test string 3')" PATH="$PATH;/opt/cibw_test_path"''', }, + single_python=True, ) # also check that we got the right wheels built - expected_wheels = utils.expected_wheels("spam", "0.1.0") + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) assert set(actual_wheels) == set(expected_wheels) @@ -61,8 +64,8 @@ def test_overridden_path(tmp_path, capfd): output_dir.mkdir() # mess up PATH, somehow - with pytest.raises(subprocess.CalledProcessError): - if utils.platform == "linux": + if utils.get_platform() == "linux": + with pytest.raises(subprocess.CalledProcessError): utils.cibuildwheel_run( project_dir, output_dir=output_dir, @@ -71,11 +74,12 @@ def test_overridden_path(tmp_path, capfd): "CIBW_ENVIRONMENT": '''PATH="$(pwd)/new_path:$PATH"''', }, ) - else: - new_path = tmp_path / "another_bin" - new_path.mkdir() - (new_path / "python").touch(mode=0o777) + else: + new_path = tmp_path / "another_bin" + new_path.mkdir() + (new_path / "python").touch(mode=0o777) + with pytest.raises(subprocess.CalledProcessError): utils.cibuildwheel_run( project_dir, output_dir=output_dir, @@ -85,6 +89,68 @@ def test_overridden_path(tmp_path, capfd): }, ) - assert len(os.listdir(output_dir)) == 0 + assert len(list(output_dir.iterdir())) == 0 captured = capfd.readouterr() - assert "python available on PATH doesn't match our installed instance" in captured.err + assert "python available on PATH doesn't match our installed instance" in captured.err.replace( + "venv", "installed" + ) + + +@pytest.mark.parametrize( + "build_frontend", + [ + pytest.param("pip", marks=utils.skip_if_pyodide("No pip for pyodide")), + pytest.param( + "build", + marks=utils.skip_if_pyodide( + "pyodide doesn't support multiple values for PIP_CONSTRAINT" + ), + ), + ], +) +def test_overridden_pip_constraint(tmp_path, build_frontend): + """ + Verify that users can use PIP_CONSTRAINT to specify a specific version of + a build-system.requires dependency, by asserting the version of pytz in the + setup.py. + """ + project_dir = tmp_path / "project" + + project = test_projects.new_c_project( + setup_py_add=textwrap.dedent( + """ + import pytz + assert pytz.__version__ == "2022.4", f"{pytz.__version__!r} != '2022.4'" + """ + ) + ) + project.files["pyproject.toml"] = textwrap.dedent( + """ + [build-system] + requires = ["setuptools", "pytz"] + build-backend = "setuptools.build_meta" + """ + ) + project.generate(project_dir) + + if utils.get_platform() == "linux": + # put the constraints file in the project directory, so it's available + # in the docker container + constraints_file = project_dir / "constraints.txt" + else: + constraints_file = tmp_path / "constraints.txt" + + constraints_file.write_text("pytz==2022.4") + + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_BUILD_FRONTEND": build_frontend, + "PIP_CONSTRAINT": str(constraints_file), + "CIBW_ENVIRONMENT_LINUX": "PIP_CONSTRAINT=./constraints.txt", + }, + single_python=True, + ) + + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_from_sdist.py b/test/test_from_sdist.py new file mode 100644 index 000000000..b916271cc --- /dev/null +++ b/test/test_from_sdist.py @@ -0,0 +1,181 @@ +import os +import subprocess +import sys +import textwrap +from collections.abc import Mapping +from pathlib import Path +from tempfile import TemporaryDirectory + +from test.test_projects.base import TestProject + +from . import test_projects, utils + +# utilities + + +def make_sdist(project: TestProject, working_dir: Path) -> Path: + project_dir = working_dir / "project" + project_dir.mkdir(parents=True, exist_ok=True) + project.generate(project_dir) + + sdist_dir = working_dir / "sdist" + subprocess.run( + [sys.executable, "-m", "build", "--sdist", "--outdir", str(sdist_dir), str(project_dir)], + check=True, + ) + + return next(sdist_dir.glob("*.tar.gz")) + + +def cibuildwheel_from_sdist_run( + sdist_path: Path | str, + add_env: Mapping[str, str] | None = None, + config_file: str | None = None, +) -> list[str]: + env = os.environ.copy() + + if add_env: + env.update(add_env) + + env["CIBW_BUILD"] = "cp{}{}-*".format(*utils.SINGLE_PYTHON_VERSION) + + with TemporaryDirectory() as tmp_output_dir: + subprocess.run( + [ + sys.executable, + "-m", + "cibuildwheel", + *(["--config-file", config_file] if config_file else []), + "--output-dir", + str(tmp_output_dir), + str(sdist_path), + ], + env=env, + check=True, + ) + return [p.name for p in Path(tmp_output_dir).iterdir()] + + +# tests + + +def test_simple(tmp_path): + basic_project = test_projects.new_c_project() + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(basic_project, sdist_dir) + + setup_py_assertion_snippet = textwrap.dedent( + """ + import os + + assert os.path.exists('setup.py') + assert os.path.exists('{package}/setup.py') + """, + ) + setup_py_assertion_cmd = f'python -c "{setup_py_assertion_snippet!s}"' + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, add_env={"CIBW_BEFORE_BUILD": setup_py_assertion_cmd} + ) + + # check that the expected wheels are produced + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) + assert set(actual_wheels) == set(expected_wheels) + + +def test_external_config_file_argument(tmp_path, capfd): + basic_project = test_projects.new_c_project() + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(basic_project, sdist_dir) + + # add a config file + config_file = tmp_path / "config.toml" + config_file.write_text( + textwrap.dedent( + """ + [tool.cibuildwheel] + before-all = 'echo "test log statement from before-all"' + """ + ) + ) + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run(sdist_path, config_file=str(config_file)) + + # check that the expected wheels are produced + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) + assert set(actual_wheels) == set(expected_wheels) + + # check that before-all was run + captured = capfd.readouterr() + assert "test log statement from before-all" in captured.out + + +def test_config_in_pyproject_toml(tmp_path, capfd): + # make a project with a pyproject.toml + project = test_projects.new_c_project() + project.files["pyproject.toml"] = textwrap.dedent( + """ + [tool.cibuildwheel] + before-build = 'echo "test log statement from before-build 8419"' + """ + ) + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(project, sdist_dir) + + # build the wheels from sdist + actual_wheels = cibuildwheel_from_sdist_run(sdist_path) + + # check that the expected wheels are produced + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) + assert set(actual_wheels) == set(expected_wheels) + + # check that before-build was run + captured = capfd.readouterr() + assert "test log statement from before-build 8419" in captured.out + + +def test_internal_config_file_argument(tmp_path, capfd): + # make a project with a config file inside + project = test_projects.new_c_project( + setup_cfg_add="include_package_data = True", + ) + project.files["wheel_build_config.toml"] = textwrap.dedent( + """ + [tool.cibuildwheel] + before-all = 'echo "test log statement from before-all 1829"' + """ + ) + project.files["MANIFEST.in"] = textwrap.dedent( + """ + include wheel_build_config.toml + """ + ) + + # make an sdist of the project + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + sdist_path = make_sdist(project, sdist_dir) + + # build the wheels from sdist, referencing the config file inside + actual_wheels = cibuildwheel_from_sdist_run( + sdist_path, config_file="{package}/wheel_build_config.toml" + ) + + # check that the expected wheels are produced + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) + assert set(actual_wheels) == set(expected_wheels) + + # check that before-all was run + captured = capfd.readouterr() + assert "test log statement from before-all 1829" in captured.out diff --git a/test/test_ios.py b/test/test_ios.py new file mode 100644 index 000000000..b2d7e892a --- /dev/null +++ b/test/test_ios.py @@ -0,0 +1,347 @@ +from __future__ import annotations + +import os +import platform +import shutil +import subprocess +import textwrap + +import pytest + +from cibuildwheel.ci import CIProvider, detect_ci_provider + +from . import test_projects, utils + +basic_project_files = { + "tests/test_platform.py": f""" +import platform +from unittest import TestCase + +class TestPlatform(TestCase): + def test_platform(self): + self.assertEqual(platform.machine(), "{platform.machine()}") + +""" +} + + +def skip_if_ios_testing_not_supported() -> None: + """Skip the test if iOS testing is not supported on this machine.""" + if utils.get_platform() != "macos": + pytest.skip("this test can only run on macOS") + if utils.get_xcode_version() < (13, 0): + pytest.skip("this test only works with Xcode 13.0 or greater") + if detect_ci_provider() == CIProvider.cirrus_ci: + pytest.skip( + "iOS testing not currently supported on Cirrus CI due to a failure " + "to start the simulator." + ) + + +# iOS tests shouldn't be run in parallel, because they're dependent on calling +# Xcode, and starting a simulator. These are both multi-threaded operations, and +# it's easy to overload the CI machine if there are multiple test processes +# running multithreaded processes. Therefore, they're put in the serial group, +# which is guaranteed to run single-process. +@pytest.mark.serial +@pytest.mark.parametrize( + "build_config", + [ + # Default to the pip build frontend + {"CIBW_PLATFORM": "ios"}, + # Also check the build frontend + {"CIBW_PLATFORM": "ios", "CIBW_BUILD_FRONTEND": "build"}, + ], +) +def test_ios_platforms(tmp_path, build_config, monkeypatch, capfd): + skip_if_ios_testing_not_supported() + + # Create a temporary "bin" directory, symlink a tool that we know eixsts + # (/usr/bin/true) into that location under a name that should be unique, + # and add the temp bin directory to the PATH. + tools_dir = tmp_path / "bin" + tools_dir.mkdir() + tools_dir.joinpath("does-exist").symlink_to(shutil.which("true")) + + monkeypatch.setenv("PATH", str(tools_dir), prepend=os.pathsep) + + # Generate a test project that has an additional before-build step using the + # known-to-exist tool. + project_dir = tmp_path / "project" + setup_py_add = "import subprocess\nsubprocess.run('does-exist', check=True)\n" + basic_project = test_projects.new_c_project(setup_py_add=setup_py_add) + basic_project.files.update(basic_project_files) + basic_project.generate(project_dir) + + # Build and test the wheels. Mark the "does-exist" tool as a cross-build + # tool, and invoke it during a `before-build` step. It will also be invoked + # when `setup.py` is invoked. + # + # Tests are only executed on simulator. The test suite passes if it's + # running on the same architecture as the current platform. + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_BEFORE_BUILD": "does-exist", + "CIBW_BUILD": "cp313-*", + "CIBW_XBUILD_TOOLS": "does-exist", + "CIBW_TEST_SOURCES": "tests", + "CIBW_TEST_COMMAND": "python -m this && python -m unittest discover tests test_platform.py", + "CIBW_BUILD_VERBOSITY": "1", + **build_config, + }, + ) + + # The expected wheels were produced. + expected_wheels = utils.expected_wheels( + "spam", "0.1.0", platform="ios", python_abi_tags=["cp313-cp313"] + ) + assert set(actual_wheels) == set(expected_wheels) + + # The user was notified that the cross-build tool was found. + captured = capfd.readouterr() + assert "'does-exist' will be included in the cross-build environment" in captured.out + + # Make sure the first command ran + assert "Zen of Python" in captured.out + + +@pytest.mark.serial +def test_no_test_sources(tmp_path, capfd): + """Build will provide a helpful error if pytest is run and test-sources is not defined.""" + skip_if_ios_testing_not_supported() + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) + basic_project.generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_REQUIRES": "pytest", + "CIBW_TEST_COMMAND": "python -m pytest", + "CIBW_XBUILD_TOOLS": "", + }, + ) + + # The error message indicates the configuration issue. + captured = capfd.readouterr() + assert ( + "you must copy your test files to the testbed app by setting the `test-sources` option" + in captured.out + captured.err + ) + + +def test_ios_testing_with_placeholder(tmp_path, capfd): + """ + Tests with the {project} placeholder are not supported on iOS, because the test command + is run in the simulator. + """ + skip_if_ios_testing_not_supported() + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) + basic_project.generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_REQUIRES": "pytest", + "CIBW_TEST_COMMAND": "pytest {project}/tests", + "CIBW_XBUILD_TOOLS": "", + }, + ) + + # The error message indicates the configuration issue. + captured = capfd.readouterr() + assert "iOS tests cannot use placeholders" in captured.out + captured.err + + +@pytest.mark.serial +def test_ios_test_command_short_circuit(tmp_path, capfd): + skip_if_ios_testing_not_supported() + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) + basic_project.generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + # `python -m not_a_module` will fail, so `python -m this` should not be run. + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_XBUILD_TOOLS": "", + "CIBW_TEST_SOURCES": "tests", + "CIBW_TEST_COMMAND": "python -m not_a_module && python -m this", + "CIBW_BUILD_VERBOSITY": "1", + }, + ) + + captured = capfd.readouterr() + + assert "No module named not_a_module" in captured.out + captured.err + # assert that `python -m this` was not run + assert "Zen of Python" not in captured.out + captured.err + + +def test_missing_xbuild_tool(tmp_path, capfd): + """Build will fail if xbuild-tools references a non-existent tool.""" + skip_if_ios_testing_not_supported() + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) + basic_project.generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_COMMAND": "python -m tests", + "CIBW_XBUILD_TOOLS": "does-not-exist", + }, + ) + + # The error message indicates the problem tool. + captured = capfd.readouterr() + assert "Could not find a 'does-not-exist' executable on the path." in captured.err + + +def test_no_xbuild_tool_definition(tmp_path, capfd): + """Build will succeed with a warning if there is no xbuild-tools definition.""" + skip_if_ios_testing_not_supported() + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) + basic_project.generate(project_dir) + + # Build, but don't test the wheels; we're only checking that the right + # warning was raised. + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_SKIP": "*", + }, + ) + + # The expected wheels were produced. + expected_wheels = utils.expected_wheels( + "spam", + "0.1.0", + platform="ios", + python_abi_tags=["cp313-cp313"], + ) + assert set(actual_wheels) == set(expected_wheels) + + # The user was notified that there was no cross-build tool definition. + captured = capfd.readouterr() + assert "Your project configuration does not define any cross-build tools." in captured.err + + +def test_empty_xbuild_tool_definition(tmp_path, capfd): + """Build will succeed with no warning if there is an empty xbuild-tools definition.""" + skip_if_ios_testing_not_supported() + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files.update(basic_project_files) + basic_project.generate(project_dir) + + # Build, but don't test the wheels; we're only checking that a warning + # wasn't raised. + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_SKIP": "*", + "CIBW_XBUILD_TOOLS": "", + }, + ) + + expected_wheels = utils.expected_wheels( + "spam", "0.1.0", platform="ios", python_abi_tags=["cp313-cp313"] + ) + assert set(actual_wheels) == set(expected_wheels) + + # The warnings about cross-build notifications were silenced. + captured = capfd.readouterr() + assert "Your project configuration does not define any cross-build tools." not in captured.err + + +@pytest.mark.serial +def test_ios_test_command_without_python_dash_m(tmp_path, capfd): + """pytest should be able to run without python -m, but it should warn.""" + skip_if_ios_testing_not_supported() + + project_dir = tmp_path / "project" + + project = test_projects.new_c_project() + project.files["tests/__init__.py"] = "" + project.files["tests/test_spam.py"] = textwrap.dedent(""" + import spam + def test_spam(): + assert spam.filter("spam") == 0 + assert spam.filter("ham") != 0 + """) + project.generate(project_dir) + + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_BUILD": "cp313-*", + "CIBW_TEST_COMMAND": "pytest ./tests", + "CIBW_TEST_SOURCES": "tests", + "CIBW_TEST_REQUIRES": "pytest", + "CIBW_XBUILD_TOOLS": "", + }, + ) + + expected_wheels = utils.expected_wheels( + "spam", "0.1.0", platform="ios", python_abi_tags=["cp313-cp313"] + ) + assert set(actual_wheels) == set(expected_wheels) + + out, err = capfd.readouterr() + + assert "iOS tests configured with a test command which doesn't start with 'python -m'" in err + + +def test_ios_test_command_invalid(tmp_path, capfd): + """Test command should raise an error if it's clearly invalid.""" + skip_if_ios_testing_not_supported() + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.files["./my_test_script.sh"] = "echo hello" + basic_project.generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_PLATFORM": "ios", + "CIBW_TEST_COMMAND": "./my_test_script.sh", + "CIBW_TEST_SOURCES": "./my_test_script.sh", + "CIBW_XBUILD_TOOLS": "", + }, + ) + out, err = capfd.readouterr() + assert "iOS tests configured with a test command which doesn't start with 'python -m'" in err diff --git a/test/test_linux_python.py b/test/test_linux_python.py new file mode 100644 index 000000000..48a70c2e1 --- /dev/null +++ b/test/test_linux_python.py @@ -0,0 +1,42 @@ +import platform +import subprocess + +import pytest + +from . import test_projects, utils + + +def test_python_exist(tmp_path, capfd): + if utils.get_platform() != "linux": + pytest.skip("the test is only relevant to the linux build") + machine = platform.machine() + if machine not in ["x86_64", "i686"]: + pytest.skip( + "this test is currently only possible on x86_64/i686 due to availability of alternative images" + ) + + project_dir = tmp_path / "project" + basic_project = test_projects.new_c_project() + basic_project.generate(project_dir) + + image = f"quay.io/pypa/manylinux2010_{machine}:2022-08-05-4535177" + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_MANYLINUX_X86_64_IMAGE": image, + "CIBW_MANYLINUX_I686_IMAGE": image, + "CIBW_BUILD": "cp3{10,11}-manylinux*", + }, + ) + + captured = capfd.readouterr() + print("out", captured.out) + print("err", captured.err) + assert f" to build 'cp310-manylinux_{machine}'." not in captured.err + message = ( + "'/opt/python/cp311-cp311/bin/python' executable doesn't exist" + f" in image '{image}' to build 'cp311-manylinux_{machine}'." + ) + assert message in captured.err diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index 3f4272501..f799e55b0 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -1,37 +1,28 @@ import platform -import subprocess -from typing import Tuple import pytest from . import test_projects, utils basic_project = test_projects.new_c_project() +basic_project.files["tests/test_suite.py"] = r""" +import platform +print("running tests on " + platform.machine()) +""" + ALL_MACOS_WHEELS = { *utils.expected_wheels("spam", "0.1.0", machine_arch="x86_64"), - *utils.expected_wheels("spam", "0.1.0", machine_arch="arm64"), + *utils.expected_wheels("spam", "0.1.0", machine_arch="arm64", include_universal2=True), } - -def get_xcode_version() -> Tuple[int, int]: - output = subprocess.run( - ["xcodebuild", "-version"], - universal_newlines=True, - check=True, - stdout=subprocess.PIPE, - ).stdout - lines = output.splitlines() - _, version_str = lines[0].split() - - version_parts = version_str.split(".") - return (int(version_parts[0]), int(version_parts[1])) +DEPLOYMENT_TARGET_TOO_LOW_WARNING = "Bumping MACOSX_DEPLOYMENT_TARGET" def test_cross_compiled_build(tmp_path): - if utils.platform != "macos": + if utils.get_platform() != "macos": pytest.skip("this test is only relevant to macos") - if get_xcode_version() < (12, 2): + if utils.get_xcode_version() < (12, 2): pytest.skip("this test only works with Xcode 12.2 or greater") project_dir = tmp_path / "project" @@ -39,21 +30,31 @@ def test_cross_compiled_build(tmp_path): actual_wheels = utils.cibuildwheel_run( project_dir, - add_env={ - "CIBW_BUILD": "cp39-*", - "CIBW_ARCHS": "x86_64, universal2, arm64", - }, + add_env={"CIBW_ARCHS": "x86_64, universal2, arm64"}, + single_python=True, ) - - expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w] + python_tag = "cp{}{}".format(*utils.SINGLE_PYTHON_VERSION) + expected_wheels = [w for w in ALL_MACOS_WHEELS if python_tag in w] assert set(actual_wheels) == set(expected_wheels) @pytest.mark.parametrize("build_universal2", [False, True]) -def test_cross_compiled_test(tmp_path, capfd, build_universal2): - if utils.platform != "macos": +@pytest.mark.parametrize( + "test_config", + [ + { + "CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''', + }, + { + "CIBW_TEST_COMMAND": "python tests/test_suite.py", + "CIBW_TEST_SOURCES": "tests", + }, + ], +) +def test_cross_compiled_test(tmp_path, capfd, build_universal2, test_config): + if utils.get_platform() != "macos": pytest.skip("this test is only relevant to macos") - if get_xcode_version() < (12, 2): + if utils.get_xcode_version() < (12, 2): pytest.skip("this test only works with Xcode 12.2 or greater") project_dir = tmp_path / "project" @@ -62,45 +63,80 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2): actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ - "CIBW_BUILD": "cp39-*", - "CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''', + "CIBW_BUILD": "cp310-*" if build_universal2 else "*p310-*", "CIBW_ARCHS": "universal2" if build_universal2 else "x86_64 arm64", + "CIBW_BUILD_VERBOSITY": "3", + **test_config, }, ) captured = capfd.readouterr() - if platform.machine() == "x86_64": + assert DEPLOYMENT_TARGET_TOO_LOW_WARNING not in captured.err + + platform_machine = platform.machine() + if platform_machine == "x86_64": # ensure that tests were run on only x86_64 assert "running tests on x86_64" in captured.out assert "running tests on arm64" not in captured.out if build_universal2: assert ( - "While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested" + "While universal2 wheels can be built on x86_64, the arm64 part of the wheel cannot be tested" in captured.err ) else: assert ( "While arm64 wheels can be built on x86_64, they cannot be tested" in captured.err ) - elif platform.machine() == "arm64": + elif platform_machine == "arm64": # ensure that tests were run on both x86_64 and arm64 assert "running tests on x86_64" in captured.out assert "running tests on arm64" in captured.out + assert ( + "While universal2 wheels can be built on x86_64, the arm64 part of the wheel cannot be tested" + not in captured.err + ) + assert ( + "While arm64 wheels can be built on x86_64, they cannot be tested" not in captured.err + ) if build_universal2: - expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" in w] + expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp310" in w and "universal2" in w] else: - expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" not in w] + expected_wheels = [w for w in ALL_MACOS_WHEELS if "p310-" in w and "universal2" not in w] + if platform_machine == "x86_64": + expected_wheels = [w for w in expected_wheels if not ("pp310" in w and "arm64" in w)] assert set(actual_wheels) == set(expected_wheels) +def test_deployment_target_warning_is_firing(tmp_path, capfd): + # force the warning to check that we can detect it if it happens + if utils.get_platform() != "macos": + pytest.skip("this test is only relevant to macos") + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_ARCHS": "x86_64", + "MACOSX_DEPLOYMENT_TARGET": "10.8", + "CIBW_BUILD_VERBOSITY": "3", + }, + single_python=True, + ) + + captured = capfd.readouterr() + assert DEPLOYMENT_TARGET_TOO_LOW_WARNING in captured.err + + @pytest.mark.parametrize("skip_arm64_test", [False, True]) -def test_universal2_testing(tmp_path, capfd, skip_arm64_test): - if utils.platform != "macos": +def test_universal2_testing_on_x86_64(tmp_path, capfd, skip_arm64_test): + if utils.get_platform() != "macos": pytest.skip("this test is only relevant to macos") - if get_xcode_version() < (12, 2): + if utils.get_xcode_version() < (12, 2): pytest.skip("this test only works with Xcode 12.2 or greater") if platform.machine() != "x86_64": pytest.skip("this test only works on x86_64") @@ -111,11 +147,11 @@ def test_universal2_testing(tmp_path, capfd, skip_arm64_test): actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ - "CIBW_BUILD": "cp39-*", "CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''', "CIBW_ARCHS": "universal2", "CIBW_TEST_SKIP": "*_universal2:arm64" if skip_arm64_test else "", }, + single_python=True, ) captured = capfd.readouterr() @@ -124,12 +160,109 @@ def test_universal2_testing(tmp_path, capfd, skip_arm64_test): assert "running tests on x86_64" in captured.out assert "running tests on arm64" not in captured.out - warning_message = "While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested" + warning_message = "While universal2 wheels can be built on x86_64, the arm64 part of the wheel cannot be tested" if skip_arm64_test: assert warning_message not in captured.err else: assert warning_message in captured.err - expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" in w] + python_tag = "cp{}{}".format(*utils.SINGLE_PYTHON_VERSION) + expected_wheels = [w for w in ALL_MACOS_WHEELS if python_tag in w and "universal2" in w] + + assert set(actual_wheels) == set(expected_wheels) + + +def test_universal2_testing_on_arm64(build_frontend_env, tmp_path, capfd): + # cibuildwheel should test the universal2 wheel on both x86_64 and arm64, when run on arm64 + if utils.get_platform() != "macos": + pytest.skip("this test is only relevant to macos") + if platform.machine() != "arm64": + pytest.skip("this test only works on arm64") + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_ARCHS": "universal2", + # check that a native dependency is correctly installed, once per each testing arch + "CIBW_TEST_REQUIRES": "--only-binary :all: pillow>=10.3", # pillow>=10.3 provides wheels for macOS 10.10, not 10.9 + "CIBW_TEST_COMMAND": '''python -c "import PIL, platform; print(f'running tests on {platform.machine()} with pillow {PIL.__version__}')"''', + **build_frontend_env, + }, + single_python=True, + ) + + captured = capfd.readouterr() + assert "running tests on arm64 with pillow" in captured.out + assert "running tests on x86_64 with pillow" in captured.out + + python_tag = "cp{}{}".format(*utils.SINGLE_PYTHON_VERSION) + expected_wheels = [w for w in ALL_MACOS_WHEELS if python_tag in w and "universal2" in w] + assert set(actual_wheels) == set(expected_wheels) + + +def test_cp38_arm64_testing(tmp_path, capfd, request): + if utils.get_platform() != "macos": + pytest.skip("this test is only relevant to macos") + if utils.get_xcode_version() < (12, 2): + pytest.skip("this test only works with Xcode 12.2 or greater") + if platform.machine() != "arm64": + pytest.skip("this test only works on arm64") + if request.config.getoption("--run-cp38-universal2"): + pytest.skip("--run-cp38-universal2 option skips this test") + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_BUILD": "cp38-*", + "CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''', + "CIBW_ARCHS": "x86_64,universal2,arm64", + }, + ) + + captured = capfd.readouterr() + + assert "running tests on x86_64" in captured.out + assert "running tests on arm64" not in captured.out + + warning_message = "While cibuildwheel can build CPython 3.8 universal2/arm64 wheels, we cannot test the arm64 part of them" + assert warning_message in captured.err + + expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp38" in w] + + assert set(actual_wheels) == set(expected_wheels) + + +def test_cp38_arm64_testing_universal2_installer(tmp_path, capfd, request): + if not request.config.getoption("--run-cp38-universal2"): + pytest.skip("needs --run-cp38-universal2 option to run") + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_BUILD": "cp38-*", + "CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''', + "CIBW_ARCHS": "x86_64,universal2,arm64", + "MACOSX_DEPLOYMENT_TARGET": "11.0", + }, + ) + + captured = capfd.readouterr() + + assert "running tests on x86_64" in captured.out + assert "running tests on arm64" in captured.out + + warning_message = "While cibuildwheel can build CPython 3.8 universal2/arm64 wheels, we cannot test the arm64 part of them" + assert warning_message not in captured.err + + expected_wheels = [w.replace("10_9", "11_0") for w in ALL_MACOS_WHEELS if "cp38" in w] assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_manylinuxXXXX_only.py b/test/test_manylinuxXXXX_only.py index f7c533d49..483d40f99 100644 --- a/test/test_manylinuxXXXX_only.py +++ b/test/test_manylinuxXXXX_only.py @@ -13,27 +13,39 @@ #include #include #include + #include #if !defined(__GLIBC_PREREQ) #error "Must run on a glibc linux environment" #endif - #if !__GLIBC_PREREQ(2, 5) /* manylinux1 is glibc 2.5 */ + #if !__GLIBC_PREREQ(2, 17) /* manylinux2014 is glibc 2.17 */ #error "Must run on a glibc >= 2.5 linux environment" #endif + + #if __GLIBC_PREREQ(2, 28) + #include + #endif """ ), spam_c_function_add=textwrap.dedent( r""" - #if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 24) + #if __GLIBC_PREREQ(2, 34) + // pthread_mutexattr_init was moved to libc.so.6 in manylinux_2_34+ + pthread_mutexattr_t attr; + sts = pthread_mutexattr_init(&attr); + if (sts == 0) { + pthread_mutexattr_destroy(&attr); + } + #elif __GLIBC_PREREQ(2, 28) + // thrd_equal & thrd_current are only available in manylinux_2_28+ + sts = thrd_equal(thrd_current(), thrd_current()) ? 0 : 1; + #elif __GLIBC_PREREQ(2, 24) // nextupf is only available in manylinux_2_24+ sts = (int)nextupf(0.0F); - #elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 17) /* manylinux2014 is glibc 2.17 */ + #elif __GLIBC_PREREQ(2, 17) /* manylinux2014 is glibc 2.17 */ // secure_getenv is only available in manylinux2014+ sts = (int)(intptr_t)secure_getenv("NON_EXISTING_ENV_VARIABLE"); - #elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10) /* manylinux2010 is glibc 2.12 */ - // malloc_info is only available on manylinux2010+ - sts = malloc_info(0, stdout); #endif """ ), @@ -41,20 +53,25 @@ @pytest.mark.parametrize( - "manylinux_image", ["manylinux1", "manylinux2010", "manylinux2014", "manylinux_2_24"] + "manylinux_image", + [ + "manylinux2014", + "manylinux_2_28", + "manylinux_2_34", + ], ) +@pytest.mark.usefixtures("docker_cleanup") def test(manylinux_image, tmp_path): - if utils.platform != "linux": - pytest.skip("the docker test is only relevant to the linux build") - elif platform.machine() not in ["x86_64", "i686"]: - if manylinux_image in ["manylinux1", "manylinux2010"]: - pytest.skip("manylinux1 and 2010 doesn't exist for non-x86 architectures") + if utils.get_platform() != "linux": + pytest.skip("the container image test is only relevant to the linux build") + elif manylinux_image in {"manylinux_2_28", "manylinux_2_34"} and platform.machine() == "i686": + pytest.skip(f"{manylinux_image} doesn't exist for i686 architecture") project_dir = tmp_path / "project" project_with_manylinux_symbols.generate(project_dir) # build the wheels - # CFLAGS environment variable is necessary to fail on 'malloc_info' (on manylinux1) during compilation/linking, + # CFLAGS environment variable is necessary to fail at build time, # rather than when dynamically loading the Python add_env = { "CIBW_BUILD": "*-manylinux*", @@ -68,16 +85,17 @@ def test(manylinux_image, tmp_path): "CIBW_MANYLINUX_PYPY_AARCH64_IMAGE": manylinux_image, "CIBW_MANYLINUX_PYPY_I686_IMAGE": manylinux_image, } - if manylinux_image in {"manylinux1"}: - # We don't have a manylinux1 image for PyPy & CPython 3.10 and above - add_env["CIBW_SKIP"] = "pp* cp31*" + if manylinux_image in {"manylinux_2_28", "manylinux_2_34"} and platform.machine() == "x86_64": + # We don't have a manylinux_2_28+ image for i686 + add_env["CIBW_ARCHS"] = "x86_64" + if platform.machine() == "aarch64": + # We just have a manylinux_2_31 image for armv7l + add_env["CIBW_ARCHS"] = "aarch64" actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env) platform_tag_map = { - "manylinux1": ["manylinux_2_5", "manylinux1"], - "manylinux2010": ["manylinux_2_12", "manylinux2010"], - "manylinux2014": ["manylinux_2_17", "manylinux2014"], + "manylinux2014": ["manylinux2014", "manylinux_2_17"], } expected_wheels = utils.expected_wheels( "spam", @@ -85,7 +103,13 @@ def test(manylinux_image, tmp_path): manylinux_versions=platform_tag_map.get(manylinux_image, [manylinux_image]), musllinux_versions=[], ) - if manylinux_image in {"manylinux1"}: - # remove PyPy & CPython 3.10 and above - expected_wheels = [w for w in expected_wheels if "-pp" not in w and "-cp31" not in w] + + if manylinux_image in {"manylinux_2_28", "manylinux_2_34"} and platform.machine() == "x86_64": + # We don't have a manylinux_2_28+ image for i686 + expected_wheels = [w for w in expected_wheels if "i686" not in w] + + if platform.machine() == "aarch64": + # We just have a manylinux_2_31 image for armv7l + expected_wheels = [w for w in expected_wheels if "armv7l" not in w] + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_musllinux_X_Y_only.py b/test/test_musllinux_X_Y_only.py new file mode 100644 index 000000000..5e2f2e246 --- /dev/null +++ b/test/test_musllinux_X_Y_only.py @@ -0,0 +1,56 @@ +import textwrap + +import pytest + +from . import test_projects, utils + +project_with_manylinux_symbols = test_projects.new_c_project( + spam_c_top_level_add=textwrap.dedent( + r""" + #include + + #if defined(__GLIBC_PREREQ) + #error "Must not run on a glibc linux environment" + #endif + """ + ), + spam_c_function_add=textwrap.dedent( + r""" + sts = 0; + """ + ), +) + + +@pytest.mark.parametrize( + "musllinux_image", + ["musllinux_1_2"], +) +@pytest.mark.usefixtures("docker_cleanup") +def test(musllinux_image, tmp_path): + if utils.get_platform() != "linux": + pytest.skip("the container image test is only relevant to the linux build") + + project_dir = tmp_path / "project" + project_with_manylinux_symbols.generate(project_dir) + + # build the wheels + add_env = { + "CIBW_SKIP": "*-manylinux*", + "CIBW_MUSLLINUX_X86_64_IMAGE": musllinux_image, + "CIBW_MUSLLINUX_I686_IMAGE": musllinux_image, + "CIBW_MUSLLINUX_AARCH64_IMAGE": musllinux_image, + "CIBW_MUSLLINUX_PPC64LE_IMAGE": musllinux_image, + "CIBW_MUSLLINUX_S390X_IMAGE": musllinux_image, + "CIBW_MUSLLINUX_ARMV7L_IMAGE": musllinux_image, + } + + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env, single_python=True) + expected_wheels = utils.expected_wheels( + "spam", + "0.1.0", + manylinux_versions=[], + musllinux_versions=[musllinux_image], + single_python=True, + ) + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_pep518.py b/test/test_pep518.py index 128834280..702e7f185 100644 --- a/test/test_pep518.py +++ b/test/test_pep518.py @@ -6,8 +6,8 @@ setup_py_add=textwrap.dedent( """ # Will fail if PEP 518 does work - import requests - assert requests.__version__ == "2.23.0", "Requests found but wrong version ({0})".format(requests.__version__) + import jmespath + assert jmespath.__version__ == "0.10.0", "'jmespath' found but wrong version ({0})".format(jmespath.__version__) # Just making sure environment is still set import os @@ -17,15 +17,12 @@ ) ) -basic_project.files[ - "pyproject.toml" -] = """ +basic_project.files["pyproject.toml"] = """ [build-system] requires = [ "setuptools >= 42", "setuptools_scm[toml]>=4.1.2", - "wheel", - "requests==2.23.0" + "jmespath==0.10.0" ] build-backend = "setuptools.build_meta" @@ -33,15 +30,29 @@ def test_pep518(tmp_path, build_frontend_env): - project_dir = tmp_path / "project" basic_project.generate(project_dir) + # GraalPy fails to discover its standard library when a venv is created + # from a virtualenv seeded executable. See + # https://github.com/oracle/graalpython/issues/491 and remove this once + # fixed upstream. + if build_frontend_env["CIBW_BUILD_FRONTEND"] == "build" and utils.get_platform() == "windows": + build_frontend_env["CIBW_SKIP"] = "gp*" + # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_frontend_env) # check that the expected wheels are produced expected_wheels = utils.expected_wheels("spam", "0.1.0") + + # GraalPy fails to discover its standard library when a venv is created + # from a virtualenv seeded executable. See + # https://github.com/oracle/graalpython/issues/491 and remove this once + # fixed upstream. + if build_frontend_env["CIBW_BUILD_FRONTEND"] == "build" and utils.get_platform() == "windows": + expected_wheels = [w for w in expected_wheels if "graalpy" not in w] + assert set(actual_wheels) == set(expected_wheels) # These checks ensure an extra file is not created when using custom @@ -49,12 +60,19 @@ def test_pep518(tmp_path, build_frontend_env): assert not (project_dir / "42").exists() assert not (project_dir / "4.1.2").exists() - # pypa/build creates a "build" folder & a "*.egg-info" folder for the wheel being built, - # this should be harmless so remove them + # pypa/build creates a "build" folder & a "*.egg-info" folder for the + # wheel being built, this should be harmless so remove them. pyodide-build + # creates a ".pyodide_build" folder, but this is gitignored with a + # .gitignore file inside. contents = [ item for item in project_dir.iterdir() - if item.name != "build" and not item.name.endswith(".egg-info") + if item.name != "build" + and not item.name.endswith(".egg-info") + and item.name != ".pyodide_build" ] + print("Project contents after build:") + print("\n".join(f" {f}" for f in contents)) + assert len(contents) == len(basic_project.files) diff --git a/test/test_projects/__main__.py b/test/test_projects/__main__.py index a1e77c432..2809b17b1 100644 --- a/test/test_projects/__main__.py +++ b/test/test_projects/__main__.py @@ -1,13 +1,17 @@ +import argparse +import functools import importlib import subprocess import sys import tempfile -from argparse import ArgumentParser from pathlib import Path -def main(): - parser = ArgumentParser( +def main() -> None: + make_parser = functools.partial(argparse.ArgumentParser, allow_abbrev=False) + if sys.version_info >= (3, 14): + make_parser = functools.partial(make_parser, color=True, suggest_on_error=True) + parser = make_parser( prog="python -m test.test_projects", description="Generate a test project to check it out" ) parser.add_argument( diff --git a/test/test_projects/base.py b/test/test_projects/base.py index 66d0006ed..dd8f2ce21 100644 --- a/test/test_projects/base.py +++ b/test/test_projects/base.py @@ -1,10 +1,10 @@ from pathlib import Path -from typing import Any, Dict, Union +from typing import Any, Self import jinja2 -FilesDict = Dict[str, Union[str, jinja2.Template]] -TemplateContext = Dict[str, Any] +FilesDict = dict[str, str | jinja2.Template] +TemplateContext = dict[str, Any] class TestProject: @@ -20,23 +20,23 @@ class TestProject: files: FilesDict template_context: TemplateContext - def __init__(self): + def __init__(self) -> None: self.files = {} self.template_context = {} - def generate(self, path: Path): + def generate(self, path: Path) -> None: for filename, content in self.files.items(): file_path = path / filename file_path.parent.mkdir(parents=True, exist_ok=True) with file_path.open("w", encoding="utf8") as f: if isinstance(content, jinja2.Template): - content = content.render(self.template_context) + content = content.render(self.template_context) # noqa: PLW2901 f.write(content) - def copy(self): - other = TestProject() + def copy(self) -> Self: + other = self.__class__() other.files = self.files.copy() other.template_context = self.template_context.copy() return other diff --git a/test/test_projects/c.py b/test/test_projects/c.py index 41b729586..55d759875 100644 --- a/test/test_projects/c.py +++ b/test/test_projects/c.py @@ -8,15 +8,16 @@ {{ spam_c_top_level_add }} static PyObject * -spam_system(PyObject *self, PyObject *args) +spam_filter(PyObject *self, PyObject *args) { - const char *command; + const char *content; int sts; - if (!PyArg_ParseTuple(args, "s", &command)) + if (!PyArg_ParseTuple(args, "s", &content)) return NULL; - sts = system(command); + // Spam should not be allowed through the filter. + sts = strcmp(content, "spam"); {{ spam_c_function_add | indent(4) }} @@ -25,7 +26,7 @@ /* Module initialization */ static PyMethodDef module_methods[] = { - {"system", (PyCFunction)spam_system, METH_VARARGS, + {"filter", (PyCFunction)spam_filter, METH_VARARGS, "Execute a shell command."}, {NULL} /* Sentinel */ }; @@ -40,20 +41,26 @@ """ SETUP_PY_TEMPLATE = r""" +import os import sys + from setuptools import setup, Extension {{ setup_py_add }} libraries = [] -if sys.platform.startswith('linux'): +# Emscripten fails if you pass -lc... +# See: https://github.com/emscripten-core/emscripten/issues/16680 +if sys.platform.startswith('linux') and "emscripten" not in os.environ.get("_PYTHON_HOST_PLATFORM", ""): libraries.extend(['m', 'c']) + setup( ext_modules=[Extension( 'spam', sources=['spam.c'], libraries=libraries, + {{ setup_py_extension_args_add | indent(8) }} )], {{ setup_py_setup_args_add | indent(4) }} ) @@ -70,12 +77,13 @@ def new_c_project( *, - spam_c_top_level_add="", - spam_c_function_add="", - setup_py_add="", - setup_py_setup_args_add="", - setup_cfg_add="", -): + spam_c_top_level_add: str = "", + spam_c_function_add: str = "", + setup_py_add: str = "", + setup_py_extension_args_add: str = "", + setup_py_setup_args_add: str = "", + setup_cfg_add: str = "", +) -> TestProject: project = TestProject() project.files.update( @@ -91,6 +99,7 @@ def new_c_project( "spam_c_top_level_add": spam_c_top_level_add, "spam_c_function_add": spam_c_function_add, "setup_py_add": setup_py_add, + "setup_py_extension_args_add": setup_py_extension_args_add, "setup_py_setup_args_add": setup_py_setup_args_add, "setup_cfg_add": setup_cfg_add, } diff --git a/test/test_pure_wheel.py b/test/test_pure_wheel.py index 2ba33711f..d309d8a9a 100644 --- a/test/test_pure_wheel.py +++ b/test/test_pure_wheel.py @@ -1,14 +1,13 @@ import subprocess -from test import test_projects import pytest +from test import test_projects + from . import utils pure_python_project = test_projects.TestProject() -pure_python_project.files[ - "setup.py" -] = """ +pure_python_project.files["setup.py"] = """ from setuptools import Extension, setup setup( @@ -18,9 +17,7 @@ ) """ -pure_python_project.files[ - "spam.py" -] = """ +pure_python_project.files["spam.py"] = """ def a_function(): pass """ @@ -32,11 +29,11 @@ def test(tmp_path, capfd): project_dir = tmp_path / "project" pure_python_project.generate(project_dir) - with pytest.raises(subprocess.CalledProcessError): - actual_wheels = utils.cibuildwheel_run(project_dir) - print("produced wheels:", actual_wheels) + with pytest.raises(subprocess.CalledProcessError) as exc_info: + print("produced wheels:", utils.cibuildwheel_run(project_dir)) captured = capfd.readouterr() print("out", captured.out) print("err", captured.err) + assert exc_info.value.returncode == 5 assert "Build failed because a pure Python wheel was generated" in captured.err diff --git a/test/test_pyodide.py b/test/test_pyodide.py new file mode 100644 index 000000000..9240e8266 --- /dev/null +++ b/test/test_pyodide.py @@ -0,0 +1,144 @@ +import contextlib +import subprocess +import sys +import textwrap + +import pytest + +from cibuildwheel.util.file import CIBW_CACHE_PATH + +from . import test_projects, utils + +basic_project = test_projects.new_c_project() +basic_project.files["check_node.py"] = r""" +import sys +import shutil +from pathlib import Path +from pyodide.code import run_js + + +def check_node(): + # cibuildwheel adds a pinned node version to the PATH + # check it's in the PATH then, check it's the one that runs Pyodide + cibw_cache_path = Path(sys.argv[1]).resolve(strict=True) + # find the node executable in PATH + node = shutil.which("node") + assert node is not None, "node is None" + node_path = Path(node).resolve(strict=True) + # it shall be in cibuildwheel cache + assert cibw_cache_path in node_path.parents, f"{cibw_cache_path} not a parent of {node_path}" + # find the path to the node executable that runs pyodide + node_js = run_js("globalThis.process.execPath") + assert node_js is not None, "node_js is None" + node_js_path = Path(node_js).resolve(strict=True) + # it shall be the one pinned by cibuildwheel + assert node_js_path == node_path, f"{node_js_path} != {node_path}" + + +if __name__ == "__main__": + check_node() +""" + + +@pytest.mark.parametrize("use_pyproject_toml", [True, False]) +def test_pyodide_build(tmp_path, use_pyproject_toml): + if sys.platform == "win32": + pytest.skip("pyodide-build doesn't work correctly on Windows") + + if use_pyproject_toml: + basic_project.files["pyproject.toml"] = textwrap.dedent( + """ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + """ + ) + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + # check for node in 1 case only to reduce CI load + add_env = {"CIBW_ENABLE": "pyodide-prerelease"} + if use_pyproject_toml: + add_env["CIBW_TEST_COMMAND"] = f"python {{project}}/check_node.py {CIBW_CACHE_PATH}" + + # build the wheels + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_args=["--platform", "pyodide"], + add_env=add_env, + ) + + # check that the expected wheels are produced + expected_wheels = [ + "spam-0.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl", + "spam-0.1.0-cp313-cp313-pyodide_2025_0_wasm32.whl", + ] + + print("actual_wheels", actual_wheels) + print("expected_wheels", expected_wheels) + + assert set(actual_wheels) == set(expected_wheels) + + +def test_pyodide_version_incompatible(tmp_path, capfd): + if sys.platform == "win32": + pytest.skip("pyodide-build doesn't work correctly on Windows") + + basic_project.generate(tmp_path) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + tmp_path, + add_args=["--platform", "pyodide"], + add_env={ + "CIBW_DEPENDENCY_VERSIONS": "packages: pyodide-build==0.29.3", + "CIBW_PYODIDE_VERSION": "0.26.0a6", + }, + ) + + out, err = capfd.readouterr() + + assert "is not compatible with the pyodide-build version" in err + + +@pytest.mark.parametrize("expect_failure", [True, False]) +def test_pyodide_build_and_test(tmp_path, expect_failure): + if sys.platform == "win32": + pytest.skip("pyodide-build doesn't work correctly on Windows") + + if expect_failure: + basic_project.files["test/spam_test.py"] = textwrap.dedent(r""" + def test_filter(): + assert 0 == 1 + """) + else: + basic_project.files["test/spam_test.py"] = textwrap.dedent(r""" + import spam + def test_filter(): + assert spam.filter("spam") == 0 + """) + basic_project.generate(tmp_path) + + context = ( + pytest.raises(subprocess.CalledProcessError) if expect_failure else contextlib.nullcontext() + ) + with context: + # build the wheels + actual_wheels = utils.cibuildwheel_run( + tmp_path, + add_args=["--platform", "pyodide"], + add_env={ + "CIBW_TEST_REQUIRES": "pytest", + "CIBW_TEST_COMMAND": "python -m pytest {project}", + "CIBW_ENABLE": "pyodide-prerelease", + }, + ) + # check that the expected wheels are produced + expected_wheels = [ + "spam-0.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl", + "spam-0.1.0-cp313-cp313-pyodide_2025_0_wasm32.whl", + ] + print("actual_wheels", actual_wheels) + print("expected_wheels", expected_wheels) + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_ssl.py b/test/test_ssl.py index 5283a59d4..0ea958ba3 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -1,3 +1,4 @@ +import socket import textwrap from . import test_projects, utils @@ -10,9 +11,9 @@ from urllib.request import urlopen context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - data = urlopen("https://www.nist.gov/", context=context) - data = urlopen("https://raw.githubusercontent.com/pypa/cibuildwheel/main/CI.md", context=context) - data = urlopen("https://raw.githubusercontent.com/pypa/cibuildwheel/main/CI.md") + # badssl.com is a HTTPS test server that can be used to test SSL connections + data = urlopen("https://tls-v1-2.badssl.com/", context=context) + data = urlopen("https://tls-v1-2.badssl.com/") """ ) ) @@ -24,6 +25,9 @@ def test(tmp_path): project_dir = tmp_path / "project" project_with_ssl_tests.generate(project_dir) + # warm up connection + socket.getaddrinfo("tls-v1-2.badssl.com", 443) + actual_wheels = utils.cibuildwheel_run(project_dir) expected_wheels = utils.expected_wheels("spam", "0.1.0") diff --git a/test/test_subdir_package.py b/test/test_subdir_package.py index 9241ae30f..2675ef2ed 100644 --- a/test/test_subdir_package.py +++ b/test/test_subdir_package.py @@ -12,9 +12,7 @@ subdir_package_project.template_context["spam_c_top_level_add"] = "" subdir_package_project.template_context["spam_c_function_add"] = "" -subdir_package_project.files[ - "src/spam/setup.py" -] = r""" +subdir_package_project.files["src/spam/setup.py"] = r""" from setuptools import Extension, setup setup( @@ -24,15 +22,11 @@ ) """ -subdir_package_project.files[ - "src/spam/test/run_tests.py" -] = r""" +subdir_package_project.files["src/spam/test/run_tests.py"] = r""" print('run_tests.py executed!') """ -subdir_package_project.files[ - "bin/before_build.py" -] = r""" +subdir_package_project.files["bin/before_build.py"] = r""" print('before_build.py executed!') """ @@ -49,13 +43,12 @@ def test(capfd, tmp_path): add_env={ "CIBW_BEFORE_BUILD": "python {project}/bin/before_build.py", "CIBW_TEST_COMMAND": "python {package}/test/run_tests.py", - # this shouldn't depend on the version of python, so build only CPython 3.6 - "CIBW_BUILD": "cp36-*", }, + single_python=True, ) # check that the expected wheels are produced - expected_wheels = [w for w in utils.expected_wheels("spam", "0.1.0") if "cp36" in w] + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) assert set(actual_wheels) == set(expected_wheels) captured = capfd.readouterr() diff --git a/test/test_testing.py b/test/test_testing.py index ef41f35a5..92de0e87e 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -1,6 +1,7 @@ -import os +import inspect import subprocess import textwrap +from pathlib import Path import pytest @@ -15,9 +16,7 @@ ) ) -project_with_a_test.files[ - "test/spam_test.py" -] = r''' +project_with_a_test.files["test/spam_test.py"] = r''' import os import platform import sys @@ -44,9 +43,9 @@ def path_contains(parent, child): class TestSpam(TestCase): - def test_system(self): - self.assertEqual(0, spam.system('python -c "exit(0)"')) - self.assertNotEqual(0, spam.system('python -c "exit(1)"')) + def test_filter(self): + self.assertEqual(0, spam.filter("spam")) + self.assertNotEqual(0, spam.filter("ham")) def test_virtualenv(self): # sys.prefix is different from sys.base_prefix when running a virtualenv @@ -57,6 +56,7 @@ def test_virtualenv(self): self.assertTrue(path_contains(sys.prefix, sys.executable)) self.assertTrue(path_contains(sys.prefix, spam.__file__)) + self.assertIn("VIRTUAL_ENV", os.environ) def test_uname(self): if platform.system() == "Windows": @@ -65,7 +65,7 @@ def test_uname(self): # See #336 for more info. bits = struct.calcsize("P") * 8 if bits == 32: - self.assertEqual(platform.machine(), "i686") + self.assertIn(platform.machine(), ["i686", "armv7l","armv8l", "wasm32"]) ''' @@ -80,8 +80,10 @@ def test(tmp_path): "CIBW_TEST_REQUIRES": "pytest", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": "false || pytest {project}/test", - "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", + # pytest fails on GraalPy 24.2.0 on Windows so we skip it there + # until https://github.com/oracle/graalpython/issues/490 is fixed + "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || where graalpy || pytest {project}/test", }, ) @@ -101,20 +103,55 @@ def test_extras_require(tmp_path): "CIBW_TEST_EXTRAS": "test", # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. - "CIBW_TEST_COMMAND": "false || pytest {project}/test", - "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test", + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", + # pytest fails on GraalPy 24.2.0 on Windows so we skip it there + # until https://github.com/oracle/graalpython/issues/490 is fixed + "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || where graalpy || pytest {project}/test", }, + single_python=True, ) # also check that we got the right wheels - expected_wheels = utils.expected_wheels("spam", "0.1.0") + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) + assert set(actual_wheels) == set(expected_wheels) + + +def test_dependency_groups(tmp_path): + group_project = project_with_a_test.copy() + group_project.files["pyproject.toml"] = inspect.cleandoc(""" + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + + [dependency-groups] + dev = ["pytest"] + """) + + project_dir = tmp_path / "project" + group_project.generate(project_dir) + + # build and test the wheels + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_TEST_GROUPS": "dev", + # the 'false ||' bit is to ensure this command runs in a shell on + # mac/linux. + "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test", + # pytest fails on GraalPy 24.2.0 on Windows so we skip it there + # until https://github.com/oracle/graalpython/issues/490 is fixed + "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || where graalpy || pytest {project}/test", + }, + single_python=True, + ) + + # also check that we got the right wheels + expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True) assert set(actual_wheels) == set(expected_wheels) project_with_a_failing_test = test_projects.new_c_project() -project_with_a_failing_test.files[ - "test/spam_test.py" -] = r""" +project_with_a_failing_test.files["test/spam_test.py"] = r""" from unittest import TestCase class TestSpam(TestCase): @@ -134,13 +171,89 @@ def test_failing_test(tmp_path): project_dir, output_dir=output_dir, add_env={ - "CIBW_TEST_REQUIRES": "nose", - "CIBW_TEST_COMMAND": "nosetests {project}/test", - # manylinux1 has a version of bash that's been shown to have - # problems with this, so let's check that. - "CIBW_MANYLINUX_I686_IMAGE": "manylinux1", - "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux1", + "CIBW_TEST_REQUIRES": "pytest", + "CIBW_TEST_COMMAND": f"{utils.invoke_pytest()} {{project}}/test", + # CPython 3.8 when running on macOS arm64 is unusual. The build + # always runs in x86_64, so the arm64 tests are not run. See + # #1169 for reasons why. That means the build succeeds, which + # we don't want. So we skip that build. + "CIBW_SKIP": "cp38-macosx_arm64", }, ) - assert len(os.listdir(output_dir)) == 0 + assert len(list(output_dir.iterdir())) == 0 + + +@pytest.mark.parametrize("test_runner", ["pytest", "unittest"]) +def test_bare_pytest_invocation( + tmp_path: Path, capfd: pytest.CaptureFixture[str], test_runner: str +) -> None: + """ + Check that if a user runs pytest in the the test cwd without setting + test-sources, it raises a helpful error + """ + project_dir = tmp_path / "project" + project_with_a_test.generate(project_dir) + output_dir = tmp_path / "output" + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + project_dir, + output_dir=output_dir, + add_env={ + "CIBW_TEST_REQUIRES": "pytest" if test_runner == "pytest" else "", + "CIBW_TEST_COMMAND": ( + "python -m pytest" if test_runner == "pytest" else "python -m unittest" + ), + # Skip CPython 3.8 on macOS arm64, see comment above in + # 'test_failing_test' + "CIBW_SKIP": "cp38-macosx_arm64", + }, + ) + + assert len(list(output_dir.iterdir())) == 0 + + captured = capfd.readouterr() + + assert ( + "Please specify a path to your tests when invoking pytest using the {project} placeholder" + in captured.out + captured.err + ) + + +def test_test_sources(tmp_path): + project_dir = tmp_path / "project" + project_with_a_test.generate(project_dir) + + # build and test the wheels in the test cwd, after copying in the test sources. + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_TEST_REQUIRES": "pytest", + # pytest fails on GraalPy 24.2.0 on Windows so we skip it there + # until https://github.com/oracle/graalpython/issues/490 is fixed + "CIBW_TEST_COMMAND_WINDOWS": "where graalpy || pytest", + "CIBW_TEST_COMMAND": utils.invoke_pytest(), + "CIBW_TEST_SOURCES": "test", + }, + ) + + # also check that we got the right wheels + expected_wheels = utils.expected_wheels("spam", "0.1.0") + assert set(actual_wheels) == set(expected_wheels) + + +def test_test_environment(tmp_path): + project_dir = tmp_path / "project" + test_projects.new_c_project().generate(project_dir) + + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_TEST_ENVIRONMENT": "MYVAR=somevalue PYTHONSAFEPATH=1", + "CIBW_TEST_COMMAND": "python -c \"import os; assert os.environ.get('MYVAR') == 'somevalue'; assert os.environ.get('PYTHONSAFEPATH') == '1'\"", + }, + ) + # also check that we got the right wheels + expected_wheels = utils.expected_wheels("spam", "0.1.0") + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_troubleshooting.py b/test/test_troubleshooting.py index 7039c2687..1b8e8a37f 100644 --- a/test/test_troubleshooting.py +++ b/test/test_troubleshooting.py @@ -15,7 +15,7 @@ def test_failed_build_with_so_files(tmp_path, capfd, build_frontend_env, project if project_contains_so_files: project.files["libnothing.so"] = "" - if utils.platform != "linux": + if utils.get_platform() != "linux": pytest.skip("this test is only relevant to the linux build") project_dir = tmp_path / "project" @@ -36,7 +36,7 @@ def test_failed_build_with_so_files(tmp_path, capfd, build_frontend_env, project @pytest.mark.parametrize("project_contains_so_files", [False, True]) def test_failed_repair_with_so_files(tmp_path, capfd, project_contains_so_files): - if utils.platform != "linux": + if utils.get_platform() != "linux": pytest.skip("this test is only relevant to the linux build") project = new_c_project() diff --git a/test/test_wheel_tag.py b/test/test_wheel_tag.py index a026b2d77..09c99ccae 100644 --- a/test/test_wheel_tag.py +++ b/test/test_wheel_tag.py @@ -6,7 +6,7 @@ def test_wheel_tag_is_correct_when_using_macosx_deployment_target(tmp_path): - if utils.platform != "macos": + if utils.get_platform() != "macos": pytest.skip("This test is only relevant to MACOSX_DEPLOYMENT_TARGET") project_dir = tmp_path / "project" @@ -16,17 +16,14 @@ def test_wheel_tag_is_correct_when_using_macosx_deployment_target(tmp_path): deployment_target = "10.11" actual_wheels = utils.cibuildwheel_run( project_dir, - add_env={ - "CIBW_BUILD": "cp39-*", - "MACOSX_DEPLOYMENT_TARGET": deployment_target, - }, + add_env={"MACOSX_DEPLOYMENT_TARGET": deployment_target}, + single_python=True, ) # check that the expected wheels are produced expected_wheels = utils.expected_wheels( - "spam", "0.1.0", macosx_deployment_target=deployment_target + "spam", "0.1.0", macosx_deployment_target=deployment_target, single_python=True ) - expected_wheels = [w for w in expected_wheels if "cp39" in w] print("actual_wheels", actual_wheels) print("expected_wheels", expected_wheels) diff --git a/test/test_windows.py b/test/test_windows.py new file mode 100644 index 000000000..073669ba7 --- /dev/null +++ b/test/test_windows.py @@ -0,0 +1,73 @@ +import os +import subprocess +import textwrap +from pathlib import Path + +import pytest + +from . import test_projects, utils + +basic_project = test_projects.new_c_project() + + +def skip_if_no_msvc(arm64: bool = False) -> None: + programfiles = os.getenv("PROGRAMFILES(X86)", "") or os.getenv("PROGRAMFILES", "") + if not programfiles: + pytest.skip("Requires %PROGRAMFILES(X86)% variable to be set") + + vswhere = Path(programfiles, "Microsoft Visual Studio", "Installer", "vswhere.exe") + if not vswhere.is_file(): + pytest.skip("Requires Visual Studio installation") + + require = "Microsoft.VisualStudio.Component.VC.Tools.x86.x64" + if arm64: + require = "Microsoft.VisualStudio.Component.VC.Tools.ARM64" + + if not subprocess.check_output( + [ + vswhere, + "-latest", + "-prerelease", + "-property", + "installationPath", + "-requires", + require, + ] + ): + pytest.skip("Requires ARM64 compiler to be installed") + + +@pytest.mark.parametrize("use_pyproject_toml", [True, False]) +def test_wheel_tag_is_correct_when_using_windows_cross_compile(tmp_path, use_pyproject_toml): + if utils.get_platform() != "windows": + pytest.skip("This test is only relevant to Windows") + + skip_if_no_msvc(arm64=True) + + if use_pyproject_toml: + basic_project.files["pyproject.toml"] = textwrap.dedent( + """ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + """ + ) + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + # build the wheels + actual_wheels = utils.cibuildwheel_run( + project_dir, + add_args=["--archs", "ARM64"], + single_python=True, + ) + + # check that the expected wheels are produced + tag = "cp{}{}".format(*utils.SINGLE_PYTHON_VERSION) + expected_wheels = [f"spam-0.1.0-{tag}-{tag}-win_arm64.whl"] + + print("actual_wheels", actual_wheels) + print("expected_wheels", expected_wheels) + + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/utils.py b/test/utils.py index 987159543..782934bdf 100644 --- a/test/utils.py +++ b/test/utils.py @@ -8,34 +8,72 @@ import platform as pm import subprocess import sys +from collections.abc import Generator, Mapping, Sequence +from pathlib import Path from tempfile import TemporaryDirectory +from typing import Any, Final + +import pytest + +from cibuildwheel.architecture import Architecture +from cibuildwheel.ci import CIProvider, detect_ci_provider +from cibuildwheel.selector import EnableGroup +from cibuildwheel.util.file import CIBW_CACHE_PATH + +EMULATED_ARCHS: Final[list[str]] = sorted( + arch.value for arch in (Architecture.all_archs("linux") - Architecture.auto_archs("linux")) +) +PYPY_ARCHS = ["x86_64", "i686", "AMD64", "aarch64", "arm64"] +GRAALPY_ARCHS = ["x86_64", "AMD64", "aarch64", "arm64"] + +SINGLE_PYTHON_VERSION: Final[tuple[int, int]] = (3, 12) + +_AARCH64_CAN_RUN_ARMV7: Final[bool] = Architecture.aarch64.value not in EMULATED_ARCHS and { + None: Architecture.armv7l.value not in EMULATED_ARCHS, + CIProvider.travis_ci: False, + CIProvider.cirrus_ci: False, +}.get(detect_ci_provider(), True) + + +def get_platform() -> str: + """Return the current platform as determined by CIBW_PLATFORM or sys.platform.""" + platform = os.environ.get("CIBW_PLATFORM", "") + if platform: + return platform + elif sys.platform.startswith("linux"): + return "linux" + elif sys.platform.startswith("darwin"): + return "macos" + elif sys.platform.startswith(("win32", "cygwin")): + return "windows" + else: + msg = f"Unsupported platform {sys.platform!r}" + raise Exception(msg) + + +DEFAULT_CIBW_ENABLE = "cpython-freethreading cpython-prerelease cpython-experimental-riscv64" -platform: str -if "CIBW_PLATFORM" in os.environ: - platform = os.environ["CIBW_PLATFORM"] -elif sys.platform.startswith("linux"): - platform = "linux" -elif sys.platform.startswith("darwin"): - platform = "macos" -elif sys.platform in ["win32", "cygwin"]: - platform = "windows" -else: - raise Exception("Unsupported platform") +def get_enable_groups() -> frozenset[EnableGroup]: + value = os.environ.get("CIBW_ENABLE", DEFAULT_CIBW_ENABLE) + return EnableGroup.parse_option_value(value) -def cibuildwheel_get_build_identifiers(project_path, env=None, *, prerelease_pythons=False): +def cibuildwheel_get_build_identifiers( + project_path: Path, + env: dict[str, str] | None = None, +) -> list[str]: """ Returns the list of build identifiers that cibuildwheel will try to build for the current platform. """ cmd = [sys.executable, "-m", "cibuildwheel", "--print-build-identifiers", str(project_path)] - if prerelease_pythons: - cmd.append("--prerelease-pythons") + if env is None: + env = os.environ.copy() cmd_output = subprocess.run( cmd, - universal_newlines=True, + text=True, env=env, check=True, stdout=subprocess.PIPE, @@ -44,7 +82,29 @@ def cibuildwheel_get_build_identifiers(project_path, env=None, *, prerelease_pyt return cmd_output.strip().split("\n") -def cibuildwheel_run(project_path, package_dir=".", env=None, add_env=None, output_dir=None): +def _update_pip_cache_dir(env: dict[str, str]) -> None: + # Fix for pip concurrency bug https://github.com/pypa/pip/issues/11340 + # See https://github.com/pypa/cibuildwheel/issues/1254 for discussion. + if get_platform() == "linux": + return + if "PIP_CACHE_DIR" in env: + return + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + if worker_id is None or worker_id == "gw0": + return + pip_cache_dir = CIBW_CACHE_PATH / "test_cache" / f"pip_cache_dir_{worker_id}" + env["PIP_CACHE_DIR"] = str(pip_cache_dir) + + +def cibuildwheel_run( + project_path: str | Path, + package_dir: str | Path = ".", + env: dict[str, str] | None = None, + add_env: Mapping[str, str] | None = None, + output_dir: Path | None = None, + add_args: Sequence[str] | None = None, + single_python: bool = False, +) -> list[str]: """ Runs cibuildwheel as a subprocess, building the project at project_path. @@ -57,6 +117,7 @@ def cibuildwheel_run(project_path, package_dir=".", env=None, add_env=None, outp :param add_env: environment used to update env :param output_dir: directory where wheels are saved. If None, a temporary directory will be used for the duration of the command. + :param add_args: Additional command-line arguments to pass to cibuildwheel. :return: list of built wheels (file names). """ if env is None: @@ -64,49 +125,120 @@ def cibuildwheel_run(project_path, package_dir=".", env=None, add_env=None, outp # If present in the host environment, remove the MACOSX_DEPLOYMENT_TARGET for consistency env.pop("MACOSX_DEPLOYMENT_TARGET", None) + if add_args is None: + add_args = [] + if add_env is not None: env.update(add_env) + _update_pip_cache_dir(env) + + if single_python: + env["CIBW_BUILD"] = "cp{}{}-*".format(*SINGLE_PYTHON_VERSION) + with TemporaryDirectory() as tmp_output_dir: subprocess.run( [ sys.executable, "-m", "cibuildwheel", - "--prerelease-pythons", "--output-dir", str(output_dir or tmp_output_dir), str(package_dir), + *add_args, ], env=env, cwd=project_path, check=True, ) - wheels = os.listdir(output_dir or tmp_output_dir) + wheels = [p.name for p in (output_dir or Path(tmp_output_dir)).iterdir()] return wheels -def _get_arm64_macosx_deployment_target(macosx_deployment_target: str) -> str: +def _floor_macosx(*args: str) -> str: """ - The first version of macOS that supports arm is 11.0. So the wheel tag - cannot contain an earlier deployment target, even if - MACOSX_DEPLOYMENT_TARGET sets it. + Make sure a deployment target is not less than some value. """ - version_tuple = tuple(map(int, macosx_deployment_target.split("."))) - if version_tuple <= (11, 0): - return "11.0" - else: - return macosx_deployment_target + return max(args, key=lambda x: tuple(map(int, x.split(".")))) def expected_wheels( - package_name, - package_version, - manylinux_versions=None, - musllinux_versions=None, - macosx_deployment_target="10.9", - machine_arch=None, -): + package_name: str, + package_version: str, + manylinux_versions: list[str] | None = None, + musllinux_versions: list[str] | None = None, + macosx_deployment_target: str | None = None, + iphoneos_deployment_target: str | None = None, + machine_arch: str | None = None, + platform: str | None = None, + python_abi_tags: list[str] | None = None, + include_universal2: bool = False, + single_python: bool = False, + single_arch: bool = False, + full_auto: bool = False, +) -> list[str]: + """ + Returns the expected wheels from a run of cibuildwheel. + """ + platform = platform or get_platform() + + if machine_arch is None: + machine_arch = pm.machine() + if platform == "linux": + machine_arch = arch_name_for_linux(machine_arch) + + if macosx_deployment_target is None: + macosx_deployment_target = os.environ.get("MACOSX_DEPLOYMENT_TARGET", "10.9") + + if iphoneos_deployment_target is None: + iphoneos_deployment_target = os.environ.get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + + architectures = [machine_arch] + if not single_arch: + if platform == "linux" and full_auto: + if machine_arch == "x86_64": + architectures.append("i686") + elif ( + machine_arch == "aarch64" + and sys.platform.startswith("linux") + and _AARCH64_CAN_RUN_ARMV7 + ): + architectures.append("armv7l") + elif platform == "windows" and machine_arch == "AMD64": + architectures.append("x86") + + return [ + wheel + for architecture in architectures + for wheel in _expected_wheels( + package_name=package_name, + package_version=package_version, + machine_arch=architecture, + manylinux_versions=manylinux_versions, + musllinux_versions=musllinux_versions, + macosx_deployment_target=macosx_deployment_target, + iphoneos_deployment_target=iphoneos_deployment_target, + platform=platform, + python_abi_tags=python_abi_tags, + include_universal2=include_universal2, + single_python=single_python, + ) + ] + + +def _expected_wheels( + package_name: str, + package_version: str, + machine_arch: str, + manylinux_versions: list[str] | None, + musllinux_versions: list[str] | None, + macosx_deployment_target: str, + iphoneos_deployment_target: str, + platform: str, + python_abi_tags: list[str] | None, + include_universal2: bool, + single_python: bool, +) -> Generator[str, None, None]: """ Returns a list of expected wheels from a run of cibuildwheel. """ @@ -115,91 +247,163 @@ def expected_wheels( # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel # so we'll merge them below as python_abi_tag - if machine_arch is None: - machine_arch = pm.machine() + enable_groups = EnableGroup.parse_option_value(os.environ.get("CIBW_ENABLE", "")) if manylinux_versions is None: - if machine_arch == "x86_64": - manylinux_versions = [ - "manylinux_2_5", - "manylinux1", - "manylinux_2_17", - "manylinux2014", - ] - else: - manylinux_versions = ["manylinux_2_17", "manylinux2014"] + manylinux_versions = { + "armv7l": ["manylinux2014", "manylinux_2_17", "manylinux_2_31"], + "i686": ["manylinux1", "manylinux2014", "manylinux_2_17", "manylinux_2_5"], + "x86_64": ["manylinux1", "manylinux_2_28", "manylinux_2_5"], + "riscv64": ["manylinux_2_31", "manylinux_2_35"], + }.get(machine_arch, ["manylinux2014", "manylinux_2_17", "manylinux_2_28"]) if musllinux_versions is None: - musllinux_versions = ["musllinux_1_1"] + musllinux_versions = ["musllinux_1_2"] + + if platform == "pyodide" and python_abi_tags is None: + python_abi_tags = ["cp312-cp312"] + if EnableGroup.PyodidePrerelease in enable_groups: + python_abi_tags.append("cp313-cp313") + elif platform == "ios" and python_abi_tags is None: + python_abi_tags = ["cp313-cp313"] + elif python_abi_tags is None: + python_abi_tags = [ + "cp38-cp38", + "cp39-cp39", + "cp310-cp310", + "cp311-cp311", + "cp312-cp312", + "cp313-cp313", + ] + + enable_groups = get_enable_groups() + if EnableGroup.CPythonFreeThreading in enable_groups: + python_abi_tags.append("cp313-cp313t") + + if EnableGroup.CPythonPrerelease in enable_groups: + python_abi_tags.append("cp314-cp314") + if EnableGroup.CPythonFreeThreading in enable_groups: + python_abi_tags.append("cp314-cp314t") + + if EnableGroup.PyPyEoL in enable_groups: + python_abi_tags += [ + "pp38-pypy38_pp73", + "pp39-pypy39_pp73", + ] + if EnableGroup.PyPy in enable_groups: + python_abi_tags += [ + "pp310-pypy310_pp73", + "pp311-pypy311_pp73", + ] + + if EnableGroup.GraalPy in enable_groups: + python_abi_tags += [ + "graalpy311-graalpy242_311_native", + ] - python_abi_tags = ["cp36-cp36m", "cp37-cp37m", "cp38-cp38", "cp39-cp39", "cp310-cp310"] + if machine_arch == "ARM64" and platform == "windows": + # no CPython 3.8 on Windows ARM64 + python_abi_tags = [t for t in python_abi_tags if not t.startswith("cp38")] - if machine_arch in ["x86_64", "AMD64", "x86", "aarch64"]: - python_abi_tags += ["pp37-pypy37_pp73", "pp38-pypy38_pp73"] + if machine_arch not in PYPY_ARCHS: + python_abi_tags = [tag for tag in python_abi_tags if not tag.startswith("pp")] - if platform == "macos" and machine_arch == "arm64": - # currently, arm64 macs are only supported by cp39 & cp310 - python_abi_tags = ["cp39-cp39", "cp310-cp310"] + if machine_arch not in GRAALPY_ARCHS: + python_abi_tags = [tag for tag in python_abi_tags if not tag.startswith("graalpy")] - wheels = [] + if single_python: + python_tag = "cp{}{}-".format(*SINGLE_PYTHON_VERSION) + python_abi_tags = [ + next( + tag + for tag in python_abi_tags + if tag.startswith(python_tag) and not tag.endswith("t") + ) + ] for python_abi_tag in python_abi_tags: platform_tags = [] if platform == "linux": - architectures = [machine_arch] - - if machine_arch == "x86_64": - architectures.append("i686") - - platform_tags = [ - ".".join( - f"{manylinux_version}_{architecture}" - for manylinux_version in manylinux_versions - ) - for architecture in architectures - ] - if len(musllinux_versions) > 0 and not python_abi_tag.startswith("pp"): - platform_tags.extend( - [ - ".".join( - f"{musllinux_version}_{architecture}" - for musllinux_version in musllinux_versions - ) - for architecture in architectures - ] + if len(manylinux_versions) > 0: + platform_tags = [ + ".".join( + f"{manylinux_version}_{machine_arch}" + for manylinux_version in manylinux_versions + ) + ] + if len(musllinux_versions) > 0 and not python_abi_tag.startswith(("pp", "graalpy")): + platform_tags.append( + ".".join( + f"{musllinux_version}_{machine_arch}" + for musllinux_version in musllinux_versions + ) ) elif platform == "windows": - if python_abi_tag.startswith("cp"): - platform_tags = ["win32", "win_amd64"] - else: - platform_tags = ["win_amd64"] + platform_tags = { + "AMD64": ["win_amd64"], + "ARM64": ["win_arm64"], + "x86": ["win32"], + }.get(machine_arch, []) elif platform == "macos": - if python_abi_tag == "cp39-cp39" and machine_arch == "arm64": - arm64_macosx_deployment_target = _get_arm64_macosx_deployment_target( - macosx_deployment_target - ) + if python_abi_tag.startswith("pp"): + if python_abi_tag.startswith("pp38"): + min_macosx = macosx_deployment_target + else: + min_macosx = _floor_macosx(macosx_deployment_target, "10.15") + elif python_abi_tag.startswith("cp"): + if python_abi_tag.startswith(("cp38", "cp39", "cp310", "cp311")): + min_macosx = macosx_deployment_target + else: + min_macosx = _floor_macosx(macosx_deployment_target, "10.13") + else: + min_macosx = macosx_deployment_target + + if machine_arch == "arm64": + arm64_macosx = _floor_macosx(min_macosx, "11.0") + platform_tags = [f"macosx_{arm64_macosx.replace('.', '_')}_arm64"] + else: + platform_tags = [f"macosx_{min_macosx.replace('.', '_')}_x86_64"] + + if include_universal2: + platform_tags.append(f"macosx_{min_macosx.replace('.', '_')}_universal2") + + elif platform == "ios": + if machine_arch == "x86_64": platform_tags = [ - f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2', - f'macosx_{arm64_macosx_deployment_target.replace(".", "_")}_arm64', + f"ios_{iphoneos_deployment_target.replace('.', '_')}_x86_64_iphonesimulator" ] - else: + elif machine_arch == "arm64": platform_tags = [ - f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', + f"ios_{iphoneos_deployment_target.replace('.', '_')}_arm64_iphoneos", + f"ios_{iphoneos_deployment_target.replace('.', '_')}_arm64_iphonesimulator", ] + else: + msg = f"Unsupported architecture {machine_arch!r} for iOS" + raise Exception(msg) + + elif platform == "pyodide": + platform_tags = { + "cp312-cp312": ["pyodide_2024_0_wasm32"], + "cp313-cp313": ["pyodide_2025_0_wasm32"], + }.get(python_abi_tag, []) + + if not platform_tags: + # for example if the python tag is `none` or `abi3`, all + # platform tags are built with that python tag + platform_tags = ["pyodide_2024_0_wasm32"] else: - raise Exception("unsupported platform") + msg = f"Unsupported platform {platform!r}" + raise Exception(msg) for platform_tag in platform_tags: - wheels.append(f"{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl") - - return wheels + yield f"{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl" -def get_macos_version(): +def get_macos_version() -> tuple[int, int]: """ Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) @@ -208,4 +412,41 @@ def get_macos_version(): (11, 2) <= (11, 0) != True """ version_str, _, _ = pm.mac_ver() - return tuple(map(int, version_str.split(".")[:2])) + return tuple(map(int, version_str.split(".")[:2])) # type: ignore[return-value] + + +def get_xcode_version() -> tuple[int, int]: + """Calls `xcodebuild -version` to retrieve the Xcode version as a 2-tuple.""" + output = subprocess.run( + ["xcodebuild", "-version"], + text=True, + check=True, + stdout=subprocess.PIPE, + ).stdout + lines = output.splitlines() + _, version_str = lines[0].split() + + version_parts = version_str.split(".") + return (int(version_parts[0]), int(version_parts[1])) + + +def skip_if_pyodide(reason: str) -> Any: + return pytest.mark.skipif(get_platform() == "pyodide", reason=reason) + + +def invoke_pytest() -> str: + # see https://github.com/pyodide/pyodide/issues/4802 + if get_platform() == "pyodide": + return "python -m pytest" + return "pytest" + + +def arch_name_for_linux(arch: str) -> str: + """ + Archs have different names on different platforms, but it's useful to be + able to run linux tests on dev machines. This function translates between + the different names. + """ + if arch == "arm64": + return "aarch64" + return arch diff --git a/unit_test/architecture_test.py b/unit_test/architecture_test.py new file mode 100644 index 000000000..98efe91d8 --- /dev/null +++ b/unit_test/architecture_test.py @@ -0,0 +1,117 @@ +import platform as platform_module +import shutil +import sys + +import pytest + +import cibuildwheel.architecture +from cibuildwheel.architecture import Architecture + + +@pytest.fixture( + params=[ + pytest.param(("linux", "linux", "x86_64", "64"), id="linux-64"), + pytest.param(("linux", "linux", "i686", "32"), id="linux-32"), + pytest.param(("linux", "linux", "aarch64", "arm"), id="linux-arm"), + pytest.param(("macos", "darwin", "x86_64", "64"), id="macos-64"), + pytest.param(("macos", "darwin", "arm64", "arm"), id="macos-arm"), + pytest.param(("windows", "win32", "x86", "32"), id="windows-32"), + pytest.param(("windows", "win32", "AMD64", "64"), id="windows-64"), + pytest.param(("windows", "win32", "ARM64", "arm"), id="windows-arm"), + ] +) +def platform_machine(request, monkeypatch): + platform_name, platform_value, machine_value, machine_name = request.param + monkeypatch.setattr(sys, "platform", platform_value) + monkeypatch.setattr(platform_module, "machine", lambda: machine_value) + monkeypatch.setattr(cibuildwheel.architecture, "_check_aarch32_el0", lambda: True) + return platform_name, machine_name + + +def test_arch_auto(platform_machine): + _, machine_name = platform_machine + + arch_set = Architecture.auto_archs("linux") + expected = { + "32": {Architecture.i686}, + "64": {Architecture.x86_64}, + "arm": {Architecture.aarch64}, + } + assert arch_set == expected[machine_name] + + arch_set = Architecture.auto_archs("macos") + expected = {"32": set(), "64": {Architecture.x86_64}, "arm": {Architecture.arm64}} + assert arch_set == expected[machine_name] + + arch_set = Architecture.auto_archs("windows") + expected = { + "32": {Architecture.x86}, + "64": {Architecture.AMD64, Architecture.x86}, + "arm": {Architecture.ARM64}, + } + assert arch_set == expected[machine_name] + + +def test_arch_auto64(platform_machine): + _, machine_name = platform_machine + + arch_set = Architecture.parse_config("auto64", "linux") + expected = {"32": set(), "64": {Architecture.x86_64}, "arm": {Architecture.aarch64}} + assert arch_set == expected[machine_name] + + arch_set = Architecture.parse_config("auto64", "macos") + expected = {"32": set(), "64": {Architecture.x86_64}, "arm": {Architecture.arm64}} + assert arch_set == expected[machine_name] + + arch_set = Architecture.parse_config("auto64", "windows") + expected = {"32": set(), "64": {Architecture.AMD64}, "arm": {Architecture.ARM64}} + assert arch_set == expected[machine_name] + + +def test_arch_auto32(platform_machine): + _, machine_name = platform_machine + + arch_set = Architecture.parse_config("auto32", "linux") + expected = {"32": {Architecture.i686}, "64": {Architecture.i686}, "arm": {Architecture.armv7l}} + assert arch_set == expected[machine_name] + + arch_set = Architecture.parse_config("auto32", "macos") + assert arch_set == set() + + arch_set = Architecture.parse_config("auto32", "windows") + expected = {"32": {Architecture.x86}, "64": {Architecture.x86}, "arm": set()} + assert arch_set == expected[machine_name] + + +def test_arch_auto_no_aarch32(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(platform_module, "machine", lambda: "aarch64") + monkeypatch.setattr(shutil, "which", lambda *args, **kwargs: None) + + arch_set = Architecture.parse_config("auto", "linux") + assert arch_set == {Architecture.aarch64} + + arch_set = Architecture.parse_config("auto64", "linux") + assert arch_set == {Architecture.aarch64} + + monkeypatch.setattr(cibuildwheel.architecture, "_check_aarch32_el0", lambda: True) + arch_set = Architecture.parse_config("auto32", "linux") + assert arch_set == {Architecture.armv7l} + + monkeypatch.setattr(cibuildwheel.architecture, "_check_aarch32_el0", lambda: False) + arch_set = Architecture.parse_config("auto32", "linux") + assert arch_set == set() + + +def test_arch_native_on_ios(monkeypatch): + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(platform_module, "machine", lambda: "arm64") + arch_set = Architecture.parse_config("native", platform="ios") + assert arch_set == {Architecture.arm64_iphonesimulator} + + +def test_arch_auto_on_ios(monkeypatch): + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(platform_module, "machine", lambda: "arm64") + arch_set = Architecture.parse_config("auto", platform="ios") + assert arch_set == {Architecture.arm64_iphonesimulator, Architecture.arm64_iphoneos} diff --git a/unit_test/build_ids_test.py b/unit_test/build_ids_test.py index 927d110a0..13014c242 100644 --- a/unit_test/build_ids_test.py +++ b/unit_test/build_ids_test.py @@ -1,18 +1,16 @@ -from typing import Dict, List +import tomllib -import tomli from packaging.version import Version from cibuildwheel.extra import Printable, dump_python_configurations -from cibuildwheel.util import resources_dir +from cibuildwheel.util import resources def test_compare_configs(): - with open(resources_dir / "build-platforms.toml") as f1: - txt = f1.read() + txt = resources.BUILD_PLATFORMS.read_text() - with open(resources_dir / "build-platforms.toml", "rb") as f2: - dict_txt = tomli.load(f2) + with resources.BUILD_PLATFORMS.open("rb") as f2: + dict_txt = tomllib.load(f2) new_txt = dump_python_configurations(dict_txt) print(new_txt) @@ -22,7 +20,7 @@ def test_compare_configs(): def test_dump_with_Version(): # MyPy doesn't understand deeply nested dicts correctly - example: Dict[str, Dict[str, List[Dict[str, Printable]]]] = { + example: dict[str, dict[str, list[dict[str, Printable]]]] = { "windows": { "python_configurations": [ {"identifier": "cp27-win32", "version": Version("2.7.18"), "arch": "32"}, diff --git a/unit_test/build_selector_test.py b/unit_test/build_selector_test.py index 7555ab33c..0b29b3887 100644 --- a/unit_test/build_selector_test.py +++ b/unit_test/build_selector_test.py @@ -1,17 +1,21 @@ -import pytest from packaging.specifiers import SpecifierSet -from cibuildwheel.util import BuildSelector +from cibuildwheel.selector import BuildSelector, EnableGroup def test_build(): - build_selector = BuildSelector(build_config="cp3*-* *-manylinux*", skip_config="") + build_selector = BuildSelector( + build_config="cp3*-* *-manylinux*", skip_config="", enable=frozenset([EnableGroup.PyPy]) + ) assert build_selector("cp36-manylinux_x86_64") assert build_selector("cp37-manylinux_x86_64") assert build_selector("cp310-manylinux_x86_64") - assert build_selector("pp36-manylinux_x86_64") - assert build_selector("pp37-manylinux_x86_64") + assert build_selector("cp311-manylinux_x86_64") + assert build_selector("cp312-manylinux_x86_64") + assert build_selector("cp313-manylinux_x86_64") + assert build_selector("pp310-manylinux_x86_64") + assert build_selector("pp311-manylinux_x86_64") assert build_selector("cp36-manylinux_i686") assert build_selector("cp37-manylinux_i686") assert build_selector("cp36-macosx_intel") @@ -19,49 +23,109 @@ def test_build(): assert build_selector("cp39-macosx_intel") assert build_selector("cp39-macosx_universal2") assert build_selector("cp39-macosx_arm64") - assert not build_selector("pp36-macosx_intel") - assert not build_selector("pp37-macosx_intel") + assert not build_selector("pp310-macosx_intel") + assert not build_selector("pp311-macosx_intel") assert build_selector("cp36-win32") assert build_selector("cp37-win32") - assert not build_selector("pp36-win32") - assert not build_selector("pp37-win32") + assert not build_selector("pp310-win32") + assert not build_selector("pp311-win32") assert build_selector("cp36-win_amd64") assert build_selector("cp37-win_amd64") assert build_selector("cp310-win_amd64") - assert not build_selector("pp36-win_amd64") - assert not build_selector("pp37-win_amd64") + assert build_selector("cp311-win_amd64") + assert build_selector("cp312-win_amd64") + assert build_selector("cp313-win_amd64") + assert not build_selector("pp310-win_amd64") + assert not build_selector("pp311-win_amd64") -@pytest.mark.skip("this test only makes sense when we have a prerelease python to test with") def test_build_filter_pre(): build_selector = BuildSelector( build_config="cp3*-* *-manylinux*", skip_config="", - prerelease_pythons=True, + enable=frozenset([EnableGroup.CPythonPrerelease, EnableGroup.PyPy]), ) assert build_selector("cp37-manylinux_x86_64") - assert build_selector("cp310-manylinux_x86_64") + assert build_selector("cp313-manylinux_x86_64") assert build_selector("cp37-win_amd64") - assert build_selector("cp310-win_amd64") + assert build_selector("cp313-win_amd64") + assert not build_selector("cp313t-manylinux_x86_64") -def test_skip(): +def test_build_filter_pypy(): + build_selector = BuildSelector( + build_config="*", + skip_config="", + enable=frozenset([EnableGroup.PyPy]), + ) + assert build_selector("pp310-manylinux_x86_64") + assert build_selector("pp311-manylinux_x86_64") + assert not build_selector("pp38-manylinux_x86_64") + assert not build_selector("pp39-manylinux_x86_64") + + +def test_build_filter_pypy_eol(): build_selector = BuildSelector( - build_config="*", skip_config="pp36-* cp3?-manylinux_i686 cp36-win* *-win32" + build_config="*", + skip_config="", + enable=frozenset([EnableGroup.PyPyEoL]), ) + assert not build_selector("pp310-manylinux_x86_64") + assert not build_selector("pp311-manylinux_x86_64") + assert build_selector("pp38-manylinux_x86_64") + assert build_selector("pp39-manylinux_x86_64") + - assert not build_selector("pp36-manylinux_x86_64") - assert build_selector("pp37-manylinux_x86_64") +def test_build_filter_pypy_all(): + build_selector = BuildSelector( + build_config="*", + skip_config="", + enable=frozenset([EnableGroup.PyPyEoL, EnableGroup.PyPy]), + ) + assert build_selector("pp310-manylinux_x86_64") + assert build_selector("pp311-manylinux_x86_64") assert build_selector("pp38-manylinux_x86_64") - assert build_selector("pp37-manylinux_i686") - assert build_selector("pp38-manylinux_i686") + assert build_selector("pp39-manylinux_x86_64") + + +def test_build_filter_pyodide_prerelease(): + build_selector = BuildSelector( + build_config="*", + skip_config="", + enable=frozenset([EnableGroup.PyodidePrerelease]), + ) + assert build_selector("cp312-pyodide_wasm32") + assert build_selector("cp313-pyodide_wasm32") + + +def test_build_filter_pyodide(): + build_selector = BuildSelector( + build_config="*", + skip_config="", + enable=frozenset(), + ) + assert build_selector("cp312-pyodide_wasm32") + assert not build_selector("cp313-pyodide_wasm32") + + +def test_skip(): + build_selector = BuildSelector( + build_config="*", + skip_config="pp310-* cp3?-manylinux_i686 cp36-win* *-win32", + enable=frozenset([EnableGroup.PyPy]), + ) + + assert not build_selector("pp310-manylinux_x86_64") + assert build_selector("pp311-manylinux_x86_64") + assert not build_selector("pp37-manylinux_i686") + assert not build_selector("pp38-manylinux_i686") assert build_selector("cp36-manylinux_x86_64") assert build_selector("cp37-manylinux_x86_64") assert not build_selector("cp36-manylinux_i686") assert not build_selector("cp37-manylinux_i686") - assert not build_selector("pp36-macosx_10_6_intel") - assert build_selector("pp37-macosx_10_6_intel") + assert not build_selector("pp39-macosx_10_6_intel") + assert build_selector("pp311-macosx_10_6_intel") assert build_selector("cp36-macosx_10_6_intel") assert build_selector("cp37-macosx_10_6_intel") assert not build_selector("cp36-win32") @@ -72,7 +136,9 @@ def test_skip(): def test_build_and_skip(): build_selector = BuildSelector( - build_config="cp36-* cp37-macosx* *-manylinux*", skip_config="pp37-* cp37-manylinux_i686" + build_config="cp36-* cp37-macosx* *-manylinux*", + skip_config="pp37-* cp37-manylinux_i686", + enable=frozenset([EnableGroup.PyPy]), ) assert not build_selector("pp37-manylinux_x86_64") @@ -103,7 +169,10 @@ def test_build_braces(): def test_build_limited_python(): build_selector = BuildSelector( - build_config="*", skip_config="", requires_python=SpecifierSet(">=3.7") + build_config="*", + skip_config="", + requires_python=SpecifierSet(">=3.7"), + enable=frozenset([EnableGroup.PyPy, EnableGroup.PyPyEoL, EnableGroup.GraalPy]), ) assert not build_selector("cp36-manylinux_x86_64") @@ -116,6 +185,7 @@ def test_build_limited_python(): assert build_selector("cp37-win32") assert build_selector("cp38-win32") assert build_selector("pp37-win_amd64") + assert build_selector("gp311_242-win_amd64") def test_build_limited_python_partial(): @@ -136,3 +206,29 @@ def test_build_limited_python_patch(): assert build_selector("cp36-manylinux_x86_64") assert build_selector("cp37-manylinux_x86_64") + + +def test_build_free_threaded_python(): + build_selector = BuildSelector(build_config="*", skip_config="", enable=frozenset(EnableGroup)) + + assert build_selector("cp313t-manylinux_x86_64") + + +def test_build_riscv64_enable(): + build_selector = BuildSelector(build_config="*", skip_config="") + assert not build_selector("cp313-manylinux_riscv64") + build_selector = BuildSelector( + build_config="*", skip_config="", enable=frozenset([EnableGroup.CPythonExperimentalRiscV64]) + ) + assert build_selector("cp313-manylinux_riscv64") + + +def test_testing_selector(): + # local import to avoid pytest trying to collect this as a test class! + from cibuildwheel.selector import TestSelector + + test_selector = TestSelector(skip_config="cp36-*") + + assert not test_selector("cp36-win_amd64") + assert test_selector("cp37-manylinux_x86_64") + assert test_selector("cp311-manylinux_x86_64") diff --git a/unit_test/conftest.py b/unit_test/conftest.py index 26a28c474..a51f10d8d 100644 --- a/unit_test/conftest.py +++ b/unit_test/conftest.py @@ -8,36 +8,31 @@ def pytest_addoption(parser): parser.addoption("--run-docker", action="http://github.com/store_true", default=False, help="run docker tests") - - -def pytest_configure(config): - config.addinivalue_line("markers", "docker: mark test requiring docker to run") - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--run-docker"): - # --run-docker given in cli: do not skip docker tests - return - skip_docker = pytest.mark.skip(reason="need --run-docker option to run") - for item in items: - if "docker" in item.keywords: - item.add_marker(skip_docker) + parser.addoption("--run-podman", action="http://github.com/store_true", default=False, help="run podman tests") + parser.addoption( + "--run-cp38-universal2", + action="http://github.com/store_true", + default=False, + help="macOS cp38 uses the universal2 installer", + ) @pytest.fixture -def fake_package_dir(monkeypatch): +def fake_package_dir(tmp_path, monkeypatch): """ Monkey-patch enough for the main() function to run """ real_path_exists = Path.exists def mock_path_exists(path): - if path == MOCK_PACKAGE_DIR / "setup.py": + if str(path).endswith(str(MOCK_PACKAGE_DIR / "setup.py")): return True else: return real_path_exists(path) args = ["cibuildwheel", str(MOCK_PACKAGE_DIR)] + tmp_path.joinpath(MOCK_PACKAGE_DIR).mkdir() monkeypatch.setattr(Path, "exists", mock_path_exists) monkeypatch.setattr(sys, "argv", args) + monkeypatch.chdir(tmp_path) return args diff --git a/unit_test/dependency_constraints_test.py b/unit_test/dependency_constraints_test.py index 952151958..78d4ab9e3 100644 --- a/unit_test/dependency_constraints_test.py +++ b/unit_test/dependency_constraints_test.py @@ -1,21 +1,55 @@ from pathlib import Path -from cibuildwheel.util import DependencyConstraints +import pytest +from cibuildwheel.util.packaging import DependencyConstraints -def test_defaults(): - dependency_constraints = DependencyConstraints.with_defaults() + +def test_defaults(tmp_path: Path) -> None: + dependency_constraints = DependencyConstraints.pinned() project_root = Path(__file__).parents[1] resources_dir = project_root / "cibuildwheel" / "resources" + assert dependency_constraints.base_file_path assert dependency_constraints.base_file_path.samefile(resources_dir / "constraints.txt") - assert dependency_constraints.get_for_python_version("3.99").samefile( - resources_dir / "constraints.txt" + + constraints_file = dependency_constraints.get_for_python_version( + version="3.99", tmp_dir=tmp_path + ) + assert constraints_file + assert constraints_file.samefile(resources_dir / "constraints.txt") + + constraints_file = dependency_constraints.get_for_python_version( + version="3.9", tmp_dir=tmp_path ) - assert dependency_constraints.get_for_python_version("3.9").samefile( - resources_dir / "constraints-python39.txt" + assert constraints_file + assert constraints_file.samefile(resources_dir / "constraints-python39.txt") + + constraints_file = dependency_constraints.get_for_python_version( + version="3.13", tmp_dir=tmp_path ) - assert dependency_constraints.get_for_python_version("3.6").samefile( - resources_dir / "constraints-python36.txt" + assert constraints_file + assert constraints_file.samefile(resources_dir / "constraints-python313.txt") + + +def test_inline_packages(tmp_path: Path) -> None: + dependency_constraints = DependencyConstraints( + base_file_path=None, + packages=["foo==1.2.3", "bar==4.5.6"], ) + + constraint_file = dependency_constraints.get_for_python_version(version="x.x", tmp_dir=tmp_path) + assert constraint_file + constraints_file_contents = constraint_file.read_text() + + assert constraints_file_contents == "foo==1.2.3\nbar==4.5.6" + + +@pytest.mark.parametrize("config_string", ["", "latest", "packages:"]) +def test_empty_constraints(config_string: str) -> None: + dependency_constraints = DependencyConstraints.from_config_string(config_string) + + assert not dependency_constraints.packages + assert not dependency_constraints.base_file_path + assert dependency_constraints == DependencyConstraints.latest() diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py deleted file mode 100644 index 3b250ad36..000000000 --- a/unit_test/docker_container_test.py +++ /dev/null @@ -1,202 +0,0 @@ -import platform -import random -import shutil -import subprocess -import textwrap -from pathlib import Path, PurePath - -import pytest - -from cibuildwheel.docker_container import DockerContainer -from cibuildwheel.environment import EnvironmentAssignmentBash - -# for these tests we use manylinux2014 images, because they're available on -# multi architectures and include python3.8 -pm = platform.machine() -if pm == "x86_64": - DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b" -elif pm == "aarch64": - DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_aarch64:2020-05-17-2f8ac3b" -elif pm == "ppc64le": - DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b" -elif pm == "s390x": - DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b" - - -@pytest.mark.docker -def test_simple(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - assert container.call(["echo", "hello"], capture_output=True) == "hello\n" - - -@pytest.mark.docker -def test_no_lf(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - assert container.call(["printf", "hello"], capture_output=True) == "hello" - - -@pytest.mark.docker -def test_environment(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - assert ( - container.call( - ["sh", "-c", "echo $TEST_VAR"], env={"TEST_VAR": "1"}, capture_output=True - ) - == "1\n" - ) - - -@pytest.mark.docker -def test_cwd(): - with DockerContainer( - docker_image=DEFAULT_IMAGE, cwd="/cibuildwheel/working_directory" - ) as container: - assert container.call(["pwd"], capture_output=True) == "/cibuildwheel/working_directory\n" - assert container.call(["pwd"], capture_output=True, cwd="/opt") == "/opt\n" - - -@pytest.mark.docker -def test_container_removed(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - docker_containers_listing = subprocess.run( - "docker container ls", - shell=True, - check=True, - stdout=subprocess.PIPE, - universal_newlines=True, - ).stdout - assert container.name is not None - assert container.name in docker_containers_listing - old_container_name = container.name - - docker_containers_listing = subprocess.run( - "docker container ls", - shell=True, - check=True, - stdout=subprocess.PIPE, - universal_newlines=True, - ).stdout - assert old_container_name not in docker_containers_listing - - -@pytest.mark.docker -def test_large_environment(): - # max environment variable size is 128kB - long_env_var_length = 127 * 1024 - large_environment = { - "a": "0" * long_env_var_length, - "b": "0" * long_env_var_length, - "c": "0" * long_env_var_length, - "d": "0" * long_env_var_length, - } - - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - # check the length of d - assert ( - container.call(["sh", "-c", "echo ${#d}"], env=large_environment, capture_output=True) - == f"{long_env_var_length}\n" - ) - - -@pytest.mark.docker -def test_binary_output(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - # note: the below embedded snippets are in python2 - - # check that we can pass though arbitrary binary data without erroring - container.call( - [ - "/usr/bin/python2", - "-c", - textwrap.dedent( - """ - import sys - sys.stdout.write(''.join(chr(n) for n in range(0, 256))) - """ - ), - ] - ) - - # check that we can capture arbitrary binary data - output = container.call( - [ - "/usr/bin/python2", - "-c", - textwrap.dedent( - """ - import sys - sys.stdout.write(''.join(chr(n % 256) for n in range(0, 512))) - """ - ), - ], - capture_output=True, - ) - - data = bytes(output, encoding="utf8", errors="surrogateescape") - - for i in range(512): - assert data[i] == i % 256 - - # check that environment variables can carry binary data, except null characters - # (https://www.gnu.org/software/libc/manual/html_node/Environment-Variables.html) - binary_data = bytes(n for n in range(1, 256)) - binary_data_string = str(binary_data, encoding="utf8", errors="surrogateescape") - output = container.call( - ["python2", "-c", 'import os, sys; sys.stdout.write(os.environ["TEST_VAR"])'], - env={"TEST_VAR": binary_data_string}, - capture_output=True, - ) - assert output == binary_data_string - - -@pytest.mark.docker -def test_file_operations(tmp_path: Path): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - # test copying a file in - test_binary_data = bytes(random.randrange(256) for _ in range(1000)) - original_test_file = tmp_path / "test.dat" - original_test_file.write_bytes(test_binary_data) - - dst_file = PurePath("/tmp/test.dat") - - container.copy_into(original_test_file, dst_file) - - output = container.call(["cat", dst_file], capture_output=True) - assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape") - - -@pytest.mark.docker -def test_dir_operations(tmp_path: Path): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - test_binary_data = bytes(random.randrange(256) for _ in range(1000)) - original_test_file = tmp_path / "test.dat" - original_test_file.write_bytes(test_binary_data) - - # test copying a dir in - test_dir = tmp_path / "test_dir" - test_dir.mkdir() - test_file = test_dir / "test.dat" - shutil.copyfile(original_test_file, test_file) - - dst_dir = PurePath("/tmp/test_dir") - dst_file = dst_dir / "test.dat" - container.copy_into(test_dir, dst_dir) - - output = container.call(["cat", dst_file], capture_output=True) - assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape") - - # test glob - assert container.glob(dst_dir, "*.dat") == [dst_file] - - # test copy dir out - new_test_dir = tmp_path / "test_dir_new" - container.copy_out(dst_dir, new_test_dir) - - assert test_binary_data == (new_test_dir / "test.dat").read_bytes() - - -@pytest.mark.docker -def test_environment_executor(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - assignment = EnvironmentAssignmentBash("TEST=$(echo 42)") - assert assignment.evaluated_value({}, container.environment_executor) == "42" diff --git a/unit_test/download_test.py b/unit_test/download_test.py index 0aac14309..4c5f9edc8 100644 --- a/unit_test/download_test.py +++ b/unit_test/download_test.py @@ -3,9 +3,9 @@ import certifi import pytest -from cibuildwheel.util import download +from cibuildwheel.util.file import download -DOWNLOAD_URL = "https://raw.githubusercontent.com/pypa/cibuildwheel/v1.6.3/requirements-dev.txt" +DOWNLOAD_URL = "https://cdn.jsdelivr.net/gh/pypa/cibuildwheel@v1.6.3/requirements-dev.txt" def test_download(monkeypatch, tmp_path): diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py index c1d0a4a1d..c5e6e57cf 100644 --- a/unit_test/environment_test.py +++ b/unit_test/environment_test.py @@ -1,7 +1,11 @@ import os +import sys from cibuildwheel.environment import parse_environment +# this command is equivalent to Unix 'echo', but works on Windows too +PYTHON_ECHO = f"'{sys.executable}' -c \"import sys; print(*sys.argv[1:])\"" + def test_basic_parsing(): environment_recipe = parse_environment("VAR=1 VBR=2") @@ -28,7 +32,7 @@ def test_inheritance(): def test_shell_eval(): - environment_recipe = parse_environment('VAR="$(echo "a test" string)"') + environment_recipe = parse_environment(f'VAR="$({PYTHON_ECHO} "a test" string)"') env_copy = os.environ.copy() env_copy.pop("VAR", None) @@ -39,11 +43,12 @@ def test_shell_eval(): def test_shell_eval_and_env(): - environment_recipe = parse_environment('VAR="$(echo "$PREV_VAR" string)"') + environment_recipe = parse_environment(f'VAR="$({PYTHON_ECHO} "$PREV_VAR" string)"') - environment_dict = environment_recipe.as_dictionary(prev_environment={"PREV_VAR": "1 2 3"}) + prev_environment = {**os.environ, "PREV_VAR": "1 2 3"} + environment_dict = environment_recipe.as_dictionary(prev_environment=prev_environment) - assert environment_dict == {"PREV_VAR": "1 2 3", "VAR": "1 2 3 string"} + assert environment_dict == {**prev_environment, "VAR": "1 2 3 string"} def test_empty_var(): @@ -73,10 +78,12 @@ def test_no_vars_pass_through(): def test_operators_inside_eval(): - environment_recipe = parse_environment('SOMETHING="$(echo a; echo b; echo c)"') + environment_recipe = parse_environment( + f'SOMETHING="$({PYTHON_ECHO} a; {PYTHON_ECHO} b; {PYTHON_ECHO} c)"' + ) - # pass the existing process env so PATH is available - environment_dict = environment_recipe.as_dictionary(os.environ.copy()) + # pass the existing process env so subcommands can be run in the evaluation + environment_dict = environment_recipe.as_dictionary(prev_environment=os.environ.copy()) assert environment_dict.get("SOMETHING") == "a\nb\nc" @@ -84,7 +91,6 @@ def test_operators_inside_eval(): def test_substitution_with_backslash(): environment_recipe = parse_environment('PATH2="somewhere_else;$PATH1"') - # pass the existing process env so PATH is available environment_dict = environment_recipe.as_dictionary(prev_environment={"PATH1": "c:\\folder\\"}) assert environment_dict.get("PATH2") == "somewhere_else;c:\\folder\\" @@ -92,11 +98,11 @@ def test_substitution_with_backslash(): def test_awkwardly_quoted_variable(): environment_recipe = parse_environment( - 'VAR2=something"like this""$VAR1"$VAR1$(echo "there is more")"$(echo "and more!")"' + f'VAR2=something"like this""$VAR1"$VAR1$({PYTHON_ECHO} "there is more")"$({PYTHON_ECHO} "and more!")"' ) - # pass the existing process env so PATH is available - environment_dict = environment_recipe.as_dictionary({"VAR1": "but wait"}) + prev_environment = {**os.environ, "VAR1": "but wait"} + environment_dict = environment_recipe.as_dictionary(prev_environment=prev_environment) assert ( environment_dict.get("VAR2") == "somethinglike thisbut waitbut waitthere is moreand more!" diff --git a/unit_test/get_platform_test.py b/unit_test/get_platform_test.py new file mode 100644 index 000000000..320b73ae7 --- /dev/null +++ b/unit_test/get_platform_test.py @@ -0,0 +1,105 @@ +import contextlib +import sys +from collections.abc import Generator +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +import setuptools._distutils.util + +from cibuildwheel.ci import CIProvider, detect_ci_provider +from cibuildwheel.errors import FatalError +from cibuildwheel.platforms.windows import PythonConfiguration, setup_setuptools_cross_compile + +# monkeypatching os.name is too flaky. E.g. It works on my machine, but fails in pipeline +if not sys.platform.startswith("win") and not TYPE_CHECKING: + pytest.skip("Windows-only tests", allow_module_level=True) + + +@contextlib.contextmanager +def patched_environment( + monkeypatch: pytest.MonkeyPatch, environment: dict[str, str] +) -> Generator[None, None, None]: + with monkeypatch.context() as mp: + for envvar, val in environment.items(): + mp.setenv(name=envvar, value=val) + yield + + +def test_x86(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + arch = "32" + environment: dict[str, str] = {} + + configuration = PythonConfiguration( + version="irrelevant", arch=arch, identifier="irrelevant", url=None + ) + + setup_setuptools_cross_compile(tmp_path, configuration, tmp_path, environment) + with patched_environment(monkeypatch, environment): + target_platform = setuptools._distutils.util.get_platform() + + assert environment["VSCMD_ARG_TGT_ARCH"] == "x86" + assert target_platform == "win32" + + +def test_x64(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + arch = "64" + environment: dict[str, str] = {} + + configuration = PythonConfiguration( + version="irrelevant", arch=arch, identifier="irrelevant", url=None + ) + + setup_setuptools_cross_compile(tmp_path, configuration, tmp_path, environment) + with patched_environment(monkeypatch, environment): + target_platform = setuptools._distutils.util.get_platform() + + assert environment["VSCMD_ARG_TGT_ARCH"] == "x64" + assert target_platform == "win-amd64" + + +@pytest.mark.skipif( + detect_ci_provider() == CIProvider.azure_pipelines, reason="arm64 not recognised on azure" +) +def test_arm(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + arch = "ARM64" + environment: dict[str, str] = {} + + configuration = PythonConfiguration( + version="irrelevant", arch=arch, identifier="irrelevant", url=None + ) + + setup_setuptools_cross_compile(tmp_path, configuration, tmp_path, environment) + with patched_environment(monkeypatch, environment): + target_platform = setuptools._distutils.util.get_platform() + + assert environment["VSCMD_ARG_TGT_ARCH"] == "arm64" + assert target_platform == "win-arm64" + + +def test_env_set(tmp_path: Path) -> None: + arch = "32" + environment = {"VSCMD_ARG_TGT_ARCH": "x64"} + + configuration = PythonConfiguration( + version="irrelevant", arch=arch, identifier="irrelevant", url=None + ) + + with pytest.raises(FatalError, match="VSCMD_ARG_TGT_ARCH"): + setup_setuptools_cross_compile(tmp_path, configuration, tmp_path, environment) + + +def test_env_blank(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + arch = "32" + environment = {"VSCMD_ARG_TGT_ARCH": ""} + + configuration = PythonConfiguration( + version="irrelevant", arch=arch, identifier="irrelevant", url=None + ) + + setup_setuptools_cross_compile(tmp_path, configuration, tmp_path, environment) + with patched_environment(monkeypatch, environment): + target_platform = setuptools._distutils.util.get_platform() + + assert environment["VSCMD_ARG_TGT_ARCH"] == "x86" + assert target_platform == "win32" diff --git a/unit_test/linux_build_steps_test.py b/unit_test/linux_build_steps_test.py index 29bc65a4a..aa4363c13 100644 --- a/unit_test/linux_build_steps_test.py +++ b/unit_test/linux_build_steps_test.py @@ -2,69 +2,94 @@ from pathlib import Path from pprint import pprint -import cibuildwheel.docker_container -import cibuildwheel.linux -from cibuildwheel.options import Options +import pytest -from .utils import get_default_command_line_arguments +import cibuildwheel.platforms.linux +from cibuildwheel.oci_container import OCIContainerEngineConfig +from cibuildwheel.options import CommandLineArguments, Options -def test_linux_container_split(tmp_path: Path, monkeypatch): +def test_linux_container_split(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """ - Tests splitting linux builds by docker image and before_all + Tests splitting linux builds by container image, container engine, and before_all """ - args = get_default_command_line_arguments() + args = CommandLineArguments.defaults() args.platform = "linux" (tmp_path / "pyproject.toml").write_text( textwrap.dedent( """ [tool.cibuildwheel] - manylinux-x86_64-image = "normal_docker_image" - manylinux-i686-image = "normal_docker_image" + manylinux-x86_64-image = "normal_container_image" + manylinux-i686-image = "normal_container_image" build = "*-manylinux_x86_64" - skip = "pp*" + skip = "[gp]p*" archs = "x86_64 i686" [[tool.cibuildwheel.overrides]] select = "cp{38,39,310}-*" - manylinux-x86_64-image = "other_docker_image" - manylinux-i686-image = "other_docker_image" + manylinux-x86_64-image = "other_container_image" + manylinux-i686-image = "other_container_image" [[tool.cibuildwheel.overrides]] select = "cp39-*" before-all = "echo 'a cp39-only command'" + + [[tool.cibuildwheel.overrides]] + select = "cp310-*" + container-engine = "docker; create_args: --privileged" """ ) ) monkeypatch.chdir(tmp_path) - options = Options("linux", command_line_arguments=args) + options = Options("linux", command_line_arguments=args, env={}) - python_configurations = cibuildwheel.linux.get_python_configurations( + python_configurations = cibuildwheel.platforms.linux.get_python_configurations( options.globals.build_selector, options.globals.architectures ) - build_steps = list(cibuildwheel.linux.get_build_steps(options, python_configurations)) + build_steps = list(cibuildwheel.platforms.linux.get_build_steps(options, python_configurations)) # helper functions to extract test info - def identifiers(step): + def identifiers(step: cibuildwheel.platforms.linux.BuildStep) -> list[str]: return [c.identifier for c in step.platform_configs] - def before_alls(step): + def before_alls(step: cibuildwheel.platforms.linux.BuildStep) -> list[str]: return [options.build_options(c.identifier).before_all for c in step.platform_configs] - pprint(build_steps) + def container_engines( + step: cibuildwheel.platforms.linux.BuildStep, + ) -> list[OCIContainerEngineConfig]: + return [options.build_options(c.identifier).container_engine for c in step.platform_configs] - assert build_steps[0].docker_image == "normal_docker_image" - assert identifiers(build_steps[0]) == ["cp36-manylinux_x86_64", "cp37-manylinux_x86_64"] - assert before_alls(build_steps[0]) == ["", ""] - - assert build_steps[1].docker_image == "other_docker_image" - assert identifiers(build_steps[1]) == ["cp38-manylinux_x86_64", "cp310-manylinux_x86_64"] - assert before_alls(build_steps[1]) == ["", ""] + pprint(build_steps) - assert build_steps[2].docker_image == "other_docker_image" - assert identifiers(build_steps[2]) == ["cp39-manylinux_x86_64"] - assert before_alls(build_steps[2]) == ["echo 'a cp39-only command'"] + default_container_engine = OCIContainerEngineConfig(name="docker") + + assert build_steps[0].container_image == "other_container_image" + assert identifiers(build_steps[0]) == ["cp38-manylinux_x86_64"] + assert before_alls(build_steps[0]) == [""] + assert container_engines(build_steps[0]) == [default_container_engine] + + assert build_steps[1].container_image == "other_container_image" + assert identifiers(build_steps[1]) == ["cp39-manylinux_x86_64"] + assert before_alls(build_steps[1]) == ["echo 'a cp39-only command'"] + assert container_engines(build_steps[1]) == [default_container_engine] + + assert build_steps[2].container_image == "other_container_image" + assert identifiers(build_steps[2]) == ["cp310-manylinux_x86_64"] + assert before_alls(build_steps[2]) == [""] + assert container_engines(build_steps[2]) == [ + OCIContainerEngineConfig(name="docker", create_args=("--privileged",)) + ] + + assert build_steps[3].container_image == "normal_container_image" + assert identifiers(build_steps[3]) == [ + "cp311-manylinux_x86_64", + "cp312-manylinux_x86_64", + "cp313-manylinux_x86_64", + ] + assert before_alls(build_steps[3]) == [""] * 3 + assert container_engines(build_steps[3]) == [default_container_engine] * 3 diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index 973012cfb..e01c8c027 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -6,11 +6,19 @@ import pytest -from cibuildwheel import linux, macos, util, windows +from cibuildwheel import __main__, architecture +from cibuildwheel.platforms import linux, macos, pyodide, windows +from cibuildwheel.util import file class ArgsInterceptor: - def __call__(self, *args, **kwargs): + def __init__(self) -> None: + self.call_count = 0 + self.args: tuple[object, ...] | None = None + self.kwargs: dict[str, object] | None = None + + def __call__(self, *args: object, **kwargs: object) -> None: + self.call_count += 1 self.args = args self.kwargs = kwargs @@ -23,18 +31,20 @@ def mock_protection(monkeypatch): """ def fail_on_call(*args, **kwargs): - raise RuntimeError("This should never be called") + msg = "This should never be called" + raise RuntimeError(msg) def ignore_call(*args, **kwargs): pass monkeypatch.setattr(subprocess, "Popen", fail_on_call) - monkeypatch.setattr(util, "download", fail_on_call) + monkeypatch.setattr(file, "download", fail_on_call) monkeypatch.setattr(windows, "build", fail_on_call) monkeypatch.setattr(linux, "build", fail_on_call) monkeypatch.setattr(macos, "build", fail_on_call) - + monkeypatch.setattr(pyodide, "build", fail_on_call) monkeypatch.setattr(Path, "mkdir", ignore_call) + monkeypatch.setattr(architecture, "_check_aarch32_el0", lambda: True) @pytest.fixture(autouse=True) @@ -48,12 +58,12 @@ def disable_print_wheels(monkeypatch): def empty_cm(*args, **kwargs): yield - monkeypatch.setattr(util, "print_new_wheels", empty_cm) + monkeypatch.setattr(__main__, "print_new_wheels", empty_cm) @pytest.fixture -def allow_empty(request, monkeypatch, fake_package_dir): - monkeypatch.setattr(sys, "argv", fake_package_dir + ["--allow-empty"]) +def allow_empty(monkeypatch, fake_package_dir): + monkeypatch.setattr(sys, "argv", [*fake_package_dir, "--allow-empty"]) @pytest.fixture(params=["linux", "macos", "windows"]) @@ -73,16 +83,15 @@ def platform(request, monkeypatch): @pytest.fixture -def intercepted_build_args(platform, monkeypatch): +def intercepted_build_args(monkeypatch): intercepted = ArgsInterceptor() - if platform == "linux": - monkeypatch.setattr(linux, "build", intercepted) - elif platform == "macos": - monkeypatch.setattr(macos, "build", intercepted) - elif platform == "windows": - monkeypatch.setattr(windows, "build", intercepted) - else: - raise ValueError(f"unknown platform value: {platform}") + monkeypatch.setattr(linux, "build", intercepted) + monkeypatch.setattr(macos, "build", intercepted) + monkeypatch.setattr(windows, "build", intercepted) + monkeypatch.setattr(pyodide, "build", intercepted) + + yield intercepted - return intercepted + # check that intercepted_build_args only ever had one set of args + assert intercepted.call_count <= 1 diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 0a345cb1f..731d10adc 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -1,65 +1,100 @@ import sys +import tomllib from fnmatch import fnmatch from pathlib import Path import pytest -import tomli from cibuildwheel.__main__ import main from cibuildwheel.environment import ParsedEnvironment -from cibuildwheel.options import BuildOptions, _get_pinned_docker_images -from cibuildwheel.util import BuildSelector, resources_dir +from cibuildwheel.frontend import _split_config_settings +from cibuildwheel.options import BuildOptions, _get_pinned_container_images +from cibuildwheel.selector import BuildSelector, EnableGroup +from cibuildwheel.util import resources +from cibuildwheel.util.packaging import DependencyConstraints # CIBW_PLATFORM is tested in main_platform_test.py -def test_output_dir(platform, intercepted_build_args, monkeypatch): +@pytest.mark.usefixtures("platform") +def test_output_dir(intercepted_build_args, monkeypatch): OUTPUT_DIR = Path("some_output_dir") monkeypatch.setenv("CIBW_OUTPUT_DIR", str(OUTPUT_DIR)) main() - assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR + assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR.resolve() -def test_output_dir_default(platform, intercepted_build_args, monkeypatch): +@pytest.mark.usefixtures("platform") +def test_output_dir_default(intercepted_build_args): main() - assert intercepted_build_args.args[0].globals.output_dir == Path("wheelhouse") + assert intercepted_build_args.args[0].globals.output_dir == Path("wheelhouse").resolve() +@pytest.mark.usefixtures("platform") @pytest.mark.parametrize("also_set_environment", [False, True]) -def test_output_dir_argument(also_set_environment, platform, intercepted_build_args, monkeypatch): +def test_output_dir_argument(also_set_environment, intercepted_build_args, monkeypatch): OUTPUT_DIR = Path("some_output_dir") - monkeypatch.setattr(sys, "argv", sys.argv + ["--output-dir", str(OUTPUT_DIR)]) + monkeypatch.setattr(sys, "argv", [*sys.argv, "--output-dir", str(OUTPUT_DIR)]) if also_set_environment: monkeypatch.setenv("CIBW_OUTPUT_DIR", "not_this_output_dir") main() - assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR + assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR.resolve() -def test_build_selector(platform, intercepted_build_args, monkeypatch, allow_empty): - BUILD = "some build* *-selector" - SKIP = "some skip* *-selector" - - monkeypatch.setenv("CIBW_BUILD", BUILD) - monkeypatch.setenv("CIBW_SKIP", SKIP) +@pytest.mark.usefixtures("platform", "allow_empty") +def test_build_selector(intercepted_build_args, monkeypatch): + monkeypatch.setenv("CIBW_BUILD", "cp313-*") + monkeypatch.setenv("CIBW_SKIP", "cp39-*") main() intercepted_build_selector = intercepted_build_args.args[0].globals.build_selector assert isinstance(intercepted_build_selector, BuildSelector) - assert intercepted_build_selector("build24-this") - assert not intercepted_build_selector("skip65-that") + assert intercepted_build_selector("cp313-something-to-build") + assert not intercepted_build_selector("cp39-something-to-skip") # This unit test is just testing the options of 'main' # Unit tests for BuildSelector are in build_selector_test.py -def test_empty_selector(platform, intercepted_build_args, monkeypatch): +@pytest.mark.usefixtures("platform", "allow_empty") +def test_invalid_build_selector(monkeypatch, capsys): + monkeypatch.setenv("CIBW_BUILD", "invalid") + + with pytest.raises(SystemExit) as e: + main() + + assert e.value.code == 2 + _, err = capsys.readouterr() + assert "Invalid build selector" in err + + +@pytest.mark.parametrize( + ("option_name", "option_env_var"), + [ + ("skip", "CIBW_SKIP"), + ("test_skip", "CIBW_TEST_SKIP"), + ], +) +@pytest.mark.usefixtures("platform", "intercepted_build_args") +def test_invalid_skip_selector(monkeypatch, capsys, option_name, option_env_var): + monkeypatch.setenv(option_env_var, "invalid") + + main() + + _, err = capsys.readouterr() + print(err) + assert f"Invalid {option_name} selector" in err + + +@pytest.mark.usefixtures("platform", "intercepted_build_args") +def test_empty_selector(monkeypatch): monkeypatch.setenv("CIBW_SKIP", "*") with pytest.raises(SystemExit) as e: @@ -69,25 +104,20 @@ def test_empty_selector(platform, intercepted_build_args, monkeypatch): @pytest.mark.parametrize( - "architecture, image, full_image", + ("architecture", "image", "full_image"), [ - ("x86_64", None, "quay.io/pypa/manylinux2014_x86_64:*"), - ("x86_64", "manylinux1", "quay.io/pypa/manylinux1_x86_64:*"), - ("x86_64", "manylinux2010", "quay.io/pypa/manylinux2010_x86_64:*"), + ("x86_64", None, "quay.io/pypa/manylinux_2_28_x86_64:*"), ("x86_64", "manylinux2014", "quay.io/pypa/manylinux2014_x86_64:*"), - ("x86_64", "manylinux_2_24", "quay.io/pypa/manylinux_2_24_x86_64:*"), + ("x86_64", "manylinux_2_28", "quay.io/pypa/manylinux_2_28_x86_64:*"), + ("x86_64", "manylinux_2_34", "quay.io/pypa/manylinux_2_34_x86_64:*"), ("x86_64", "custom_image", "custom_image"), ("i686", None, "quay.io/pypa/manylinux2014_i686:*"), - ("i686", "manylinux1", "quay.io/pypa/manylinux1_i686:*"), - ("i686", "manylinux2010", "quay.io/pypa/manylinux2010_i686:*"), ("i686", "manylinux2014", "quay.io/pypa/manylinux2014_i686:*"), - ("i686", "manylinux_2_24", "quay.io/pypa/manylinux_2_24_i686:*"), ("i686", "custom_image", "custom_image"), - ("pypy_x86_64", None, "quay.io/pypa/manylinux2014_x86_64:*"), - ("pypy_x86_64", "manylinux1", "manylinux1"), # Does not exist - ("pypy_x86_64", "manylinux2010", "quay.io/pypa/manylinux2010_x86_64:*"), + ("pypy_x86_64", None, "quay.io/pypa/manylinux_2_28_x86_64:*"), ("pypy_x86_64", "manylinux2014", "quay.io/pypa/manylinux2014_x86_64:*"), - ("pypy_x86_64", "manylinux_2_24", "quay.io/pypa/manylinux_2_24_x86_64:*"), + ("pypy_x86_64", "manylinux_2_28", "quay.io/pypa/manylinux_2_28_x86_64:*"), + ("pypy_x86_64", "manylinux_2_34", "quay.io/pypa/manylinux_2_34_x86_64:*"), ("pypy_x86_64", "custom_image", "custom_image"), ], ) @@ -110,15 +140,16 @@ def test_manylinux_images( assert build_options.manylinux_images is None -def get_default_repair_command(platform): +def get_default_repair_command(platform: str) -> str: if platform == "linux": return "auditwheel repair -w {dest_dir} {wheel}" elif platform == "macos": - return "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" - elif platform == "windows": + return "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" + elif platform == "windows" or platform == "pyodide": return "" else: - raise ValueError("Unknown platform", platform) + msg = f"Unknown platform: {platform!r}" + raise ValueError(msg) @pytest.mark.parametrize("repair_command", [None, "repair", "repair -w {dest_dir} {wheel}"]) @@ -255,6 +286,27 @@ def test_build_verbosity( assert build_options.build_verbosity == expected_verbosity +@pytest.mark.parametrize("platform_specific", [False, True]) +def test_config_settings(platform_specific, platform, intercepted_build_args, monkeypatch): + config_settings = 'setting=value setting=value2 other="something else"' + if platform_specific: + monkeypatch.setenv("CIBW_CONFIG_SETTINGS_" + platform.upper(), config_settings) + monkeypatch.setenv("CIBW_CONFIG_SETTINGS", "a=b") + else: + monkeypatch.setenv("CIBW_CONFIG_SETTINGS", config_settings) + + main() + build_options = intercepted_build_args.args[0].build_options(identifier=None) + + assert build_options.config_settings == config_settings + + assert _split_config_settings(config_settings) == [ + "-Csetting=value", + "-Csetting=value2", + "-Cother=something else", + ] + + @pytest.mark.parametrize( "selector", [ @@ -268,15 +320,16 @@ def test_build_verbosity( [ "cp27-*", "cp35-*", + "?p36-*", "?p27*", "?p2*", "?p35*", ], ) -def test_build_selector_deprecated_error( - monkeypatch, platform, intercepted_build_args, selector, pattern, allow_empty, capsys -): +@pytest.mark.usefixtures("platform", "intercepted_build_args", "allow_empty") +def test_build_selector_deprecated_error(monkeypatch, selector, pattern, capsys): monkeypatch.setenv(selector, pattern) + monkeypatch.delenv("CIBW_ENABLE", raising=False) if selector == "CIBW_BUILD": with pytest.raises(SystemExit) as ex: @@ -287,7 +340,8 @@ def test_build_selector_deprecated_error( main() stderr = capsys.readouterr().err - msg = f"cibuildwheel 2.x no longer supports Python < 3.6. Please use the 1.x series or update {selector}" + series = "2" if "6" in pattern else "1" + msg = f"cibuildwheel 3.x no longer supports Python < 3.8. Please use the {series}.x series or update" assert msg in stderr @@ -308,13 +362,123 @@ def test_before_all(before_all, platform_specific, platform, intercepted_build_a assert build_options.before_all == (before_all or "") +@pytest.mark.parametrize( + "dependency_versions", + [None, "pinned", "latest", "FILE", "packages: pip==21.0.0"], +) +@pytest.mark.parametrize("platform_specific", [False, True]) +def test_dependency_versions( + dependency_versions, platform_specific, platform, intercepted_build_args, monkeypatch, tmp_path +): + option_value = dependency_versions + + if dependency_versions == "FILE": + constraints_file = tmp_path / "constraints.txt" + constraints_file.write_text("foo==1.2.3\nbar==4.5.6") + option_value = str(constraints_file) + + if option_value is not None: + if platform_specific: + monkeypatch.setenv("CIBW_DEPENDENCY_VERSIONS_" + platform.upper(), option_value) + monkeypatch.setenv("CIBW_DEPENDENCY_VERSIONS", "overwritten") + else: + monkeypatch.setenv("CIBW_DEPENDENCY_VERSIONS", option_value) + + main() + + build_options: BuildOptions = intercepted_build_args.args[0].build_options(identifier=None) + dependency_constraints = build_options.dependency_constraints + if dependency_versions is None or dependency_versions == "pinned": + assert dependency_constraints == DependencyConstraints.pinned() + elif dependency_versions == "latest": + assert dependency_constraints == DependencyConstraints.latest() + elif dependency_versions == "FILE": + assert dependency_constraints.base_file_path + assert dependency_constraints.base_file_path.samefile(Path(option_value)) + elif dependency_versions.startswith("packages:"): + assert dependency_constraints.packages == ["pip==21.0.0"] + + +@pytest.mark.parametrize("method", ["unset", "command_line", "env_var"]) +def test_debug_traceback(monkeypatch, method, capfd): + if method == "command_line": + monkeypatch.setattr(sys, "argv", [*sys.argv, "--debug-traceback"]) + elif method == "env_var": + monkeypatch.setenv("CIBW_DEBUG_TRACEBACK", "TRUE") + + # set an option that produces a configuration error + monkeypatch.setenv("CIBW_BUILD_FRONTEND", "invalid_value") + + with pytest.raises(SystemExit) as exit: + main() + assert exit.value.code == 2 + + _, err = capfd.readouterr() + + if method == "unset": + assert "Traceback (most recent call last)" not in err + else: + assert "Traceback (most recent call last)" in err + + +@pytest.mark.parametrize("method", ["unset", "command_line", "env_var"]) +def test_enable(method, intercepted_build_args, monkeypatch): + monkeypatch.delenv("CIBW_ENABLE", raising=False) + + if method == "command_line": + monkeypatch.setattr(sys, "argv", [*sys.argv, "--enable", "pypy", "--enable", "graalpy"]) + elif method == "env_var": + monkeypatch.setenv("CIBW_ENABLE", "pypy graalpy") + + main() + + enable_groups = intercepted_build_args.args[0].globals.build_selector.enable + + if method == "unset": + assert enable_groups == frozenset() + else: + assert enable_groups == frozenset([EnableGroup.PyPy, EnableGroup.GraalPy]) + + +def test_enable_all(intercepted_build_args, monkeypatch): + monkeypatch.setattr(sys, "argv", [*sys.argv, "--enable", "all"]) + + main() + + enable_groups = intercepted_build_args.args[0].globals.build_selector.enable + assert enable_groups == EnableGroup.all_groups() + + +def test_enable_arg_inherits(intercepted_build_args, monkeypatch): + monkeypatch.setenv("CIBW_ENABLE", "pypy graalpy") + monkeypatch.setattr(sys, "argv", [*sys.argv, "--enable", "cpython-prerelease"]) + + main() + + enable_groups = intercepted_build_args.args[0].globals.build_selector.enable + + assert enable_groups == frozenset( + (EnableGroup.PyPy, EnableGroup.GraalPy, EnableGroup.CPythonPrerelease) + ) + + +def test_enable_arg_error_message(monkeypatch, capsys): + monkeypatch.setattr(sys, "argv", [*sys.argv, "--enable", "invalid_group"]) + + with pytest.raises(SystemExit) as ex: + main() + assert ex.value.code == 2 + + _, err = capsys.readouterr() + assert "Valid group names are:" in err + + def test_defaults(platform, intercepted_build_args): main() build_options: BuildOptions = intercepted_build_args.args[0].build_options(identifier=None) - defaults_config_path = resources_dir / "defaults.toml" - with defaults_config_path.open("rb") as f: - defaults_toml = tomli.load(f) + with resources.DEFAULTS.open("rb") as f: + defaults_toml = tomllib.load(f) root_defaults = defaults_toml["tool"]["cibuildwheel"] platform_defaults = defaults_toml["tool"]["cibuildwheel"][platform] @@ -329,10 +493,10 @@ def test_defaults(platform, intercepted_build_args): if isinstance(repair_wheel_default, list): repair_wheel_default = " && ".join(repair_wheel_default) assert build_options.repair_command == repair_wheel_default - assert build_options.build_frontend == defaults["build-frontend"] + assert build_options.build_frontend is None if platform == "linux": assert build_options.manylinux_images - pinned_images = _get_pinned_docker_images() + pinned_images = _get_pinned_container_images() default_x86_64_image = pinned_images["x86_64"][defaults["manylinux-x86_64-image"]] assert build_options.manylinux_images["x86_64"] == default_x86_64_image diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 09f5b9db2..620f86e4e 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -4,28 +4,31 @@ from cibuildwheel.__main__ import main from cibuildwheel.architecture import Architecture +from cibuildwheel.selector import EnableGroup from ..conftest import MOCK_PACKAGE_DIR -def test_unknown_platform_non_ci(monkeypatch, capsys): - monkeypatch.delenv("CI", raising=False) - monkeypatch.delenv("BITRISE_BUILD_NUMBER", raising=False) - monkeypatch.delenv("AZURE_HTTP_USER_AGENT", raising=False) - monkeypatch.delenv("TRAVIS", raising=False) - monkeypatch.delenv("APPVEYOR", raising=False) - monkeypatch.delenv("GITHUB_ACTIONS", raising=False) - monkeypatch.delenv("GITLAB_CI", raising=False) - monkeypatch.delenv("CIRCLECI", raising=False) - monkeypatch.delenv("CIBW_PLATFORM", raising=False) +@pytest.mark.parametrize("option_value", [None, "auto", ""]) +def test_platform_unset_or_auto(monkeypatch, intercepted_build_args, option_value): + if option_value is None: + monkeypatch.delenv("CIBW_PLATFORM", raising=False) + else: + monkeypatch.setenv("CIBW_PLATFORM", option_value) - with pytest.raises(SystemExit) as exit: - main() - assert exit.value.code == 2 - _, err = capsys.readouterr() + main() - assert "cibuildwheel: Unable to detect platform." in err - assert "cibuildwheel should run on your CI server" in err + options = intercepted_build_args.args[0] + + # check that the platform was auto detected to build for the current system + if sys.platform.startswith("linux"): + assert options.platform == "linux" + elif sys.platform == "darwin": + assert options.platform == "macos" + elif sys.platform == "win32": + assert options.platform == "windows" + else: + pytest.fail(f"Unknown platform: {sys.platform}") def test_unknown_platform_on_ci(monkeypatch, capsys): @@ -35,10 +38,10 @@ def test_unknown_platform_on_ci(monkeypatch, capsys): with pytest.raises(SystemExit) as exit: main() - assert exit.value.code == 2 + assert exit.value.code == 2 _, err = capsys.readouterr() - assert 'cibuildwheel: Unable to detect platform from "sys.platform"' in err + assert 'Unable to detect platform from "sys.platform"' in err def test_unknown_platform(monkeypatch, capsys): @@ -49,34 +52,34 @@ def test_unknown_platform(monkeypatch, capsys): _, err = capsys.readouterr() assert exit.value.code == 2 - assert "cibuildwheel: Unsupported platform: nonexistent" in err + assert "Unsupported platform: nonexistent" in err def test_platform_argument(platform, intercepted_build_args, monkeypatch): monkeypatch.setenv("CIBW_PLATFORM", "nonexistent") - monkeypatch.setattr(sys, "argv", sys.argv + ["--platform", platform]) + monkeypatch.setattr(sys, "argv", [*sys.argv, "--platform", platform]) main() options = intercepted_build_args.args[0] - assert options.globals.package_dir == MOCK_PACKAGE_DIR + assert options.globals.package_dir == MOCK_PACKAGE_DIR.resolve() -def test_platform_environment(platform, intercepted_build_args, monkeypatch): +@pytest.mark.usefixtures("platform") +def test_platform_environment(intercepted_build_args): main() options = intercepted_build_args.args[0] - assert options.globals.package_dir == MOCK_PACKAGE_DIR - + assert options.globals.package_dir == MOCK_PACKAGE_DIR.resolve() -def test_archs_default(platform, intercepted_build_args, monkeypatch): +def test_archs_default(platform, intercepted_build_args): main() options = intercepted_build_args.args[0] if platform == "linux": - assert options.globals.architectures == {Architecture.x86_64, Architecture.i686} + assert options.globals.architectures == {Architecture.x86_64} elif platform == "windows": assert options.globals.architectures == {Architecture.AMD64, Architecture.x86} else: @@ -85,12 +88,11 @@ def test_archs_default(platform, intercepted_build_args, monkeypatch): @pytest.mark.parametrize("use_env_var", [False, True]) def test_archs_argument(platform, intercepted_build_args, monkeypatch, use_env_var): - if use_env_var: monkeypatch.setenv("CIBW_ARCHS", "ppc64le") else: monkeypatch.setenv("CIBW_ARCHS", "unused") - monkeypatch.setattr(sys, "argv", sys.argv + ["--archs", "ppc64le"]) + monkeypatch.setattr(sys, "argv", [*sys.argv, "--archs", "ppc64le"]) if platform in {"macos", "windows"}: with pytest.raises(SystemExit) as exit: @@ -176,6 +178,8 @@ def test_archs_platform_all(platform, intercepted_build_args, monkeypatch): Architecture.aarch64, Architecture.ppc64le, Architecture.s390x, + Architecture.armv7l, + Architecture.riscv64, } elif platform == "windows": assert options.globals.architectures == { @@ -189,3 +193,88 @@ def test_archs_platform_all(platform, intercepted_build_args, monkeypatch): Architecture.arm64, Architecture.universal2, } + + +@pytest.mark.parametrize( + ("only", "plat"), + ( + ("cp311-manylinux_x86_64", "linux"), + ("cp310-win_amd64", "windows"), + ("cp310-win32", "windows"), + ("cp311-macosx_x86_64", "macos"), + ), +) +def test_only_argument(intercepted_build_args, monkeypatch, only, plat): + monkeypatch.setenv("CIBW_BUILD", "unused") + monkeypatch.setenv("CIBW_SKIP", "unused") + monkeypatch.setattr(sys, "argv", [*sys.argv, "--only", only]) + + main() + + options = intercepted_build_args.args[0] + assert options.globals.build_selector.build_config == only + assert options.globals.build_selector.skip_config == "" + assert options.platform == plat + assert options.globals.architectures == Architecture.all_archs(plat) + assert EnableGroup.PyPy in options.globals.build_selector.enable + + +@pytest.mark.parametrize("only", ("cp311-manylxinux_x86_64", "some_linux_thing")) +def test_only_failed(monkeypatch, only): + monkeypatch.setattr(sys, "argv", [*sys.argv, "--only", only]) + + with pytest.raises(SystemExit): + main() + + +def test_only_no_platform(monkeypatch): + monkeypatch.setattr( + sys, "argv", [*sys.argv, "--only", "cp311-manylinux_x86_64", "--platform", "macos"] + ) + + with pytest.raises(SystemExit): + main() + + +def test_only_no_archs(monkeypatch): + monkeypatch.setattr( + sys, "argv", [*sys.argv, "--only", "cp311-manylinux_x86_64", "--archs", "x86_64"] + ) + + with pytest.raises(SystemExit): + main() + + +@pytest.mark.parametrize( + ("envvar_name", "envvar_value"), + ( + ("CIBW_BUILD", "cp310-*"), + ("CIBW_SKIP", "cp311-*"), + ("CIBW_ARCHS", "auto32"), + ("CIBW_PLATFORM", "macos"), + ), +) +def test_only_overrides_env_vars(monkeypatch, intercepted_build_args, envvar_name, envvar_value): + monkeypatch.setattr(sys, "argv", [*sys.argv, "--only", "cp311-manylinux_x86_64"]) + monkeypatch.setenv(envvar_name, envvar_value) + + main() + + options = intercepted_build_args.args[0] + assert options.globals.build_selector.build_config == "cp311-manylinux_x86_64" + assert options.globals.build_selector.skip_config == "" + assert options.platform == "linux" + assert options.globals.architectures == Architecture.all_archs("linux") + + +def test_pyodide_on_windows(monkeypatch, capsys): + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr(sys, "argv", [*sys.argv, "--only", "cp312-pyodide_wasm32"]) + + with pytest.raises(SystemExit) as exit: + main() + + _, err = capsys.readouterr() + + assert exit.value.code == 2 + assert "Building for pyodide is not supported on Windows" in err diff --git a/unit_test/main_tests/main_requires_python_test.py b/unit_test/main_tests/main_requires_python_test.py index 0edb09375..2fff52750 100644 --- a/unit_test/main_tests/main_requires_python_test.py +++ b/unit_test/main_tests/main_requires_python_test.py @@ -7,7 +7,7 @@ from cibuildwheel.__main__ import main -@pytest.fixture(autouse=True, scope="function") +@pytest.fixture(autouse=True) def fake_package_dir(monkeypatch, tmp_path): """ Set up a fake project @@ -23,8 +23,8 @@ def fake_package_dir(monkeypatch, tmp_path): return local_path -def test_no_override(platform, monkeypatch, intercepted_build_args): - +@pytest.mark.usefixtures("platform") +def test_no_override(intercepted_build_args): main() options = intercepted_build_args.args[0] @@ -36,7 +36,8 @@ def test_no_override(platform, monkeypatch, intercepted_build_args): assert intercepted_build_selector.requires_python is None -def test_override_env(platform, monkeypatch, intercepted_build_args): +@pytest.mark.usefixtures("platform") +def test_override_env(monkeypatch, intercepted_build_args): monkeypatch.setenv("CIBW_PROJECT_REQUIRES_PYTHON", ">=3.8") main() @@ -50,8 +51,8 @@ def test_override_env(platform, monkeypatch, intercepted_build_args): assert not intercepted_build_selector("cp36-win32") -def test_override_setup_cfg(platform, monkeypatch, intercepted_build_args, fake_package_dir): - +@pytest.mark.usefixtures("platform") +def test_override_setup_cfg(intercepted_build_args, fake_package_dir): fake_package_dir.joinpath("setup.cfg").write_text( textwrap.dedent( """ @@ -72,8 +73,8 @@ def test_override_setup_cfg(platform, monkeypatch, intercepted_build_args, fake_ assert not intercepted_build_selector("cp36-win32") -def test_override_pyproject_toml(platform, monkeypatch, intercepted_build_args, fake_package_dir): - +@pytest.mark.usefixtures("platform") +def test_override_pyproject_toml(intercepted_build_args, fake_package_dir): fake_package_dir.joinpath("pyproject.toml").write_text( textwrap.dedent( """ @@ -94,8 +95,8 @@ def test_override_pyproject_toml(platform, monkeypatch, intercepted_build_args, assert not intercepted_build_selector("cp36-win32") -def test_override_setup_py_simple(platform, monkeypatch, intercepted_build_args, fake_package_dir): - +@pytest.mark.usefixtures("platform") +def test_override_setup_py_simple(intercepted_build_args, fake_package_dir): fake_package_dir.joinpath("setup.py").write_text( textwrap.dedent( """ diff --git a/unit_test/oci_container_test.py b/unit_test/oci_container_test.py new file mode 100644 index 000000000..373144a1b --- /dev/null +++ b/unit_test/oci_container_test.py @@ -0,0 +1,676 @@ +import json +import os +import platform +import random +import shutil +import subprocess +import sys +import textwrap +import time +from contextlib import nullcontext +from pathlib import Path, PurePath, PurePosixPath + +import pytest +import tomli_w + +import cibuildwheel.oci_container +from cibuildwheel.ci import CIProvider, detect_ci_provider +from cibuildwheel.environment import EnvironmentAssignmentBash +from cibuildwheel.errors import OCIEngineTooOldError +from cibuildwheel.oci_container import ( + OCIContainer, + OCIContainerEngineConfig, + OCIPlatform, + _check_engine_version, +) + +# Test utilities + +# for these tests we use manylinux2014 images, because they're available on +# multi architectures and include python3.8 +DEFAULT_IMAGE = "quay.io/pypa/manylinux2014:2025.03.08-1" +pm = platform.machine() +DEFAULT_OCI_PLATFORM = { + "AMD64": OCIPlatform.AMD64, + "x86_64": OCIPlatform.AMD64, + "ppc64le": OCIPlatform.PPC64LE, + "s390x": OCIPlatform.S390X, + "aarch64": OCIPlatform.ARM64, + "arm64": OCIPlatform.ARM64, + "ARM64": OCIPlatform.ARM64, +}[pm] + +PODMAN = OCIContainerEngineConfig(name="podman") + + +@pytest.fixture(params=["docker", "podman"], scope="module") +def container_engine(request): + if request.param == "docker" and not request.config.getoption("--run-docker"): + pytest.skip("need --run-docker option to run") + if request.param == "podman" and not request.config.getoption("--run-podman"): + pytest.skip("need --run-podman option to run") + + def get_images() -> set[str]: + if detect_ci_provider() is None: + return set() + images = subprocess.run( + [request.param, "image", "ls", "--format", "{{json .ID}}"], + text=True, + check=True, + stdout=subprocess.PIPE, + ).stdout + return {json.loads(image.strip()) for image in images.splitlines() if image.strip()} + + images_before = get_images() + try: + yield OCIContainerEngineConfig(name=request.param) + finally: + images_after = get_images() + for image in images_after - images_before: + subprocess.run([request.param, "rmi", image], check=False) + + +# Tests + + +def test_simple(container_engine): + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + assert container.call(["echo", "hello"], capture_output=True) == "hello\n" + + +def test_no_lf(container_engine): + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + assert container.call(["printf", "hello"], capture_output=True) == "hello" + + +def test_debug_info(container_engine): + container = OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) + print(container.debug_info()) + with container: + pass + + +def test_environment(container_engine): + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + assert ( + container.call( + ["sh", "-c", "echo $TEST_VAR"], env={"TEST_VAR": "1"}, capture_output=True + ) + == "1\n" + ) + + +def test_environment_pass(container_engine, monkeypatch): + monkeypatch.setenv("CIBUILDWHEEL", "1") + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1489957071") + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + assert container.call(["sh", "-c", "echo $CIBUILDWHEEL"], capture_output=True) == "1\n" + assert ( + container.call(["sh", "-c", "echo $SOURCE_DATE_EPOCH"], capture_output=True) + == "1489957071\n" + ) + + +def test_cwd(container_engine): + with OCIContainer( + engine=container_engine, + image=DEFAULT_IMAGE, + oci_platform=DEFAULT_OCI_PLATFORM, + cwd="/cibuildwheel/working_directory", + ) as container: + assert container.call(["pwd"], capture_output=True) == "/cibuildwheel/working_directory\n" + assert container.call(["pwd"], capture_output=True, cwd="/opt") == "/opt\n" + + +def test_container_removed(container_engine): + # test is flaky on some platforms, implement retry for 5 second + timeout = 50 # * 100 ms = 5s + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + assert container.name is not None + container_name = container.name + for _ in range(timeout): + docker_containers_listing = subprocess.run( + f"{container.engine.name} container ls", + shell=True, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout + if container_name in docker_containers_listing: + break + time.sleep(0.1) + assert container_name in docker_containers_listing + + for _ in range(timeout): + docker_containers_listing = subprocess.run( + f"{container.engine.name} container ls", + shell=True, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout + if container_name not in docker_containers_listing: + break + time.sleep(0.1) + assert container_name not in docker_containers_listing + + +def test_large_environment(container_engine): + # max environment variable size is 128kB + long_env_var_length = 127 * 1024 + large_environment = { + "a": "0" * long_env_var_length, + "b": "0" * long_env_var_length, + "c": "0" * long_env_var_length, + "d": "0" * long_env_var_length, + } + + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + # check the length of d + assert ( + container.call(["sh", "-c", "echo ${#d}"], env=large_environment, capture_output=True) + == f"{long_env_var_length}\n" + ) + + +def test_binary_output(container_engine): + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + # note: the below embedded snippets are in python2 + + # check that we can pass though arbitrary binary data without erroring + container.call( + [ + "/usr/bin/python2", + "-c", + textwrap.dedent( + """ + import sys + sys.stdout.write(''.join(chr(n) for n in range(0, 256))) + """ + ), + ] + ) + + # check that we can capture arbitrary binary data + output = container.call( + [ + "/usr/bin/python2", + "-c", + textwrap.dedent( + """ + import sys + sys.stdout.write(''.join(chr(n % 256) for n in range(0, 512))) + """ + ), + ], + capture_output=True, + ) + + data = bytes(output, encoding="utf8", errors="surrogateescape") + + for i in range(512): + assert data[i] == i % 256 + + # check that environment variables can carry binary data, except null characters + # (https://www.gnu.org/software/libc/manual/html_node/Environment-Variables.html) + binary_data = bytes(n for n in range(1, 256)) + binary_data_string = str(binary_data, encoding="utf8", errors="surrogateescape") + output = container.call( + ["python2", "-c", 'import os, sys; sys.stdout.write(os.environ["TEST_VAR"])'], + env={"TEST_VAR": binary_data_string}, + capture_output=True, + ) + assert output == binary_data_string + + +@pytest.mark.parametrize( + "file_path", + ["test.dat", "path/to/test.dat"], +) +def test_file_operation( + tmp_path: Path, container_engine: OCIContainerEngineConfig, file_path: str +) -> None: + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + # test copying a file in + test_binary_data = bytes(random.randrange(256) for _ in range(1000)) + original_test_file = tmp_path / file_path + original_test_file.parent.mkdir(parents=True, exist_ok=True) + original_test_file.write_bytes(test_binary_data) + + dst_file = PurePath("/tmp") / file_path + + container.copy_into(original_test_file, dst_file) + + owner = container.call(["stat", "-c", "%u:%g", dst_file], capture_output=True).strip() + assert owner == "0:0" + + output = container.call(["cat", dst_file], capture_output=True) + assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape") + + +def test_dir_operations(tmp_path: Path, container_engine: OCIContainerEngineConfig) -> None: + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + test_binary_data = bytes(random.randrange(256) for _ in range(1000)) + original_test_file = tmp_path / "test.dat" + original_test_file.write_bytes(test_binary_data) + + # test copying a dir in + test_dir = tmp_path / "test_dir" + test_dir.mkdir() + test_file = test_dir / "test.dat" + shutil.copyfile(original_test_file, test_file) + + dst_dir = PurePosixPath("/tmp/test_dir") + dst_file = dst_dir / "test.dat" + container.copy_into(test_dir, dst_dir) + + owner = container.call(["stat", "-c", "%u:%g", dst_dir], capture_output=True).strip() + assert owner == "0:0" + + owner = container.call(["stat", "-c", "%u:%g", dst_file], capture_output=True).strip() + assert owner == "0:0" + + output = container.call(["cat", dst_file], capture_output=True) + assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape") + + # test glob + assert container.glob(dst_dir, "*.dat") == [dst_file] + + # test copy dir out + new_test_dir = tmp_path / "test_dir_new" + container.copy_out(dst_dir, new_test_dir) + + assert os.getuid() == new_test_dir.stat().st_uid + assert os.getgid() == new_test_dir.stat().st_gid + assert os.getuid() == (new_test_dir / "test.dat").stat().st_uid + assert os.getgid() == (new_test_dir / "test.dat").stat().st_gid + + assert test_binary_data == (new_test_dir / "test.dat").read_bytes() + + +def test_environment_executor(container_engine: OCIContainerEngineConfig) -> None: + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + assignment = EnvironmentAssignmentBash("TEST=$(echo 42)") + assert assignment.evaluated_value({}, container.environment_executor) == "42" + + +def test_podman_vfs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, container_engine: OCIContainerEngineConfig +) -> None: + if container_engine.name != "podman": + pytest.skip("only runs with podman") + if sys.platform.startswith("darwin"): + pytest.skip("Skipping test because podman on this platform does not support vfs") + + # create the VFS configuration + vfs_path = tmp_path / "podman_vfs" + vfs_path.mkdir() + + # This requires that we write configuration files and point to them + # with environment variables before we run podman + # https://github.com/containers/common/blob/main/docs/containers.conf.5.md + vfs_containers_conf_data = { + "containers": { + "default_capabilities": [ + "CHOWN", + "DAC_OVERRIDE", + "FOWNER", + "FSETID", + "KILL", + "NET_BIND_SERVICE", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_CHROOT", + ] + }, + "engine": {"cgroup_manager": "cgroupfs", "events_logger": "file"}, + } + # https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + storage_root = vfs_path / ".local/share/containers/vfs-storage" + run_root = vfs_path / ".local/share/containers/vfs-runroot" + storage_root.mkdir(parents=True, exist_ok=True) + run_root.mkdir(parents=True, exist_ok=True) + vfs_containers_storage_conf_data = { + "storage": { + "driver": "vfs", + "graphroot": os.fspath(storage_root), + "runroot": os.fspath(run_root), + "rootless_storage_path": os.fspath(storage_root), + "options": { + # "remap-user": "containers", + "aufs": {"mountopt": "rw"}, + "overlay": {"mountopt": "rw", "force_mask": "shared"}, + # "vfs": {"ignore_chown_errors": "true"}, + }, + } + } + + vfs_containers_conf_fpath = vfs_path / "temp_vfs_containers.conf" + vfs_containers_storage_conf_fpath = vfs_path / "temp_vfs_containers_storage.conf" + with open(vfs_containers_conf_fpath, "wb") as file: + tomli_w.dump(vfs_containers_conf_data, file) + + with open(vfs_containers_storage_conf_fpath, "wb") as file: + tomli_w.dump(vfs_containers_storage_conf_data, file) + + monkeypatch.setenv("CONTAINERS_CONF", str(vfs_containers_conf_fpath)) + monkeypatch.setenv("CONTAINERS_STORAGE_CONF", str(vfs_containers_storage_conf_fpath)) + + with OCIContainer( + engine=PODMAN, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + # test running a command + assert container.call(["echo", "hello"], capture_output=True) == "hello\n" + + # test copying a file into the container + (tmp_path / "some_file.txt").write_text("1234") + container.copy_into(tmp_path / "some_file.txt", PurePosixPath("some_file.txt")) + assert container.call(["cat", "some_file.txt"], capture_output=True) == "1234" + + # Clean up + + # When using the VFS, user is not given write permissions by default in + # new directories. As a workaround we use 'podman unshare' to delete them + # as UID 0. The reason why permission errors occur on podman is documented + # in https://podman.io/blogs/2018/10/03/podman-remove-content-homedir.html + subprocess.run(["podman", "unshare", "rm", "-rf", vfs_path], check=True) + + +def test_create_args_volume(tmp_path: Path, container_engine: OCIContainerEngineConfig) -> None: + if container_engine.name != "docker": + pytest.skip("only runs with docker") + + if "CIRCLECI" in os.environ or "GITLAB_CI" in os.environ: + pytest.skip( + "Skipping test on CircleCI/GitLab because docker there does not support --volume" + ) + + test_mount_dir = tmp_path / "test_mount" + test_mount_dir.mkdir() + (test_mount_dir / "test_file.txt").write_text("1234") + container_engine = OCIContainerEngineConfig( + name="docker", create_args=(f"--volume={test_mount_dir}:/test_mount",) + ) + + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + assert container.call(["cat", "/test_mount/test_file.txt"], capture_output=True) == "1234" + + +@pytest.mark.parametrize( + ("config", "name", "create_args"), + [ + ( + "docker", + "docker", + (), + ), + ( + "docker;create_args:", + "docker", + (), + ), + ( + "docker;create_args:--abc --def", + "docker", + ("--abc", "--def"), + ), + ( + "docker; create_args: --abc --def", + "docker", + ("--abc", "--def"), + ), + ( + "name:docker; create_args: --abc --def", + "docker", + ("--abc", "--def"), + ), + ( + 'docker; create_args: --some-option="value with spaces"', + "docker", + ("--some-option=value with spaces",), + ), + ( + 'docker; create_args: --some-option="value; with; semicolons" --another-option', + "docker", + ("--some-option=value; with; semicolons", "--another-option"), + ), + ( + "docker; create_args: --platform=linux/amd64", + "docker", + (), + ), + ( + "podman; create_args: --platform=linux/amd64", + "podman", + (), + ), + ( + "docker; create_args: --platform linux/amd64", + "docker", + (), + ), + ( + "podman; create_args: --platform linux/amd64", + "podman", + (), + ), + ], +) +def test_parse_engine_config(config, name, create_args, capsys): + engine_config = OCIContainerEngineConfig.from_config_string(config) + assert engine_config.name == name + assert engine_config.create_args == create_args + if "--platform" in config: + captured = capsys.readouterr() + assert ( + "Using '--platform' in 'container-engine::create_args' is deprecated. It will be ignored." + in captured.err + ) + + +@pytest.mark.skipif(pm != "x86_64", reason="Only runs on x86_64") +def test_enforce_32_bit(container_engine): + with OCIContainer( + engine=container_engine, image=DEFAULT_IMAGE, oci_platform=OCIPlatform.i386 + ) as container: + assert container.call(["uname", "-m"], capture_output=True).strip() == "i686" + container_args = subprocess.run( + f"{container.engine.name} inspect -f '{{{{json .Args }}}}' {container.name}", + shell=True, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout + assert json.loads(container_args) == ["/bin/bash"] + + +@pytest.mark.parametrize( + ("config", "should_have_host_mount"), + [ + ("{name}", True), + ("{name}; disable_host_mount: false", True), + ("{name}; disable_host_mount: true", False), + ], +) +def test_disable_host_mount( + tmp_path: Path, + container_engine: OCIContainerEngineConfig, + config: str, + should_have_host_mount: bool, +) -> None: + if detect_ci_provider() in {CIProvider.circle_ci, CIProvider.gitlab}: + pytest.skip("Skipping test because docker on this platform does not support host mounts") + if sys.platform.startswith("darwin"): + pytest.skip("Skipping test because docker on this platform does not support host mounts") + + engine = OCIContainerEngineConfig.from_config_string(config.format(name=container_engine.name)) + + sentinel_file = tmp_path / "sentinel" + sentinel_file.write_text("12345") + + with OCIContainer( + engine=engine, image=DEFAULT_IMAGE, oci_platform=DEFAULT_OCI_PLATFORM + ) as container: + host_mount_path = "/host" + str(sentinel_file) + if should_have_host_mount: + assert container.call(["cat", host_mount_path], capture_output=True) == "12345" + else: + with pytest.raises(subprocess.CalledProcessError): + container.call(["cat", host_mount_path], capture_output=True) + + +@pytest.mark.parametrize("platform", list(OCIPlatform)) +def test_local_image( + container_engine: OCIContainerEngineConfig, platform: OCIPlatform, tmp_path: Path +) -> None: + if ( + detect_ci_provider() == CIProvider.travis_ci + and pm != "x86_64" + and platform != DEFAULT_OCI_PLATFORM + ): + pytest.skip("Skipping test because docker on this platform does not support QEMU") + if container_engine.name == "podman" and platform == OCIPlatform.ARMV7: + # both GHA & local macOS arm64 podman desktop are failing + pytest.xfail("podman fails with armv7l images") + + remote_image = "debian:trixie-slim" + platform_name = platform.value.replace("/", "_") + local_image = f"cibw_{container_engine.name}_{platform_name}_local:latest" + dockerfile = tmp_path / "Dockerfile" + dockerfile.write_text(f"FROM {remote_image}") + subprocess.run( + [container_engine.name, "pull", f"--platform={platform.value}", remote_image], + check=True, + ) + subprocess.run( + [container_engine.name, "build", f"--platform={platform.value}", "-t", local_image, "."], + check=True, + cwd=tmp_path, + ) + with OCIContainer(engine=container_engine, image=local_image, oci_platform=platform): + pass + + +@pytest.mark.parametrize("platform", list(OCIPlatform)) +def test_multiarch_image(container_engine, platform): + if ( + detect_ci_provider() == CIProvider.travis_ci + and pm != "x86_64" + and platform != DEFAULT_OCI_PLATFORM + ): + pytest.skip("Skipping test because docker on this platform does not support QEMU") + if container_engine.name == "podman" and platform == OCIPlatform.ARMV7: + # both GHA & local macOS arm64 podman desktop are failing + pytest.xfail("podman fails with armv7l images") + with OCIContainer( + engine=container_engine, image="debian:trixie-slim", oci_platform=platform + ) as container: + output = container.call(["uname", "-m"], capture_output=True) + output_map_kernel = { + OCIPlatform.i386: ("i686",), + OCIPlatform.AMD64: ("x86_64",), + OCIPlatform.ARMV7: ("armv7l", "armv8l"), + OCIPlatform.ARM64: ("aarch64",), + OCIPlatform.PPC64LE: ("ppc64le",), + OCIPlatform.RISCV64: ("riscv64",), + OCIPlatform.S390X: ("s390x",), + } + assert output.strip() in output_map_kernel[platform] + output = container.call(["dpkg", "--print-architecture"], capture_output=True) + output_map_dpkg = { + OCIPlatform.i386: "i386", + OCIPlatform.AMD64: "amd64", + OCIPlatform.ARMV7: "armhf", + OCIPlatform.ARM64: "arm64", + OCIPlatform.PPC64LE: "ppc64el", + OCIPlatform.RISCV64: "riscv64", + OCIPlatform.S390X: "s390x", + } + assert output_map_dpkg[platform] == output.strip() + + +@pytest.mark.parametrize( + ("engine_name", "version", "context"), + [ + ( + "docker", + None, # 17.12.1-ce does supports "docker version --format '{{json . }}'" so a version before that + pytest.raises(OCIEngineTooOldError), + ), + ( + "docker", + '{"Client":{"Version":"19.03.15","ApiVersion": "1.40"},"Server":{"ApiVersion": "1.40"}}', + pytest.raises(OCIEngineTooOldError), + ), + ( + "docker", + '{"Client":{"Version":"20.10.0","ApiVersion":"1.41"},"Server":{"ApiVersion":"1.41"}}', + nullcontext(), + ), + ( + "docker", + '{"Client":{"Version":"24.0.0","ApiVersion":"1.43"},"Server":{"ApiVersion":"1.43"}}', + nullcontext(), + ), + ( + "docker", + '{"Client":{"ApiVersion":"1.43"},"Server":{"ApiVersion":"1.30"}}', + pytest.raises(OCIEngineTooOldError), + ), + ( + "docker", + '{"Client":{"ApiVersion":"1.30"},"Server":{"ApiVersion":"1.43"}}', + pytest.raises(OCIEngineTooOldError), + ), + ("podman", '{"Client":{"Version":"5.2.0"},"Server":{"Version":"5.1.2"}}', nullcontext()), + ("podman", '{"Client":{"Version":"4.9.4-rhel"}}', nullcontext()), + ( + "podman", + '{"Client":{"Version":"5.2.0"},"Server":{"Version":"2.1.2"}}', + pytest.raises(OCIEngineTooOldError), + ), + ( + "podman", + '{"Client":{"Version":"2.2.0"},"Server":{"Version":"5.1.2"}}', + pytest.raises(OCIEngineTooOldError), + ), + ("podman", '{"Client":{"Version":"3.0~rc1-rhel"}}', nullcontext()), + ("podman", '{"Client":{"Version":"2.1.0~rc1"}}', pytest.raises(OCIEngineTooOldError)), + ], +) +def test_engine_version(engine_name, version, context, monkeypatch): + def mockcall(*args, **kwargs): + if version is None: + raise subprocess.CalledProcessError(1, " ".join(str(arg) for arg in args)) + return version + + monkeypatch.setattr(cibuildwheel.oci_container, "call", mockcall) + engine = OCIContainerEngineConfig.from_config_string(engine_name) + with context: + _check_engine_version(engine) diff --git a/unit_test/option_prepare_test.py b/unit_test/option_prepare_test.py index d21ba65d5..d2a1a0826 100644 --- a/unit_test/option_prepare_test.py +++ b/unit_test/option_prepare_test.py @@ -1,23 +1,27 @@ import platform as platform_module import subprocess import sys +import typing from contextlib import contextmanager -from pathlib import Path -from typing import cast +from pathlib import PurePosixPath from unittest import mock import pytest -from cibuildwheel import linux, util +from cibuildwheel import platforms from cibuildwheel.__main__ import main +from cibuildwheel.oci_container import OCIPlatform +from cibuildwheel.util import file -ALL_IDS = {"cp36", "cp37", "cp38", "cp39", "cp310", "pp37", "pp38"} +DEFAULT_IDS = {"cp38", "cp39", "cp310", "cp311", "cp312", "cp313"} +ALL_IDS = DEFAULT_IDS | {"cp313t", "pp38", "pp39", "pp310", "pp311", "gp311_242"} @pytest.fixture -def mock_build_docker(monkeypatch): +def mock_build_container(monkeypatch): def fail_on_call(*args, **kwargs): - raise RuntimeError("This should never be called") + msg = "This should never be called" + raise RuntimeError(msg) def ignore_call(*args, **kwargs): pass @@ -34,59 +38,64 @@ def ignore_context_call(*args, **kwargs): monkeypatch.setattr(subprocess, "Popen", fail_on_call) monkeypatch.setattr(subprocess, "run", ignore_call) - monkeypatch.setattr(util, "download", fail_on_call) - monkeypatch.setattr("cibuildwheel.linux.DockerContainer", ignore_context_call) + monkeypatch.setattr(file, "download", fail_on_call) + monkeypatch.setattr("cibuildwheel.platforms.linux.OCIContainer", ignore_context_call) - monkeypatch.setattr("cibuildwheel.linux.build_on_docker", mock.Mock(spec=linux.build_on_docker)) - monkeypatch.setattr("cibuildwheel.util.print_new_wheels", ignore_context_call) + monkeypatch.setattr( + "cibuildwheel.platforms.linux.build_in_container", + mock.Mock(spec=platforms.linux.build_in_container), + ) + monkeypatch.setattr("cibuildwheel.__main__.print_new_wheels", ignore_context_call) -def test_build_default_launches(mock_build_docker, fake_package_dir, monkeypatch): - monkeypatch.setattr(sys, "argv", ["cibuildwheel", "--platform=linux"]) +@pytest.mark.usefixtures("mock_build_container", "fake_package_dir") +def test_build_default_launches(monkeypatch): + monkeypatch.setattr(sys, "argv", [*sys.argv, "--platform=linux"]) + monkeypatch.setenv("CIBW_ARCHS", "auto64 auto32") + monkeypatch.delenv("CIBW_ENABLE", raising=False) main() - build_on_docker = cast(mock.Mock, linux.build_on_docker) + build_in_container = typing.cast(mock.Mock, platforms.linux.build_in_container) - assert build_on_docker.call_count == 4 + assert build_in_container.call_count == 4 # In Python 3.8+, this can be simplified to [0].kwargs - kwargs = build_on_docker.call_args_list[0][1] - assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert not kwargs["docker"]["simulate_32_bit"] + kwargs = build_in_container.call_args_list[0][1] + assert "quay.io/pypa/manylinux_2_28_x86_64" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.AMD64 identifiers = {x.identifier for x in kwargs["platform_configs"]} - assert identifiers == {f"{x}-manylinux_x86_64" for x in ALL_IDS} + assert identifiers == {f"{x}-manylinux_x86_64" for x in DEFAULT_IDS} - kwargs = build_on_docker.call_args_list[1][1] - assert "quay.io/pypa/manylinux2014_i686" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert kwargs["docker"]["simulate_32_bit"] + kwargs = build_in_container.call_args_list[1][1] + assert "quay.io/pypa/manylinux2014_i686" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.i386 identifiers = {x.identifier for x in kwargs["platform_configs"]} - assert identifiers == {f"{x}-manylinux_i686" for x in ALL_IDS} + assert identifiers == {f"{x}-manylinux_i686" for x in DEFAULT_IDS} - kwargs = build_on_docker.call_args_list[2][1] - assert "quay.io/pypa/musllinux_1_1_x86_64" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert not kwargs["docker"]["simulate_32_bit"] + kwargs = build_in_container.call_args_list[2][1] + assert "quay.io/pypa/musllinux_1_2_x86_64" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.AMD64 identifiers = {x.identifier for x in kwargs["platform_configs"]} - assert identifiers == { - f"{x}-musllinux_x86_64" for x in ALL_IDS for x in ALL_IDS if "pp" not in x - } + assert identifiers == {f"{x}-musllinux_x86_64" for x in DEFAULT_IDS} - kwargs = build_on_docker.call_args_list[3][1] - assert "quay.io/pypa/musllinux_1_1_i686" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert kwargs["docker"]["simulate_32_bit"] + kwargs = build_in_container.call_args_list[3][1] + assert "quay.io/pypa/musllinux_1_2_i686" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.i386 identifiers = {x.identifier for x in kwargs["platform_configs"]} - assert identifiers == {f"{x}-musllinux_i686" for x in ALL_IDS if "pp" not in x} + assert identifiers == {f"{x}-musllinux_i686" for x in DEFAULT_IDS} -def test_build_with_override_launches(mock_build_docker, monkeypatch, tmp_path): +@pytest.mark.usefixtures("mock_build_container") +def test_build_with_override_launches(monkeypatch, tmp_path): pkg_dir = tmp_path / "cibw_package" pkg_dir.mkdir() @@ -94,7 +103,10 @@ def test_build_with_override_launches(mock_build_docker, monkeypatch, tmp_path): cibw_toml.write_text( """ [tool.cibuildwheel] -manylinux-x86_64-image = "manylinux_2_24" +manylinux-x86_64-image = "manylinux_2_28" +musllinux-x86_64-image = "musllinux_1_2" +enable = ["pypy", "pypy-eol", "graalpy", "cpython-freethreading"] +archs = ["auto64", "auto32"] # Before Python 3.10, use manylinux2014 [[tool.cibuildwheel.overrides]] @@ -103,73 +115,99 @@ def test_build_with_override_launches(mock_build_docker, monkeypatch, tmp_path): manylinux-i686-image = "manylinux2014" [[tool.cibuildwheel.overrides]] -select = "cp36-manylinux_x86_64" +select = "cp38-manylinux_x86_64" before-all = "true" """ ) monkeypatch.chdir(pkg_dir) monkeypatch.setattr(sys, "argv", ["cibuildwheel", "--platform=linux"]) + monkeypatch.delenv("CIBW_ENABLE", raising=False) main() - build_on_docker = cast(mock.Mock, linux.build_on_docker) + build_in_container = typing.cast(mock.Mock, platforms.linux.build_in_container) - assert build_on_docker.call_count == 6 + assert build_in_container.call_count == 6 - kwargs = build_on_docker.call_args_list[0][1] - assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert not kwargs["docker"]["simulate_32_bit"] + kwargs = build_in_container.call_args_list[0][1] + assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.AMD64 identifiers = {x.identifier for x in kwargs["platform_configs"]} - assert identifiers == {"cp36-manylinux_x86_64"} - assert kwargs["options"].build_options("cp36-manylinux_x86_64").before_all == "true" + assert identifiers == {"cp38-manylinux_x86_64"} + assert kwargs["options"].build_options("cp38-manylinux_x86_64").before_all == "true" - kwargs = build_on_docker.call_args_list[1][1] - assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert not kwargs["docker"]["simulate_32_bit"] + kwargs = build_in_container.call_args_list[1][1] + assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.AMD64 identifiers = {x.identifier for x in kwargs["platform_configs"]} assert identifiers == { - f"{x}-manylinux_x86_64" for x in ALL_IDS - {"cp36", "cp310", "pp37", "pp38"} + f"{x}-manylinux_x86_64" + for x in ALL_IDS + - { + "cp38", + "cp310", + "cp311", + "cp312", + "cp313", + "cp313t", + "pp38", + "pp39", + "pp310", + "pp311", + "gp311_242", + } } - assert kwargs["options"].build_options("cp37-manylinux_x86_64").before_all == "" + assert kwargs["options"].build_options("cp39-manylinux_x86_64").before_all == "" - kwargs = build_on_docker.call_args_list[2][1] - assert "quay.io/pypa/manylinux_2_24_x86_64" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert not kwargs["docker"]["simulate_32_bit"] + kwargs = build_in_container.call_args_list[2][1] + assert "quay.io/pypa/manylinux_2_28_x86_64" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.AMD64 identifiers = {x.identifier for x in kwargs["platform_configs"]} assert identifiers == { - "cp310-manylinux_x86_64", - "pp37-manylinux_x86_64", - "pp38-manylinux_x86_64", + f"{x}-manylinux_x86_64" + for x in [ + "cp310", + "cp311", + "cp312", + "cp313", + "cp313t", + "pp38", + "pp39", + "pp310", + "pp311", + "gp311_242", + ] } - kwargs = build_on_docker.call_args_list[3][1] - assert "quay.io/pypa/manylinux2014_i686" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert kwargs["docker"]["simulate_32_bit"] + kwargs = build_in_container.call_args_list[3][1] + assert "quay.io/pypa/manylinux2014_i686" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.i386 identifiers = {x.identifier for x in kwargs["platform_configs"]} - assert identifiers == {f"{x}-manylinux_i686" for x in ALL_IDS} - - kwargs = build_on_docker.call_args_list[4][1] - assert "quay.io/pypa/musllinux_1_1_x86_64" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert not kwargs["docker"]["simulate_32_bit"] + assert identifiers == {f"{x}-manylinux_i686" for x in ALL_IDS if "gp" not in x} + kwargs = build_in_container.call_args_list[4][1] + assert "quay.io/pypa/musllinux_1_2_x86_64" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.AMD64 identifiers = {x.identifier for x in kwargs["platform_configs"]} assert identifiers == { - f"{x}-musllinux_x86_64" for x in ALL_IDS for x in ALL_IDS if "pp" not in x + f"{x}-musllinux_x86_64" for x in ALL_IDS if "pp" not in x and "gp" not in x } - kwargs = build_on_docker.call_args_list[5][1] - assert "quay.io/pypa/musllinux_1_1_i686" in kwargs["docker"]["docker_image"] - assert kwargs["docker"]["cwd"] == Path("/project") - assert kwargs["docker"]["simulate_32_bit"] + kwargs = build_in_container.call_args_list[5][1] + assert "quay.io/pypa/musllinux_1_2_i686" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert kwargs["container"]["oci_platform"] == OCIPlatform.i386 identifiers = {x.identifier for x in kwargs["platform_configs"]} - assert identifiers == {f"{x}-musllinux_i686" for x in ALL_IDS if "pp" not in x} + assert identifiers == { + f"{x}-musllinux_i686" for x in ALL_IDS if "pp" not in x and "gp" not in x + } diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 01a98f3b8..8bf7e5691 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -1,31 +1,49 @@ +import os import platform as platform_module +import textwrap +import unittest.mock +from pathlib import Path +from typing import Literal import pytest -from cibuildwheel.__main__ import get_build_identifiers -from cibuildwheel.environment import parse_environment -from cibuildwheel.options import Options, _get_pinned_docker_images - -from .utils import get_default_command_line_arguments +from cibuildwheel import errors +from cibuildwheel.bashlex_eval import local_environment_executor +from cibuildwheel.frontend import BuildFrontendConfig, get_build_frontend_extra_flags +from cibuildwheel.logger import Logger +from cibuildwheel.options import ( + CommandLineArguments, + Options, + _get_pinned_container_images, +) +from cibuildwheel.platforms import ALL_PLATFORM_MODULES, get_build_identifiers +from cibuildwheel.selector import EnableGroup +from cibuildwheel.util import resources +from cibuildwheel.util.packaging import DependencyConstraints PYPROJECT_1 = """ [tool.cibuildwheel] -build = ["cp38*", "cp37*"] +build = ["cp38-*", "cp313-*"] +skip = ["*musllinux*"] +archs = ["auto64", "auto32"] environment = {FOO="BAR"} test-command = "pyproject" +test-sources = ["test", "other dir"] -manylinux-x86_64-image = "manylinux1" +manylinux-x86_64-image = "manylinux_2_28" environment-pass = ["EXAMPLE_ENV"] +pyodide-version = "0.27.6" + [tool.cibuildwheel.macos] test-requires = "else" [[tool.cibuildwheel.overrides]] -select = "cp37*" +select = "cp313-*" test-command = "pyproject-override" -manylinux-x86_64-image = "manylinux2014" +manylinux-x86_64-image = "manylinux_2_34" """ @@ -33,56 +51,60 @@ def test_options_1(tmp_path, monkeypatch): with tmp_path.joinpath("pyproject.toml").open("w") as f: f.write(PYPROJECT_1) - args = get_default_command_line_arguments() - args.package_dir = str(tmp_path) + args = CommandLineArguments.defaults() + args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") - options = Options(platform="linux", command_line_arguments=args) + options = Options(platform="linux", command_line_arguments=args, env={}) + module = ALL_PLATFORM_MODULES["linux"] identifiers = get_build_identifiers( - platform="linux", + platform_module=module, build_selector=options.globals.build_selector, architectures=options.globals.architectures, ) override_display = """\ -test_command: 'pyproject' - cp37-manylinux_x86_64: 'pyproject-override'""" - + *: pyproject + cp313-manylinux_x86_64, cp313-manylinux_i686: pyproject-override""" print(options.summary(identifiers)) assert override_display in options.summary(identifiers) default_build_options = options.build_options(identifier=None) - assert default_build_options.environment == parse_environment('FOO="BAR"') + assert default_build_options.environment.as_dictionary(prev_environment={}) == {"FOO": "BAR"} - all_pinned_docker_images = _get_pinned_docker_images() - pinned_x86_64_docker_image = all_pinned_docker_images["x86_64"] + all_pinned_container_images = _get_pinned_container_images() + pinned_x86_64_container_image = all_pinned_container_images["x86_64"] local = options.build_options("cp38-manylinux_x86_64") assert local.manylinux_images is not None assert local.test_command == "pyproject" - assert local.manylinux_images["x86_64"] == pinned_x86_64_docker_image["manylinux1"] + assert local.test_sources == ["test", "other dir"] + assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux_2_28"] - local = options.build_options("cp37-manylinux_x86_64") + local = options.build_options("cp313-manylinux_x86_64") assert local.manylinux_images is not None assert local.test_command == "pyproject-override" - assert local.manylinux_images["x86_64"] == pinned_x86_64_docker_image["manylinux2014"] + assert local.test_sources == ["test", "other dir"] + assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux_2_34"] + + local = options.build_options("cp312-pyodide_wasm32") + assert local.pyodide_version == "0.27.6" def test_passthrough(tmp_path, monkeypatch): with tmp_path.joinpath("pyproject.toml").open("w") as f: f.write(PYPROJECT_1) - args = get_default_command_line_arguments() - args.package_dir = str(tmp_path) + args = CommandLineArguments.defaults() + args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") - monkeypatch.setenv("EXAMPLE_ENV", "ONE") - options = Options(platform="linux", command_line_arguments=args) + options = Options(platform="linux", command_line_arguments=args, env={"EXAMPLE_ENV": "ONE"}) default_build_options = options.build_options(identifier=None) @@ -104,13 +126,529 @@ def test_passthrough(tmp_path, monkeypatch): ], ) def test_passthrough_evil(tmp_path, monkeypatch, env_var_value): - args = get_default_command_line_arguments() - args.package_dir = str(tmp_path) + args = CommandLineArguments.defaults() + args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") - monkeypatch.setenv("CIBW_ENVIRONMENT_PASS_LINUX", "ENV_VAR") - options = Options(platform="linux", command_line_arguments=args) + options = Options( + platform="linux", + command_line_arguments=args, + env={"CIBW_ENVIRONMENT_PASS_LINUX": "ENV_VAR", "ENV_VAR": env_var_value}, + ) - monkeypatch.setenv("ENV_VAR", env_var_value) parsed_environment = options.build_options(identifier=None).environment assert parsed_environment.as_dictionary(prev_environment={}) == {"ENV_VAR": env_var_value} + + +xfail_env_parse = pytest.mark.xfail( + raises=errors.ConfigurationError, + reason="until we can figure out the right way to quote these values", +) + + +@pytest.mark.parametrize( + "env_var_value", + [ + "normal value", + pytest.param('"value wrapped in quotes"', marks=[xfail_env_parse]), + pytest.param('an unclosed double-quote: "', marks=[xfail_env_parse]), + "string\nwith\ncarriage\nreturns\n", + pytest.param("a trailing backslash \\", marks=[xfail_env_parse]), + ], +) +def test_toml_environment_evil(tmp_path, env_var_value): + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + tmp_path.joinpath("pyproject.toml").write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel.environment] + EXAMPLE='''{env_var_value}''' + """ + ) + ) + + options = Options(platform="linux", command_line_arguments=args, env={}) + parsed_environment = options.build_options(identifier=None).environment + assert parsed_environment.as_dictionary(prev_environment={}) == {"EXAMPLE": env_var_value} + + +@pytest.mark.parametrize( + ("toml_assignment", "result_value"), + [ + ('TEST_VAR="simple_value"', "simple_value"), + # spaces + ('TEST_VAR="simple value"', "simple value"), + # env var + ('TEST_VAR="$PARAM"', "spam"), + ('TEST_VAR="$PARAM $PARAM"', "spam spam"), + # env var extension + ('TEST_VAR="before:$PARAM:after"', "before:spam:after"), + # env var extension with spaces + ('TEST_VAR="before $PARAM after"', "before spam after"), + # literal $ - this test is just for reference, I'm not sure if this + # syntax will work if we change the TOML quoting behaviour + (r'TEST_VAR="before\\$after"', "before$after"), + ], +) +def test_toml_environment_quoting(tmp_path: Path, toml_assignment: str, result_value: str) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + tmp_path.joinpath("pyproject.toml").write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel.environment] + {toml_assignment} + """ + ) + ) + + options = Options(platform="linux", command_line_arguments=args, env={}) + parsed_environment = options.build_options(identifier=None).environment + environment_values = parsed_environment.as_dictionary( + prev_environment={**os.environ, "PARAM": "spam"}, + executor=local_environment_executor, + ) + + assert environment_values["TEST_VAR"] == result_value + + +@pytest.mark.parametrize( + ("toml_assignment", "result_name", "result_create_args", "result_disable_host_mount"), + [ + ( + 'container-engine = "podman"', + "podman", + (), + False, + ), + ( + 'container-engine = {name = "podman"}', + "podman", + (), + False, + ), + ( + 'container-engine = "docker; create_args: --some-option"', + "docker", + ("--some-option",), + False, + ), + ( + 'container-engine = {name = "docker", create-args = ["--some-option"]}', + "docker", + ("--some-option",), + False, + ), + ( + 'container-engine = {name = "docker", create-args = ["--some-option", "value that contains spaces"]}', + "docker", + ("--some-option", "value that contains spaces"), + False, + ), + ( + 'container-engine = {name = "docker", create-args = ["--some-option", "value;that;contains;semicolons"]}', + "docker", + ("--some-option", "value;that;contains;semicolons"), + False, + ), + ( + 'container-engine = {name = "docker", disable-host-mount = true}', + "docker", + (), + True, + ), + ( + 'container-engine = {name = "docker", disable_host_mount = true}', + "docker", + (), + True, + ), + ], +) +def test_container_engine_option( + tmp_path: Path, + toml_assignment: str, + result_name: str, + result_create_args: tuple[str, ...], + result_disable_host_mount: bool, +) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + tmp_path.joinpath("pyproject.toml").write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {toml_assignment} + """ + ) + ) + + options = Options(platform="linux", command_line_arguments=args, env={}) + parsed_container_engine = options.build_options(None).container_engine + + assert parsed_container_engine.name == result_name + assert parsed_container_engine.create_args == result_create_args + assert parsed_container_engine.disable_host_mount == result_disable_host_mount + + +def test_environment_pass_references(): + options = Options( + platform="linux", + command_line_arguments=CommandLineArguments.defaults(), + env={ + "CIBW_ENVIRONMENT_PASS_LINUX": "STARTER MAIN_COURSE", + "STARTER": "green eggs", + "MAIN_COURSE": "ham", + "CIBW_ENVIRONMENT": 'MEAL="$STARTER and $MAIN_COURSE"', + }, + ) + parsed_environment = options.build_options(identifier=None).environment + assert parsed_environment.as_dictionary(prev_environment={}) == { + "MEAL": "green eggs and ham", + "STARTER": "green eggs", + "MAIN_COURSE": "ham", + } + + +@pytest.mark.parametrize( + ("toml_assignment", "result_name", "result_args"), + [ + ( + "", + None, + None, + ), + ( + 'build-frontend = "build"', + "build", + [], + ), + ( + 'build-frontend = {name = "build"}', + "build", + [], + ), + ( + 'build-frontend = "pip; args: --some-option"', + "pip", + ["--some-option"], + ), + ( + 'build-frontend = {name = "pip", args = ["--some-option"]}', + "pip", + ["--some-option"], + ), + ], +) +def test_build_frontend_option( + tmp_path: Path, toml_assignment: str, result_name: str, result_args: list[str] +) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + tmp_path.joinpath("pyproject.toml").write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {toml_assignment} + """ + ) + ) + + options = Options(platform="linux", command_line_arguments=args, env={}) + parsed_build_frontend = options.build_options(identifier=None).build_frontend + + if toml_assignment: + assert parsed_build_frontend is not None + assert parsed_build_frontend.name == result_name + assert parsed_build_frontend.args == result_args + else: + assert parsed_build_frontend is None + + +def test_override_inherit_environment(tmp_path: Path) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + """\ + [tool.cibuildwheel] + environment = {FOO="BAR", "HAM"="EGGS"} + + [[tool.cibuildwheel.overrides]] + select = "cp37*" + inherit.environment = "append" + environment = {FOO="BAZ", "PYTHON"="MONTY"} + """ + ) + ) + + options = Options(platform="linux", command_line_arguments=args, env={}) + parsed_environment = options.build_options(identifier=None).environment + assert parsed_environment.as_dictionary(prev_environment={}) == { + "FOO": "BAR", + "HAM": "EGGS", + } + + assert options.build_options("cp37-manylinux_x86_64").environment.as_dictionary( + prev_environment={} + ) == { + "FOO": "BAZ", + "HAM": "EGGS", + "PYTHON": "MONTY", + } + + +def test_override_inherit_environment_with_references(tmp_path: Path) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + """\ + [tool.cibuildwheel] + environment = {PATH="/opt/bin:$PATH"} + + [[tool.cibuildwheel.overrides]] + select = "cp37*" + inherit.environment = "append" + environment = {PATH="/opt/local/bin:$PATH"} + """ + ) + ) + + options = Options(platform="linux", command_line_arguments=args, env={"MONTY": "PYTHON"}) + parsed_environment = options.build_options(identifier=None).environment + prev_environment = {"PATH": "/usr/bin:/bin"} + assert parsed_environment.as_dictionary(prev_environment=prev_environment) == { + "PATH": "/opt/bin:/usr/bin:/bin", + } + + assert options.build_options("cp37-manylinux_x86_64").environment.as_dictionary( + prev_environment=prev_environment + ) == { + "PATH": "/opt/local/bin:/opt/bin:/usr/bin:/bin", + } + + +@pytest.mark.parametrize( + ("toml_assignment", "env", "enable_args", "expected_result"), + [ + ("", {}, [], False), + ("enable = ['cpython-freethreading']", {}, [], True), + ("enable = []", {}, [], False), + ("", {}, ["cpython-freethreading"], True), + ("", {}, ["cpython-freethreading", "pypy"], True), + ("", {"CIBW_ENABLE": "pypy"}, [], False), + ("", {"CIBW_ENABLE": "cpython-freethreading"}, [], True), + ("enable = []", {"CIBW_ENABLE": "cpython-freethreading"}, [], True), + ("enable = ['cpython-freethreading']", {"CIBW_ENABLE": "pypy"}, [], True), + ("enable = ['cpython-freethreading']", {}, ["pypy"], True), + ("enable = ['cpython-freethreading']", {"CIBW_ENABLE": ""}, [], True), + ("enable = []", {"CIBW_ENABLE": ""}, [], False), + ], +) +def test_free_threaded_support( + tmp_path: Path, + toml_assignment: str, + env: dict[str, str], + enable_args: list[str], + expected_result: bool, +) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + args.enable = enable_args + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {toml_assignment} + """ + ) + ) + options = Options(platform="linux", command_line_arguments=args, env=env) + if expected_result: + assert EnableGroup.CPythonFreeThreading in options.globals.build_selector.enable + else: + assert EnableGroup.CPythonFreeThreading not in options.globals.build_selector.enable + + +@pytest.mark.parametrize( + ("toml_assignment", "base_file_path", "packages"), + [ + ("", resources.CONSTRAINTS, []), + ("dependency-versions = 'pinned'", resources.CONSTRAINTS, []), + ("dependency-versions = 'latest'", None, []), + ("dependency-versions = 'constraints file.txt'", Path("constraints file.txt"), []), + ( + "dependency-versions = \"file:'constraints file.txt'\"", + Path("constraints file.txt"), + [], + ), + ( + "dependency-versions = {file = 'constraints file.txt'}", + Path("constraints file.txt"), + [], + ), + ( + "dependency-versions = 'packages: foo==1.2.3 bar==4.5.6'", + None, + ["foo==1.2.3", "bar==4.5.6"], + ), + ], +) +def test_dependency_versions_toml( + tmp_path: Path, + toml_assignment: str, + base_file_path: Path | None, + packages: list[str] | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + (tmp_path / "constraints file.txt").write_text("") + monkeypatch.chdir(tmp_path) + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {toml_assignment} + """ + ) + ) + + options = Options(platform="linux", command_line_arguments=args, env={}) + parsed_dependency_constraints = options.build_options(None).dependency_constraints + if base_file_path is None and packages is None: + assert parsed_dependency_constraints == DependencyConstraints.latest() + else: + if parsed_dependency_constraints.base_file_path and base_file_path: + assert parsed_dependency_constraints.base_file_path.samefile(base_file_path) + else: + assert parsed_dependency_constraints.base_file_path == base_file_path + assert parsed_dependency_constraints.packages == packages + + +@pytest.mark.parametrize( + ("image", "deprecated", "raises"), + [ + ("manylinux1", True, True), + ("manylinux2010", True, True), + ("manylinux2014", False, False), + ("manylinux_2_24", True, True), + ("manylinux_2_28", False, False), + ("manylinux_2_34", False, False), + ("musllinux_1_1", True, True), + ("musllinux_1_2", False, False), + ], +) +def test_deprecated_image( + image: str, deprecated: bool, raises: bool, capsys: pytest.CaptureFixture[str] +) -> None: + assert not raises or deprecated + args = CommandLineArguments.defaults() + env = { + "CIBW_ARCHS": "x86_64", + "CIBW_MANYLINUX_X86_64_IMAGE": image if image.startswith("manylinux") else "", + "CIBW_MUSLLINUX_X86_64_IMAGE": image if image.startswith("musllinux") else "", + } + options = Options(platform="linux", command_line_arguments=args, env=env) + try: + bo = options.build_options(None) + assert not raises + except errors.DeprecationError: + assert raises + return + images = bo.manylinux_images if image.startswith("manylinux") else bo.musllinux_images + assert images is not None + resolved_image = images["x86_64"] + captured = capsys.readouterr() + if deprecated: + assert f"Deprecated image {image!r}" in captured.err + assert f"{resolved_image!r}" in captured.err + else: + assert "Deprecated image" not in captured.err + + +@pytest.mark.parametrize( + ("frontend", "verbosity", "result"), + [ + ("pip", 3, ["-Ca", "-Cb", "-1", "-vvv"]), + ("pip", 2, ["-Ca", "-Cb", "-1", "-vv"]), + ("pip", -1, ["-Ca", "-Cb", "-1", "-q"]), + ("build", 0, ["-Ca", "-Cb", "-1"]), + ("build", 1, ["-Ca", "-Cb", "-1"]), + ("build", 2, ["-Ca", "-Cb", "-1", "-v"]), + ("build", 3, ["-Ca", "-Cb", "-1", "-vv"]), + ("build[uv]", 3, ["-Ca", "-Cb", "-1", "-vv"]), + ], +) +def test_get_build_frontend_extra_flags( + frontend: Literal["pip", "build", "build[uv]"], + verbosity: int, + result: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + mock_warning = unittest.mock.MagicMock() + monkeypatch.setattr(Logger, "warning", mock_warning) + build_frontend = BuildFrontendConfig(frontend, ["-1"]) + args = get_build_frontend_extra_flags( + build_frontend=build_frontend, verbosity_level=verbosity, config_settings="a b" + ) + + assert args == result + mock_warning.assert_not_called() + + +@pytest.mark.parametrize("frontend", ["build", "build[uv]"]) +def test_get_build_frontend_extra_flags_warning( + frontend: Literal["build", "build[uv]"], monkeypatch: pytest.MonkeyPatch +) -> None: + mock_warning = unittest.mock.MagicMock() + monkeypatch.setattr(Logger, "warning", mock_warning) + build_frontend = BuildFrontendConfig(frontend, ["-1"]) + args = get_build_frontend_extra_flags( + build_frontend=build_frontend, verbosity_level=-1, config_settings="a b" + ) + assert args == ["-Ca", "-Cb", "-1"] + mock_warning.assert_called_once() + + +@pytest.mark.parametrize( + ("definition", "expected"), + [ + ("", None), + ("xbuild-tools = []", []), + ('xbuild-tools = ["cmake", "rustc"]', ["cmake", "rustc"]), + ], +) +def test_xbuild_tools_handling(tmp_path: Path, definition: str, expected: list[str] | None) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {definition} + """ + ) + ) + + options = Options(platform="ios", command_line_arguments=args, env={}) + + local = options.build_options("cp313-ios_13_0_arm64_iphoneos") + assert local.xbuild_tools == expected diff --git a/unit_test/options_toml_test.py b/unit_test/options_toml_test.py index 1dcfdc212..6ec634152 100644 --- a/unit_test/options_toml_test.py +++ b/unit_test/options_toml_test.py @@ -1,19 +1,31 @@ +import shlex from pathlib import Path import pytest -from cibuildwheel.options import ConfigOptionError, OptionsReader, _dig_first +from cibuildwheel.options import ( + EnvironmentFormat, + InheritRule, + ListFormat, + OptionsReader, + OptionsReaderError, + ShlexTableFormat, + _resolve_cascade, +) PYPROJECT_1 = """ [tool.cibuildwheel] build = "cp39*" environment = {THING = "OTHER", FOO="BAR"} +xbuild-tools = ["first"] test-command = "pyproject" test-requires = "something" test-extras = ["one", "two"] +test-groups = ["three", "four"] +test-sources = ["five", "six and seven"] -manylinux-x86_64-image = "manylinux1" +manylinux-x86_64-image = "manylinux_2_28" [tool.cibuildwheel.macos] test-requires = "else" @@ -33,64 +45,83 @@ def test_simple_settings(tmp_path, platform, fname): config_file_path: Path = tmp_path / fname config_file_path.write_text(PYPROJECT_1) - options_reader = OptionsReader(config_file_path, platform=platform) + options_reader = OptionsReader(config_file_path, platform=platform, env={}) - assert options_reader.get("build", env_plat=False, sep=" ") == "cp39*" + assert options_reader.get("build", option_format=ListFormat(" "), env_plat=False) == "cp39*" assert options_reader.get("test-command") == "pyproject" - assert options_reader.get("archs", sep=" ") == "auto" + assert options_reader.get("archs", option_format=ListFormat(" ")) == "auto" + assert ( + options_reader.get("test-sources", option_format=ListFormat(" ", quote=shlex.quote)) + == "five 'six and seven'" + ) assert ( - options_reader.get("test-requires", sep=" ") + options_reader.get("test-requires", option_format=ListFormat(" ")) == {"windows": "something", "macos": "else", "linux": "other many"}[platform] ) # Also testing options for support for both lists and tables assert ( - options_reader.get("environment", table={"item": '{k}="{v}"', "sep": " "}) + options_reader.get("environment", option_format=EnvironmentFormat()) == 'THING="OTHER" FOO="BAR"' ) - assert ( - options_reader.get("environment", sep="x", table={"item": '{k}="{v}"', "sep": " "}) - == 'THING="OTHER" FOO="BAR"' - ) - assert options_reader.get("test-extras", sep=",") == "one,two" - assert ( - options_reader.get("test-extras", sep=",", table={"item": '{k}="{v}"', "sep": " "}) - == "one,two" - ) + assert options_reader.get("test-extras", option_format=ListFormat(",")) == "one,two" + assert options_reader.get("test-groups", option_format=ListFormat(" ")) == "three four" - assert options_reader.get("manylinux-x86_64-image") == "manylinux1" + assert options_reader.get("manylinux-x86_64-image") == "manylinux_2_28" assert options_reader.get("manylinux-i686-image") == "manylinux2014" - with pytest.raises(ConfigOptionError): - options_reader.get("environment", sep=" ") - - with pytest.raises(ConfigOptionError): - options_reader.get("test-extras", table={"item": '{k}="{v}"', "sep": " "}) + with pytest.raises(OptionsReaderError): + # fails because the option is a table and the option_format only works with lists + options_reader.get("environment", option_format=ListFormat(" ")) + with pytest.raises(OptionsReaderError): + # fails because the option is a list and the option_format only works with tables + options_reader.get("test-extras", option_format=ShlexTableFormat()) -def test_envvar_override(tmp_path, platform, monkeypatch): - monkeypatch.setenv("CIBW_BUILD", "cp38*") - monkeypatch.setenv("CIBW_MANYLINUX_X86_64_IMAGE", "manylinux_2_24") - monkeypatch.setenv("CIBW_TEST_COMMAND", "mytest") - monkeypatch.setenv("CIBW_TEST_REQUIRES", "docs") - monkeypatch.setenv("CIBW_TEST_REQUIRES_LINUX", "scod") +def test_envvar_override(tmp_path, platform): config_file_path: Path = tmp_path / "pyproject.toml" config_file_path.write_text(PYPROJECT_1) - options_reader = OptionsReader(config_file_path, platform=platform) + options_reader = OptionsReader( + config_file_path, + platform=platform, + env={ + "CIBW_BUILD": "cp38*", + "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux_2_24", + "CIBW_XBUILD_TOOLS": "cmake rustc", + "CIBW_TEST_COMMAND": "mytest", + "CIBW_TEST_REQUIRES": "docs", + "CIBW_TEST_GROUPS": "mgroup two", + "CIBW_TEST_REQUIRES_LINUX": "scod", + "CIBW_TEST_GROUPS_LINUX": "lgroup", + "CIBW_TEST_SOURCES": 'first "second third"', + }, + ) - assert options_reader.get("archs", sep=" ") == "auto" + assert options_reader.get("archs", option_format=ListFormat(" ")) == "auto" - assert options_reader.get("build", sep=" ") == "cp38*" + assert options_reader.get("build") == "cp38*" assert options_reader.get("manylinux-x86_64-image") == "manylinux_2_24" assert options_reader.get("manylinux-i686-image") == "manylinux2014" assert ( - options_reader.get("test-requires", sep=" ") + options_reader.get("xbuild-tools", option_format=ListFormat(" ", quote=shlex.quote)) + == "cmake rustc" + ) + assert ( + options_reader.get("test-sources", option_format=ListFormat(" ", quote=shlex.quote)) + == 'first "second third"' + ) + assert ( + options_reader.get("test-requires", option_format=ListFormat(" ")) == {"windows": "docs", "macos": "docs", "linux": "scod"}[platform] ) + assert ( + options_reader.get("test-groups", option_format=ListFormat(" ")) + == {"windows": "mgroup two", "macos": "mgroup two", "linux": "lgroup"}[platform] + ) assert options_reader.get("test-command") == "mytest" @@ -102,18 +133,18 @@ def test_project_global_override_default_platform(tmp_path, platform): repair-wheel-command = "repair-project-global" """ ) - options_reader = OptionsReader(pyproject_toml, platform=platform) + options_reader = OptionsReader(pyproject_toml, platform=platform, env={}) assert options_reader.get("repair-wheel-command") == "repair-project-global" -def test_env_global_override_default_platform(tmp_path, platform, monkeypatch): - monkeypatch.setenv("CIBW_REPAIR_WHEEL_COMMAND", "repair-env-global") - options_reader = OptionsReader(platform=platform) +def test_env_global_override_default_platform(platform): + options_reader = OptionsReader( + platform=platform, env={"CIBW_REPAIR_WHEEL_COMMAND": "repair-env-global"} + ) assert options_reader.get("repair-wheel-command") == "repair-env-global" -def test_env_global_override_project_platform(tmp_path, platform, monkeypatch): - monkeypatch.setenv("CIBW_REPAIR_WHEEL_COMMAND", "repair-env-global") +def test_env_global_override_project_platform(tmp_path, platform): pyproject_toml = tmp_path / "pyproject.toml" pyproject_toml.write_text( """ @@ -125,7 +156,13 @@ def test_env_global_override_project_platform(tmp_path, platform, monkeypatch): repair-wheel-command = "repair-project-macos" """ ) - options_reader = OptionsReader(pyproject_toml, platform=platform) + options_reader = OptionsReader( + pyproject_toml, + platform=platform, + env={ + "CIBW_REPAIR_WHEEL_COMMAND": "repair-env-global", + }, + ) assert options_reader.get("repair-wheel-command") == "repair-env-global" @@ -143,7 +180,7 @@ def test_global_platform_order(tmp_path, platform): repair-wheel-command = "repair-project-global" """ ) - options_reader = OptionsReader(pyproject_toml, platform=platform) + options_reader = OptionsReader(pyproject_toml, platform=platform, env={}) assert options_reader.get("repair-wheel-command") == f"repair-project-{platform}" @@ -158,8 +195,27 @@ def test_unexpected_key(tmp_path): """ ) - with pytest.raises(ConfigOptionError): - OptionsReader(pyproject_toml, platform="linux") + with pytest.raises(OptionsReaderError) as excinfo: + OptionsReader(pyproject_toml, platform="linux", env={}) + + assert "repair-wheel-command" in str(excinfo.value) + + +def test_underscores_in_key(tmp_path): + # Note that platform contents are only checked when running + # for that platform. + pyproject_toml = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + """ +[tool.cibuildwheel] +repair_wheel_command = "repair-project-linux" +""" + ) + + with pytest.raises(OptionsReaderError) as excinfo: + OptionsReader(pyproject_toml, platform="linux", env={}) + + assert "repair-wheel-command" in str(excinfo.value) def test_unexpected_table(tmp_path): @@ -170,8 +226,8 @@ def test_unexpected_table(tmp_path): repair-wheel-command = "repair-project-linux" """ ) - with pytest.raises(ConfigOptionError): - OptionsReader(pyproject_toml, platform="linux") + with pytest.raises(OptionsReaderError): + OptionsReader(pyproject_toml, platform="linux", env={}) def test_unsupported_join(tmp_path): @@ -182,10 +238,10 @@ def test_unsupported_join(tmp_path): build = ["1", "2"] """ ) - options_reader = OptionsReader(pyproject_toml, platform="linux") + options_reader = OptionsReader(pyproject_toml, platform="linux", env={}) - assert "1, 2" == options_reader.get("build", sep=", ") - with pytest.raises(ConfigOptionError): + assert options_reader.get("build", option_format=ListFormat(", ")) == "1, 2" + with pytest.raises(OptionsReaderError): options_reader.get("build") @@ -198,12 +254,12 @@ def test_disallowed_a(tmp_path): """ ) disallow = {"windows": {"manylinux-x86_64-image"}} - OptionsReader(pyproject_toml, platform="linux", disallow=disallow) - with pytest.raises(ConfigOptionError): - OptionsReader(pyproject_toml, platform="windows", disallow=disallow) + OptionsReader(pyproject_toml, platform="linux", disallow=disallow, env={}) + with pytest.raises(OptionsReaderError): + OptionsReader(pyproject_toml, platform="windows", disallow=disallow, env={}) -def test_environment_override_empty(tmp_path, monkeypatch): +def test_environment_override_empty(tmp_path): pyproject_toml = tmp_path / "pyproject.toml" pyproject_toml.write_text( """ @@ -213,61 +269,125 @@ def test_environment_override_empty(tmp_path, monkeypatch): """ ) - monkeypatch.setenv("CIBW_MANYLINUX_I686_IMAGE", "") - monkeypatch.setenv("CIBW_MANYLINUX_AARCH64_IMAGE", "manylinux1") - - options_reader = OptionsReader(pyproject_toml, platform="linux") + options_reader = OptionsReader( + pyproject_toml, + platform="linux", + env={ + "CIBW_MANYLINUX_I686_IMAGE": "", + "CIBW_MANYLINUX_AARCH64_IMAGE": "manylinux1", + "CIBW_XBUILD_TOOLS": "", + }, + ) assert options_reader.get("manylinux-x86_64-image") == "" assert options_reader.get("manylinux-i686-image") == "" assert options_reader.get("manylinux-aarch64-image") == "manylinux1" - assert options_reader.get("manylinux-x86_64-image", ignore_empty=True) == "manylinux2014" + assert options_reader.get("manylinux-x86_64-image", ignore_empty=True) == "manylinux_2_28" assert options_reader.get("manylinux-i686-image", ignore_empty=True) == "manylinux1" assert options_reader.get("manylinux-aarch64-image", ignore_empty=True) == "manylinux1" + assert ( + options_reader.get("xbuild-tools", option_format=ListFormat(" ", quote=shlex.quote)) == "" + ) -@pytest.mark.parametrize("ignore_empty", (True, False)) -def test_dig_first(ignore_empty): - d1 = {"random": "thing"} - d2 = {"this": "that", "empty": ""} - d3 = {"other": "hi"} - d4 = {"this": "d4", "empty": "not"} - answer = _dig_first( - (d1, "empty"), - (d2, "empty"), - (d3, "empty"), - (d4, "empty"), +@pytest.mark.parametrize("ignore_empty", [True, False], ids=["ignore_empty", "no_ignore_empty"]) +def test_resolve_cascade(ignore_empty): + answer = _resolve_cascade( + ("not", InheritRule.NONE), + (None, InheritRule.NONE), + ("", InheritRule.NONE), + (None, InheritRule.NONE), ignore_empty=ignore_empty, ) assert answer == ("not" if ignore_empty else "") - answer = _dig_first( - (d1, "this"), - (d2, "this"), - (d3, "this"), - (d4, "this"), + answer = _resolve_cascade( + ("d4", InheritRule.NONE), + (None, InheritRule.NONE), + ("that", InheritRule.NONE), + (None, InheritRule.NONE), ignore_empty=ignore_empty, ) assert answer == "that" - with pytest.raises(KeyError): - _dig_first( - (d1, "this"), - (d2, "other"), - (d3, "this"), - (d4, "other"), + with pytest.raises(ValueError, match="a setting should at least have a default value"): + _resolve_cascade( + (None, InheritRule.NONE), + (None, InheritRule.NONE), + (None, InheritRule.NONE), + (None, InheritRule.NONE), ignore_empty=ignore_empty, ) +@pytest.mark.parametrize("ignore_empty", [True, False], ids=["ignore_empty", "no_ignore_empty"]) +@pytest.mark.parametrize("rule", [InheritRule.PREPEND, InheritRule.NONE, InheritRule.APPEND]) +def test_resolve_cascade_merge_list(ignore_empty, rule): + answer = _resolve_cascade( + (["a1", "a2"], InheritRule.NONE), + ([], InheritRule.NONE), + (["b1", "b2"], rule), + (None, InheritRule.NONE), + ignore_empty=ignore_empty, + option_format=ListFormat(" "), + ) + + if not ignore_empty: + assert answer == "b1 b2" + else: + if rule == InheritRule.PREPEND: + assert answer == "b1 b2 a1 a2" + elif rule == InheritRule.NONE: + assert answer == "b1 b2" + elif rule == InheritRule.APPEND: + assert answer == "a1 a2 b1 b2" + + +@pytest.mark.parametrize("rule", [InheritRule.PREPEND, InheritRule.NONE, InheritRule.APPEND]) +def test_resolve_cascade_merge_dict(rule): + answer = _resolve_cascade( + ({"value": "a1", "base": "b1"}, InheritRule.NONE), + (None, InheritRule.NONE), + ({"value": "override"}, rule), + (None, InheritRule.NONE), + option_format=ShlexTableFormat(), + ) + + if rule == InheritRule.PREPEND: + assert answer == "value=a1 base=b1" + elif rule == InheritRule.NONE: + assert answer == "value=override" + elif rule == InheritRule.APPEND: + assert answer == "value=override base=b1" + + +def test_resolve_cascade_merge_strings(): + answer = _resolve_cascade( + ("value=a1 base=b1", InheritRule.NONE), + ("value=override", InheritRule.APPEND), + option_format=ShlexTableFormat(), + ) + assert answer == "value=override base=b1" + + +def test_resolve_cascade_merge_different_types(): + answer = _resolve_cascade( + ("value=a1 base=b1", InheritRule.NONE), + ({"value": "override"}, InheritRule.APPEND), + ("extra_string_var=c1", InheritRule.APPEND), + option_format=ShlexTableFormat(), + ) + assert answer == "value=override base=b1 extra_string_var=c1" + + PYPROJECT_2 = """ [tool.cibuildwheel] build = ["cp38*", "cp37*"] -environment = {FOO="BAR"} +environment = {FOO="BAR", "HAM"="EGGS"} -test-command = "pyproject" +test-command = ["pyproject"] manylinux-x86_64-image = "manylinux1" @@ -276,8 +396,25 @@ def test_dig_first(ignore_empty): [[tool.cibuildwheel.overrides]] select = "cp37*" -test-command = "pyproject-override" +inherit = {test-command="prepend", environment="append"} +test-command = ["pyproject-override", "override2"] manylinux-x86_64-image = "manylinux2014" +environment = {FOO="BAZ", "PYTHON"="MONTY"} + +[[tool.cibuildwheel.overrides]] +select = "*-final" +inherit = {test-command="append"} +test-command = ["pyproject-finalize", "finalize2"] + +[[tool.cibuildwheel.overrides]] +select = "*-final" +inherit = {test-command="append"} +test-command = ["extra-finalize"] + +[[tool.cibuildwheel.overrides]] +select = "*-final" +inherit = {test-command="prepend"} +test-command = ["extra-prepend"] """ @@ -285,14 +422,31 @@ def test_pyproject_2(tmp_path, platform): pyproject_toml: Path = tmp_path / "pyproject.toml" pyproject_toml.write_text(PYPROJECT_2) - options_reader = OptionsReader(config_file_path=pyproject_toml, platform=platform) - assert options_reader.get("test-command") == "pyproject" + options_reader = OptionsReader(config_file_path=pyproject_toml, platform=platform, env={}) + assert options_reader.get("test-command", option_format=ListFormat(" && ")) == "pyproject" with options_reader.identifier("random"): - assert options_reader.get("test-command") == "pyproject" + assert options_reader.get("test-command", option_format=ListFormat(" && ")) == "pyproject" with options_reader.identifier("cp37-something"): - assert options_reader.get("test-command") == "pyproject-override" + assert ( + options_reader.get("test-command", option_format=ListFormat(" && ")) + == "pyproject-override && override2 && pyproject" + ) + assert ( + options_reader.get("environment", option_format=EnvironmentFormat()) + == 'FOO="BAR" HAM="EGGS" FOO="BAZ" PYTHON="MONTY"' + ) + + with options_reader.identifier("cp37-final"): + assert ( + options_reader.get("test-command", option_format=ListFormat(" && ")) + == "extra-prepend && pyproject-override && override2 && pyproject && pyproject-finalize && finalize2 && extra-finalize" + ) + assert ( + options_reader.get("environment", option_format=EnvironmentFormat()) + == 'FOO="BAR" HAM="EGGS" FOO="BAZ" PYTHON="MONTY"' + ) def test_overrides_not_a_list(tmp_path, platform): @@ -308,5 +462,79 @@ def test_overrides_not_a_list(tmp_path, platform): """ ) - with pytest.raises(ConfigOptionError): - OptionsReader(config_file_path=pyproject_toml, platform=platform) + with pytest.raises(OptionsReaderError): + OptionsReader(config_file_path=pyproject_toml, platform=platform, env={}) + + +def test_config_settings(tmp_path): + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + """\ +[tool.cibuildwheel.config-settings] +example = "one" +other = ["two", "three"] +""" + ) + + options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux", env={}) + assert ( + options_reader.get("config-settings", option_format=ShlexTableFormat(pair_sep="=", sep=" ")) + == "example=one other=two other=three" + ) + + +def test_pip_config_settings(tmp_path): + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + """\ +[tool.cibuildwheel.config-settings] +--build-option="--use-mypyc" +""" + ) + + options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux", env={}) + assert ( + options_reader.get("config-settings", option_format=ShlexTableFormat(sep=" ", pair_sep="=")) + == "--build-option=--use-mypyc" + ) + + +def test_overrides_inherit(tmp_path): + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + """\ +[tool.cibuildwheel] +before-all = ["before-all"] +config-settings = {key1="value1", key2="value2", empty=""} + +[[tool.cibuildwheel.overrides]] +select = "cp37*" +inherit.before-all = "append" +before-all = ["override1"] + +inherit.config-settings = "append" +config-settings = {key3="value3", key2="override2"} + +[[tool.cibuildwheel.overrides]] +select = "cp37*" +inherit.before-all = "prepend" +before-all = ["override2"] +""" + ) + + options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux", env={}) + with options_reader.identifier("cp38-something"): + assert options_reader.get("before-all", option_format=ListFormat(" && ")) == "before-all" + assert ( + options_reader.get("config-settings", option_format=ShlexTableFormat()) + == "key1=value1 key2=value2 empty=''" + ) + with options_reader.identifier("cp37-something"): + assert ( + options_reader.get("before-all", option_format=ListFormat(" && ")) + == "override2 && before-all && override1" + ) + assert ( + options_reader.get("config-settings", option_format=ShlexTableFormat()) + == "key1=value1 key2=override2 empty='' key3=value3" + ) diff --git a/unit_test/projectfiles_test.py b/unit_test/projectfiles_test.py index 6c55d46a1..3bb659f7e 100644 --- a/unit_test/projectfiles_test.py +++ b/unit_test/projectfiles_test.py @@ -1,6 +1,13 @@ +import tomllib from textwrap import dedent -from cibuildwheel.projectfiles import get_requires_python_str, setup_py_python_requires +import pytest + +from cibuildwheel.projectfiles import ( + get_requires_python_str, + resolve_dependency_groups, + setup_py_python_requires, +) def test_read_setup_py_simple(tmp_path): @@ -21,7 +28,73 @@ def test_read_setup_py_simple(tmp_path): ) assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) == "1.23" - assert get_requires_python_str(tmp_path) == "1.23" + assert get_requires_python_str(tmp_path, {}) == "1.23" + + +def test_read_setup_py_if_main(tmp_path): + with open(tmp_path / "setup.py", "w") as f: + f.write( + dedent( + """ + from setuptools import setup + + if __name__ == "__main__": + setup( + name = "hello", + other = 23, + example = ["item", "other"], + python_requires = "1.23", + ) + """ + ) + ) + + assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) == "1.23" + assert get_requires_python_str(tmp_path, {}) == "1.23" + + +def test_read_setup_py_if_main_reversed(tmp_path): + with open(tmp_path / "setup.py", "w") as f: + f.write( + dedent( + """ + from setuptools import setup + + if "__main__" == __name__: + setup( + name = "hello", + other = 23, + example = ["item", "other"], + python_requires = "1.23", + ) + """ + ) + ) + + assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) == "1.23" + assert get_requires_python_str(tmp_path, {}) == "1.23" + + +def test_read_setup_py_if_invalid(tmp_path): + with open(tmp_path / "setup.py", "w") as f: + f.write( + dedent( + """ + from setuptools import setup + + if True: + setup( + name = "hello", + other = 23, + example = ["item", "other"], + python_requires = "1.23", + ) + """ + ) + ) + + assert not setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) + assert not get_requires_python_str(tmp_path, {}) def test_read_setup_py_full(tmp_path): @@ -47,7 +120,7 @@ def test_read_setup_py_full(tmp_path): assert ( setup_py_python_requires(tmp_path.joinpath("setup.py").read_text(encoding="utf8")) == "1.24" ) - assert get_requires_python_str(tmp_path) == "1.24" + assert get_requires_python_str(tmp_path, {}) == "1.24" def test_read_setup_py_assign(tmp_path): @@ -70,7 +143,7 @@ def test_read_setup_py_assign(tmp_path): ) assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) is None - assert get_requires_python_str(tmp_path) is None + assert get_requires_python_str(tmp_path, {}) is None def test_read_setup_py_None(tmp_path): @@ -93,7 +166,7 @@ def test_read_setup_py_None(tmp_path): ) assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) is None - assert get_requires_python_str(tmp_path) is None + assert get_requires_python_str(tmp_path, {}) is None def test_read_setup_py_empty(tmp_path): @@ -115,7 +188,7 @@ def test_read_setup_py_empty(tmp_path): ) assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) is None - assert get_requires_python_str(tmp_path) is None + assert get_requires_python_str(tmp_path, {}) is None def test_read_setup_cfg(tmp_path): @@ -131,7 +204,7 @@ def test_read_setup_cfg(tmp_path): ) ) - assert get_requires_python_str(tmp_path) == "1.234" + assert get_requires_python_str(tmp_path, {}) == "1.234" def test_read_setup_cfg_empty(tmp_path): @@ -147,7 +220,7 @@ def test_read_setup_cfg_empty(tmp_path): ) ) - assert get_requires_python_str(tmp_path) is None + assert get_requires_python_str(tmp_path, {}) is None def test_read_pyproject_toml(tmp_path): @@ -163,8 +236,10 @@ def test_read_pyproject_toml(tmp_path): """ ) ) + with open(tmp_path / "pyproject.toml", "rb") as f: + pyproject_toml = tomllib.load(f) - assert get_requires_python_str(tmp_path) == "1.654" + assert get_requires_python_str(tmp_path, pyproject_toml) == "1.654" def test_read_pyproject_toml_empty(tmp_path): @@ -177,5 +252,25 @@ def test_read_pyproject_toml_empty(tmp_path): """ ) ) + with open(tmp_path / "pyproject.toml", "rb") as f: + pyproject_toml = tomllib.load(f) + + assert get_requires_python_str(tmp_path, pyproject_toml) is None + + +def test_read_dep_groups(): + pyproject_toml = {"dependency-groups": {"group1": ["pkg1", "pkg2"], "group2": ["pkg3"]}} + assert resolve_dependency_groups(pyproject_toml) == () + assert resolve_dependency_groups(pyproject_toml, "group1") == ("pkg1", "pkg2") + assert resolve_dependency_groups(pyproject_toml, "group2") == ("pkg3",) + assert resolve_dependency_groups(pyproject_toml, "group1", "group2") == ("pkg1", "pkg2", "pkg3") + + +def test_dep_group_no_file_error(): + with pytest.raises(FileNotFoundError, match=r"pyproject\.toml"): + resolve_dependency_groups(None, "test") + - assert get_requires_python_str(tmp_path) is None +def test_dep_group_no_section_error(): + with pytest.raises(KeyError, match=r"pyproject\.toml"): + resolve_dependency_groups({}, "test") diff --git a/unit_test/utils.py b/unit_test/utils.py deleted file mode 100644 index 61833fa2d..000000000 --- a/unit_test/utils.py +++ /dev/null @@ -1,16 +0,0 @@ -from cibuildwheel.options import CommandLineArguments - - -def get_default_command_line_arguments() -> CommandLineArguments: - defaults = CommandLineArguments() - - defaults.platform = "auto" - defaults.allow_empty = False - defaults.archs = None - defaults.config_file = "" - defaults.output_dir = None - defaults.package_dir = "." - defaults.prerelease_pythons = False - defaults.print_build_identifiers = False - - return defaults diff --git a/unit_test/utils_test.py b/unit_test/utils_test.py index 41e376fa3..8b0be33d3 100644 --- a/unit_test/utils_test.py +++ b/unit_test/utils_test.py @@ -1,4 +1,21 @@ -from cibuildwheel.util import format_safe, prepare_command +import textwrap +from pathlib import PurePath +from unittest.mock import Mock, call + +import pytest + +from cibuildwheel import errors +from cibuildwheel.ci import fix_ansi_codes_for_github_actions +from cibuildwheel.util.file import copy_test_sources +from cibuildwheel.util.helpers import ( + FlexibleVersion, + format_safe, + parse_key_value_string, + prepare_command, + unwrap, + unwrap_preserving_paragraphs, +) +from cibuildwheel.util.packaging import find_compatible_wheel def test_format_safe(): @@ -46,3 +63,335 @@ def test_prepare_command(): prepare_command("{a}{a,b}{b:.2e}{c}{d%s}{e:3}{f[0]}", a="42", b="3.14159") == "42{a,b}{b:.2e}{c}{d%s}{e:3}{f[0]}" ) + + +@pytest.mark.parametrize( + ("wheel", "identifier"), + [ + ("foo-0.1-cp38-abi3-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-cp38-abi3-macosx_11_0_x86_64.whl", "cp310-macosx_x86_64"), + ("foo-0.1-cp38-abi3-manylinux2014_x86_64.whl", "cp310-manylinux_x86_64"), + ("foo-0.1-cp38-abi3-musllinux_1_1_x86_64.whl", "cp310-musllinux_x86_64"), + ("foo-0.1-py2.py3-none-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-py2.py3-none-win_amd64.whl", "pp310-win_amd64"), + ("foo-0.1-py3-none-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-py38-none-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-py38-none-win_amd64.whl", "pp310-win_amd64"), + ], +) +def test_find_compatible_wheel_found(wheel: str, identifier: str) -> None: + wheel_ = PurePath(wheel) + found = find_compatible_wheel([wheel_], identifier) + assert found is wheel_ + + +@pytest.mark.parametrize( + ("wheel", "identifier"), + [ + ("foo-0.1-cp38-abi3-win_amd64.whl", "cp310-win32"), + ("foo-0.1-cp38-abi3-win_amd64.whl", "cp37-win_amd64"), + ("foo-0.1-cp38-abi3-macosx_11_0_x86_64.whl", "cp310-macosx_universal2"), + ("foo-0.1-cp38-abi3-manylinux2014_x86_64.whl", "cp310-musllinux_x86_64"), + ("foo-0.1-cp38-abi3-musllinux_1_1_x86_64.whl", "cp310-manylinux_x86_64"), + ("foo-0.1-py2-none-win_amd64.whl", "cp310-win_amd64"), + ("foo-0.1-py38-none-win_amd64.whl", "cp37-win_amd64"), + ("foo-0.1-py38-none-win_amd64.whl", "pp37-win_amd64"), + ("foo-0.1-cp38-cp38-win_amd64.whl", "cp310-win_amd64"), + ], +) +def test_find_compatible_wheel_not_found(wheel: str, identifier: str) -> None: + assert find_compatible_wheel([PurePath(wheel)], identifier) is None + + +def test_fix_ansi_codes_for_github_actions(): + input = textwrap.dedent( + """ + This line is normal + \033[1mThis line is bold + This line is also bold + \033[31m this line is red and bold + This line is red and bold, too\033[0m + This line is normal again + """ + ) + + expected = textwrap.dedent( + """ + This line is normal + \033[1mThis line is bold + \033[1mThis line is also bold + \033[1m\033[31m this line is red and bold + \033[1m\033[31mThis line is red and bold, too\033[0m + This line is normal again + """ + ) + + output = fix_ansi_codes_for_github_actions(input) + + assert output == expected + + +def test_parse_key_value_string(): + assert parse_key_value_string("bar", positional_arg_names=["foo"]) == {"foo": ["bar"]} + assert parse_key_value_string("foo:bar", kw_arg_names=["foo"]) == {"foo": ["bar"]} + with pytest.raises(ValueError, match="Too many positional arguments"): + parse_key_value_string("bar") + with pytest.raises(ValueError, match="Unknown field name"): + parse_key_value_string("foo:bar") + assert parse_key_value_string("foo:bar", kw_arg_names=["foo"]) == {"foo": ["bar"]} + assert parse_key_value_string("foo:bar", positional_arg_names=["foo"]) == {"foo": ["bar"]} + assert parse_key_value_string("foo: bar", kw_arg_names=["foo"]) == {"foo": ["bar"]} + assert parse_key_value_string("foo: bar", kw_arg_names=["foo"]) == {"foo": ["bar"]} + assert parse_key_value_string("foo: bar; baz: qux", kw_arg_names=["foo", "baz"]) == { + "foo": ["bar"], + "baz": ["qux"], + } + + # some common options + assert parse_key_value_string( + "docker; create_args: --some-option --another-option=foo", + positional_arg_names=["name"], + kw_arg_names=["create_args"], + ) == { + "name": ["docker"], + "create_args": ["--some-option", "--another-option=foo"], + } + # semicolon in value + assert parse_key_value_string( + "docker; create_args: --some-option='this; that'", + positional_arg_names=["name"], + kw_arg_names=["create_args"], + ) == { + "name": ["docker"], + "create_args": ["--some-option=this; that"], + } + # colon in value + assert parse_key_value_string( + "docker; create_args: --mount a:b", + positional_arg_names=["name"], + kw_arg_names=["create_args"], + ) == { + "name": ["docker"], + "create_args": ["--mount", "a:b"], + } + assert parse_key_value_string( + "docker;create_args:--mount a:b", + positional_arg_names=["name"], + kw_arg_names=["create_args"], + ) == { + "name": ["docker"], + "create_args": ["--mount", "a:b"], + } + # quoted value with spaces + assert parse_key_value_string( + "docker;create_args:'some string with spaces'", + positional_arg_names=["name"], + kw_arg_names=["create_args"], + ) == { + "name": ["docker"], + "create_args": ["some string with spaces"], + } + + # colon in positional value + assert parse_key_value_string( + "docker; --mount a:b", + positional_arg_names=["name", "create_args"], + ) == { + "name": ["docker"], + "create_args": ["--mount", "a:b"], + } + + # empty option gives empty array + assert parse_key_value_string( + "docker;create_args:", + positional_arg_names=["name"], + kw_arg_names=["create_args"], + ) == { + "name": ["docker"], + "create_args": [], + } + + +def test_flexible_version_comparisons(): + assert FlexibleVersion("2.0") == FlexibleVersion("2") + assert FlexibleVersion("2.0") < FlexibleVersion("2.1") + assert FlexibleVersion("2.1") > FlexibleVersion("2") + assert FlexibleVersion("1.9.9") < FlexibleVersion("2.0") + assert FlexibleVersion("1.10") > FlexibleVersion("1.9.9") + assert FlexibleVersion("3.0.1") > FlexibleVersion("3.0") + assert FlexibleVersion("3.0") < FlexibleVersion("3.0.1") + # Suffix should not affect comparisons + assert FlexibleVersion("1.0.1-rhel") > FlexibleVersion("1.0") + assert FlexibleVersion("1.0.1-rhel") < FlexibleVersion("1.1") + assert FlexibleVersion("1.0.1") == FlexibleVersion("v1.0.1") + + +@pytest.fixture +def sample_project(tmp_path): + """Create a directory structure that contains a range of files.""" + project_path = tmp_path / "project" + + (project_path / "src/deep").mkdir(parents=True) + (project_path / "tests/deep").mkdir(parents=True) + (project_path / "other").mkdir(parents=True) + + (project_path / "pyproject.toml").write_text("A pyproject.toml file") + (project_path / "test.cfg").write_text("A test config file") + + (project_path / "src/__init__.py").write_text("source init") + (project_path / "src/module.py").write_text("source module") + (project_path / "src/deep/__init__.py").write_text("deep source init") + + (project_path / "tests/test_module.py").write_text("test module") + (project_path / "tests/deep/test_module.py").write_text("deep test module") + (project_path / "tests/deep/__init__.py").write_text("deep test init") + + (project_path / "other/module.py").write_text("other module") + + return project_path + + +@pytest.mark.parametrize( + ("test_sources", "expected", "not_expected"), + [ + # Empty test_sources copies nothing. + pytest.param( + [], + [], + [ + "pyproject.toml", + "test.cfg", + "other/module.py", + "src/__init__.py", + "src/module.py", + "src/deep/__init__.py", + "tests/test_module.py", + "tests/deep/__init__.py", + "tests/deep/test_module.py", + ], + id="empty", + ), + # Single standalone files + pytest.param( + ["pyproject.toml", "tests/deep/test_module.py"], + ["pyproject.toml", "tests/deep/test_module.py"], + [ + "test.cfg", + "other/module.py", + "src/__init__.py", + "src/module.py", + "src/deep/__init__.py", + "tests/test_module.py", + "tests/deep/__init__.py", + ], + id="single-file", + ), + # A full Directory + pytest.param( + ["tests"], + [ + "tests/test_module.py", + "tests/deep/__init__.py", + "tests/deep/test_module.py", + ], + [ + "pyproject.toml", + "test.cfg", + "other/module.py", + "src/__init__.py", + "src/module.py", + "src/deep/__init__.py", + ], + id="top-level-directory", + ), + # A partial deep directory + pytest.param( + ["tests/deep"], + [ + "tests/deep/__init__.py", + "tests/deep/test_module.py", + ], + [ + "pyproject.toml", + "test.cfg", + "other/module.py", + "src/__init__.py", + "src/module.py", + "src/deep/__init__.py", + "tests/test_module.py", + ], + id="partial-directory", + ), + ], +) +def test_copy_test_sources(tmp_path, sample_project, test_sources, expected, not_expected): + """Test sources can be copied into the test directory.""" + target = tmp_path / "somewhere/test_cwd" + copy_test_sources(test_sources, sample_project, target) + + for path in expected: + assert (tmp_path / "somewhere/test_cwd" / path).is_file() + + for path in not_expected: + assert not (tmp_path / "somewhere/test_cwd" / path).exists() + + +def test_copy_test_sources_missing_file(tmp_path, sample_project): + """If test_sources references a folder that doesn't exist, an error is raised.""" + + with pytest.raises( + errors.FatalError, + match=r"Test source tests/does_not_exist.py does not exist.", + ): + copy_test_sources( + ["pyproject.toml", "tests/does_not_exist.py"], + sample_project, + tmp_path / "somewhere/test_cwd", + ) + + +def test_copy_test_sources_alternate_copy_into(sample_project): + """If an alternate copy_into method is provided, it is used.""" + + target = PurePath("/container/test_cwd") + copy_into = Mock() + + copy_test_sources(["pyproject.toml", "tests"], sample_project, target, copy_into=copy_into) + + copy_into.assert_has_calls( + [ + call(sample_project / "pyproject.toml", target / "pyproject.toml"), + call(sample_project / "tests", target / "tests"), + ], + any_order=True, + ) + + +def test_unwrap(): + assert ( + unwrap(""" + This is a + multiline + string + """) + == "This is a multiline string" + ) + + +def test_unwrap_preserving_paragraphs(): + assert ( + unwrap(""" + This is a + multiline + string + """) + == "This is a multiline string" + ) + assert ( + unwrap_preserving_paragraphs(""" + paragraph one + + paragraph two + """) + == "paragraph one\n\nparagraph two" + ) diff --git a/unit_test/validate_schema_test.py b/unit_test/validate_schema_test.py new file mode 100644 index 000000000..85f5eae6b --- /dev/null +++ b/unit_test/validate_schema_test.py @@ -0,0 +1,187 @@ +import re +import tomllib +from pathlib import Path + +import pytest +import validate_pyproject.api + +from cibuildwheel.util import resources + +DIR = Path(__file__).parent.resolve() + + +@pytest.fixture(scope="session") +def validator() -> validate_pyproject.api.Validator: + """ + Reuse the validator for all tests, to keep unit tests fast. + """ + return validate_pyproject.api.Validator() + + +def test_validate_default_schema(validator: validate_pyproject.api.Validator) -> None: + with resources.DEFAULTS.open("rb") as f: + example = tomllib.load(f) + + assert validator(example) is not None + + +def test_validate_container_engine(validator: validate_pyproject.api.Validator) -> None: + """ + This test checks container engine can be overridden - it used to be a + global option but is now a build option. + """ + + example = tomllib.loads( + """ + [tool.cibuildwheel] + container-engine = "docker" + + [tool.cibuildwheel.linux] + container-engine = "docker" + + [[tool.cibuildwheel.overrides]] + select = "*_x86_64" + container-engine = "docker; create_args: --platform linux/arm64/v8" + """ + ) + + assert validator(example) is not None + + +@pytest.mark.parametrize("platform", ["macos", "windows"]) +def test_validate_bad_container_engine( + validator: validate_pyproject.api.Validator, platform: str +) -> None: + """ + container-engine is not a valid option for macos or windows + """ + example = tomllib.loads( + f""" + [tool.cibuildwheel.{platform}] + container-engine = "docker" + """ + ) + + with pytest.raises(validate_pyproject.error_reporting.ValidationError): + validator(example) + + +def test_overrides_select(validator: validate_pyproject.api.Validator) -> None: + example = tomllib.loads( + """ + [[tool.cibuildwheel.overrides]] + select = "somestring" + repair-wheel-command = "something" + """ + ) + + assert validator(example) is not None + + +def test_overrides_no_select(validator: validate_pyproject.api.Validator) -> None: + example = tomllib.loads( + """ + [[tool.cibuildwheel.overrides]] + repair-wheel-command = "something" + """ + ) + + with pytest.raises(validate_pyproject.error_reporting.ValidationError): + validator(example) + + +def test_overrides_only_select(validator: validate_pyproject.api.Validator) -> None: + example = tomllib.loads( + """ + [[tool.cibuildwheel.overrides]] + select = "somestring" + """ + ) + + with pytest.raises(validate_pyproject.error_reporting.ValidationError): + validator(example) + + +def test_overrides_valid_inherit(validator: validate_pyproject.api.Validator) -> None: + example = tomllib.loads( + """ + [[tool.cibuildwheel.overrides]] + inherit.repair-wheel-command = "append" + select = "somestring" + repair-wheel-command = ["something"] + """ + ) + + assert validator(example) is not None + + +def test_overrides_invalid_inherit(validator: validate_pyproject.api.Validator) -> None: + example = tomllib.loads( + """ + [[tool.cibuildwheel.overrides]] + inherit.something = "append" + select = "somestring" + repair-wheel-command = "something" + """ + ) + + with pytest.raises(validate_pyproject.error_reporting.ValidationError): + validator(example) + + +def test_overrides_invalid_inherit_value(validator: validate_pyproject.api.Validator) -> None: + example = tomllib.loads( + """ + [[tool.cibuildwheel.overrides]] + inherit.repair-wheel-command = "nothing" + select = "somestring" + repair-wheel-command = "something" + """ + ) + + with pytest.raises(validate_pyproject.error_reporting.ValidationError): + validator(example) + + +def test_docs_examples(validator: validate_pyproject.api.Validator) -> None: + """ + Parse out all the configuration examples, build valid TOML out of them, and + make sure they pass. + """ + + expr = re.compile( + r""" +!!! tab examples "pyproject.toml" +\s* +\s*```toml +(.*?)```""", + re.MULTILINE | re.DOTALL, + ) + + txt = DIR.parent.joinpath("docs/options.md").read_text() + + blocks: list[str] = [] + for match in expr.finditer(txt): + lines = (line.strip() for line in match.group(1).strip().splitlines() if line.strip()) + block: list[str] = [] + header = "" + for line in lines: + if line.startswith(("[tool.cibuildwheel", "[[tool.cibuildwheel")): + header = line + elif line.startswith("#"): + if block: + blocks.append("\n".join([header, *block])) + block = [] + elif " = " in line and any(x.startswith(line.partition(" = ")[0]) for x in block): + blocks.append("\n".join([header, *block])) + block = [line] + else: + block.append(line) + blocks.append("\n".join([header, *block])) + + for example_txt in blocks: + print(example_txt) + print() + example = tomllib.loads(example_txt) + + assert validator(example) is not None diff --git a/unit_test/wheel_print_test.py b/unit_test/wheel_print_test.py index 3fcadeb29..c3e4933af 100644 --- a/unit_test/wheel_print_test.py +++ b/unit_test/wheel_print_test.py @@ -1,6 +1,6 @@ import pytest -from cibuildwheel.util import print_new_wheels +from cibuildwheel.__main__ import print_new_wheels def test_printout_wheels(tmp_path, capsys): @@ -21,10 +21,9 @@ def test_printout_wheels(tmp_path, capsys): def test_no_printout_on_error(tmp_path, capsys): tmp_path.joinpath("example.0").touch() - with pytest.raises(RuntimeError): - with print_new_wheels("TEST_MSG: {n}", tmp_path): - tmp_path.joinpath("example.1").touch() - raise RuntimeError() + with pytest.raises(RuntimeError), print_new_wheels("TEST_MSG: {n}", tmp_path): # noqa: PT012 + tmp_path.joinpath("example.1").touch() + raise RuntimeError() captured = capsys.readouterr() assert captured.err == ""