From 4663c21afa0dc5c313704e4b1cc5e0962d22a424 Mon Sep 17 00:00:00 2001 From: Ni Jingzhe <1448976235@qq.com> Date: Thu, 12 Nov 2026 16:30:04 +0800 Subject: [PATCH] ci(release): build the sixteen wheels in sixteen parallel jobs (platform x Python), not four jobs of four DECISION: the wheels matrix is platform x Python version. Each job builds one wheel with cibuildwheel (CIBW_BUILD=cpXYZ-*). Not a separate OpenCASCADE job, not fewer Pythons. PROBLEM A release took 1 h 48 min (2.2.9 dry run) to 1 h 40+ min (v2.3.0) and the wall-clock time was one job: the Windows one, which compiles the C++ module and links the static OpenCASCADE for cp310, cp311, cp312 and cp313 one after the other (1 h 32 min for the 2.2.9 dry run); macOS and Linux needed 48-69 min the same way. Nothing in the four builds of a platform depends on the others. CHANGE - plan: the matrix is the product of the chosen platforms and the chosen Pythons (new dry-run input `pythons`, default all, like `platforms`), 16 jobs by default. Checked by running the plan script for the default and for a narrowed dry run. - wheels: job name and artifact name carry the Python (`wheels--cpXYZ`), because sixteen jobs uploading `wheels-` would overwrite each other. Every build step sets CIBW_BUILD. - test: its matrix carries the cp tag of each Python and downloads `wheels-linux-x86_64-`. - release: unchanged. It collects `artifacts/wheels-*/*.whl` and still requires exactly 16 wheels, four per platform, each with one compiled module for its own Python. WHAT IT COSTS, AND WHY IT IS ACCEPTED Every job restores the OpenCASCADE prefix from the cache (the key is the hash of build_occt.py, unchanged, so a hit). When build_occt.py does change, the four jobs of a platform each build OCCT at the same time: four times the runner minutes of that platform, the same wall-clock time as now, and the first to finish saves the cache (the others' save is continue-on-error). That happens rarely and the wall-clock is what hurts. A separate OCCT job that the wheel jobs wait for would avoid the duplicate work but adds a job and an artifact hand-off of a multi-gigabyte prefix; not worth it for a rare case. NOT KNOWN YET Restoring the cache sixteen times is sixteen downloads; the dry run of this branch measures what that costs. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release.yml | 43 ++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 886bb205..1cd0b7bf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,8 @@ name: Release -# A release is one distribution, `simplecadapi`, built as four platform wheels per Python version (3.10-3.13): +# A release is one distribution, `simplecadapi`, built as four platform wheels per Python version (3.10-3.13), each of the +# sixteen wheels in a job of its own so that they build in parallel (the Windows wheels, the slowest, took 1.5 hours one +# after the other): # # linux-x86_64 manylinux_2_28 # linux-aarch64 manylinux_2_28 @@ -27,6 +29,11 @@ on: required: false default: all type: string + pythons: + description: "Pythons to build wheels for: all, or a comma list of cp310, cp311, cp312, cp313" + required: false + default: all + type: string dry_run: description: Build and verify everything, but publish nothing (no GitHub release, no PyPI upload) required: false @@ -121,7 +128,8 @@ jobs: if-no-files-found: error plan: - # Which wheels to build. A release builds all four; a dry run may pick some (to iterate on one platform). + # Which wheels to build. A release builds all sixteen (4 platforms x 4 Pythons); a dry run may pick some (to iterate on + # one platform or one Python). name: Plan the wheels needs: validate runs-on: ubuntu-latest @@ -134,6 +142,7 @@ jobs: shell: bash env: PLATFORMS: ${{ github.event_name == 'workflow_dispatch' && inputs.platforms || 'all' }} + PYTHONS: ${{ github.event_name == 'workflow_dispatch' && inputs.pythons || 'all' }} run: | set -euo pipefail python3 - <<'PY' >> "$GITHUB_OUTPUT" @@ -144,16 +153,27 @@ jobs: {"name": "macos-arm64", "os": "macos-15", "archs": "arm64", "occt_cache": "/tmp/occt"}, {"name": "windows-x86_64", "os": "windows-2022", "archs": "AMD64", "occt_cache": "C:\\occt"}, ] + pythons = ["cp310", "cp311", "cp312", "cp313"] wanted = [p.strip() for p in os.environ["PLATFORMS"].split(",") if p.strip()] chosen = every if wanted in ([], ["all"]) else [e for e in every if e["name"] in wanted] unknown = [p for p in wanted if p != "all" and p not in {e["name"] for e in every}] if unknown or not chosen: raise SystemExit(f"unknown platforms {unknown}; choose from {[e['name'] for e in every]}") - print("matrix=" + json.dumps({"include": chosen})) + wanted_pythons = [p.strip() for p in os.environ["PYTHONS"].split(",") if p.strip()] + chosen_pythons = pythons if wanted_pythons in ([], ["all"]) else [p for p in pythons if p in wanted_pythons] + unknown = [p for p in wanted_pythons if p != "all" and p not in pythons] + if unknown or not chosen_pythons: + raise SystemExit(f"unknown pythons {unknown}; choose from {pythons}") + # one job per platform and Python: the wheels of a platform build side by side, not one after the other + include = [{**e, "python": py} for e in chosen for py in chosen_pythons] + print("matrix=" + json.dumps({"include": include})) PY wheels: - name: Wheels (${{ matrix.name }}) + # One job per platform and Python. Each restores the same static OpenCASCADE prefix; when build_occt.py changed (the + # cache key is its hash) the four jobs of a platform each build OCCT at the same time, which costs four times the runner + # minutes but no more wall-clock time than one job did, and the first to finish saves the cache. + name: Wheels (${{ matrix.name }}, ${{ matrix.python }}) needs: [webui, plan] runs-on: ${{ matrix.os }} timeout-minutes: 240 @@ -200,6 +220,7 @@ jobs: uses: pypa/cibuildwheel@b9e566a2b284570af229f6c2a33673410d0389a3 # v4.3.4 env: CIBW_ARCHS: ${{ matrix.archs }} + CIBW_BUILD: ${{ matrix.python }}-* # share the cached prefix with the container (before-all in pyproject.toml builds OCCT only if it is absent) CIBW_CONTAINER_ENGINE: "docker; create_args: --volume ${{ matrix.occt_cache }}:/opt/occt" @@ -208,12 +229,14 @@ jobs: uses: pypa/cibuildwheel@b9e566a2b284570af229f6c2a33673410d0389a3 # v4.3.4 env: CIBW_ARCHS: ${{ matrix.archs }} + CIBW_BUILD: ${{ matrix.python }}-* - name: Build the wheels (Windows) if: runner.os == 'Windows' uses: pypa/cibuildwheel@b9e566a2b284570af229f6c2a33673410d0389a3 # v4.3.4 env: CIBW_ARCHS: ${{ matrix.archs }} + CIBW_BUILD: ${{ matrix.python }}-* # the MSVC environment is active (above): Ninja, every translation unit in parallel, and keep going after a # failure so one run reports all the compiler errors rather than the first CIBW_ENVIRONMENT_WINDOWS: >- @@ -233,7 +256,7 @@ jobs: - name: Keep the wheels uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: wheels-${{ matrix.name }} + name: wheels-${{ matrix.name }}-${{ matrix.python }} path: wheelhouse/*.whl if-no-files-found: error @@ -280,17 +303,21 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + include: + - {python-version: "3.10", tag: cp310} + - {python-version: "3.11", tag: cp311} + - {python-version: "3.12", tag: cp312} + - {python-version: "3.13", tag: cp313} steps: - name: Check out tagged revision uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 with: ref: ${{ env.RELEASE_TAG }} - - name: Fetch the Linux x86_64 wheels + - name: Fetch the Linux x86_64 wheel of this Python uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: wheels-linux-x86_64 + name: wheels-linux-x86_64-${{ matrix.tag }} path: dist - name: Install uv and Python