From 494823e4caf6b4a56b99f07ef51741de0e3d3e48 Mon Sep 17 00:00:00 2001 From: gasoonjia Date: Thu, 19 Nov 2026 10:23:27 -0800 Subject: [PATCH 1/7] Cover CUDA 13.4 in the Windows CUDA pull request jobs PyTorch publishes CUDA 13.2 and 13.4 wheels, and ExecuTorch supports both, but every job in cuda-windows.yml ran 13.2 only, because the cross-compile image carries one train's toolkits. The 13.4 Windows build was only exercised by the nightly wheel pipeline, so a pull request that broke it would land green. The model jobs now take the train from their matrix. A train other than the image's fetches its own Linux nvcc and Windows cudart with install_cuda_redist.py, the way the wheel pipeline does, so the image is unchanged. 13.4 runs on dinov2, the smallest model, since the toolkit is what differs between trains; the six models keep running on 13.2. The CUDA runtime unit tests run on both trains. The Windows GPU runners already carry the v13.4 toolkit. test_cuda_workflow.py now checks that these jobs cover every supported train, that export and e2e list the same rows, and that the export installs the toolkits for a train the image does not carry. --- .ci/scripts/tests/test_cuda_workflow.py | 50 +++++++++++++++--- .github/workflows/cuda-windows.yml | 68 +++++++++++++++++++++---- 2 files changed, 102 insertions(+), 16 deletions(-) diff --git a/.ci/scripts/tests/test_cuda_workflow.py b/.ci/scripts/tests/test_cuda_workflow.py index 4ab845d8fab..4ebb1c9bfc1 100644 --- a/.ci/scripts/tests/test_cuda_workflow.py +++ b/.ci/scripts/tests/test_cuda_workflow.py @@ -25,6 +25,13 @@ def _supported_cuda_versions(): return [f"{major}.{minor}" for major, minor in versions] +def _matrix_cuda_versions(job): + matrix = job["strategy"]["matrix"] + if "include" in matrix: + return [row["cuda"] for row in matrix["include"]] + return list(matrix.get("cuda") or matrix["cuda-version"]) + + def _all_keys(value): if isinstance(value, dict): for key, child in value.items(): @@ -57,13 +64,41 @@ def test_every_cuda_job_uses_a_supported_version(self): for name, job in jobs.items(): with_ = job.get("with", {}) version = with_.get("gpu-arch-version") - if with_.get("gpu-arch-type") != "cuda" or "${{" in str(version): + if with_.get("gpu-arch-type") != "cuda": continue - checked += 1 - with self.subTest(workflow=workflow, job=name): - self.assertIn(version, supported) + if "${{" in str(version): + versions = _matrix_cuda_versions(job) + else: + versions = [version] + for version in versions: + checked += 1 + with self.subTest(workflow=workflow, job=name, version=version): + self.assertIn(version, supported) self.assertGreater(checked, 0) + def test_windows_jobs_cover_every_supported_train(self): + jobs = yaml.safe_load( + (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() + )["jobs"] + for name in ( + "export-model-cuda-windows-artifact", + "test-model-cuda-windows-e2e", + "unittest-cuda-runtime-windows", + ): + with self.subTest(job=name): + self.assertEqual( + sorted(set(_matrix_cuda_versions(jobs[name]))), + sorted(_supported_cuda_versions()), + ) + export, e2e = ( + jobs[name]["strategy"]["matrix"]["include"] + for name in ( + "export-model-cuda-windows-artifact", + "test-model-cuda-windows-e2e", + ) + ) + self.assertEqual(export, e2e) + def test_windows_export_image_matches_the_export_job(self): build = (ROOT / ".ci" / "docker" / "build.sh").read_text() match = re.search( @@ -75,8 +110,11 @@ def test_windows_export_image_matches_the_export_job(self): jobs = yaml.safe_load( (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() )["jobs"] - export = jobs["export-model-cuda-windows-artifact"]["with"] - self.assertEqual(export["gpu-arch-version"], image_version) + export = jobs["export-model-cuda-windows-artifact"] + self.assertIn(image_version, _matrix_cuda_versions(export)) + script = export["with"]["script"] + self.assertIn('!= "${{ matrix.cuda }}" ]; then', script) + self.assertIn('install_cuda_redist.py --train "${{ matrix.cuda }}"', script) def test_cuda_builds_take_the_node_driver_on_an_unpinned_v3(self): # This job used to pin linux_job_v2 to a test-infra SHA because that was diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index 09f7251d909..4e7fef57afa 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -79,15 +79,19 @@ jobs: - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "non-quantized" + cuda: "13.2" - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "quantized-int4-weight-only" + cuda: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "non-quantized" + cuda: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "quantized-int4-weight-only" + cuda: "13.2" # TODO: sortformer produces 0 segments on Windows after D97788666. # Temporarily disabled until root cause is debugged. # - model_repo: "nvidia" @@ -97,18 +101,26 @@ jobs: model_name: "Voxtral-Mini-4B-Realtime-2602" quant: "quantized-int4-tile-packed" mode: "vr-offline" + cuda: "13.2" - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" + cuda: "13.2" + # The other CUDA train PyTorch publishes, on the smallest model: the build and + # link against that toolkit is what differs between trains, not the model. + - model_repo: "facebook" + model_name: "dinov2-small-imagenet1k-1-layer" + quant: "non-quantized" + cuda: "13.4" with: timeout: 90 secrets-env: EXECUTORCH_HF_TOKEN runner: mt-l-x86aavx2-11-41-a10g gpu-arch-type: cuda - gpu-arch-version: "13.2" + gpu-arch-version: ${{ matrix.cuda }} docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-cuda-windows-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive - upload-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }} + upload-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | set -eux @@ -129,6 +141,26 @@ jobs: ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /opt/conda/lib/libstdc++.so.6 echo "::endgroup::" + # The image carries one train's toolkits; another train fetches its own Linux nvcc, + # which decides the torch install_executorch picks, and the Windows runtime it links. + # Under the job's temp directory: the image runs as ci-user, which cannot write to /opt. + if [ "$(nvcc --version | sed -n 's/.*release \([0-9.]*\),.*/\1/p')" != "${{ matrix.cuda }}" ]; then + echo "::group::Install the CUDA ${{ matrix.cuda }} toolkits" + TOOLKITS="${RUNNER_TEMP:-/tmp}/cuda-toolkits" + mkdir -p "${TOOLKITS}" 2>/dev/null || TOOLKITS=/tmp/cuda-toolkits + mkdir -p "${TOOLKITS}" + python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda }}" \ + --platform linux-x86_64 --dest "${TOOLKITS}/cuda-${{ matrix.cuda }}" + export CUDA_HOME="${TOOLKITS}/cuda-${{ matrix.cuda }}" + export PATH="${CUDA_HOME}/bin:${PATH}" + export CUDACXX="${CUDA_HOME}/bin/nvcc" + export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH:-}" + python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda }}" \ + --platform windows-x86_64 --dest "${TOOLKITS}/cuda-windows-${{ matrix.cuda }}" --components cuda_cudart + export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda }}" + echo "::endgroup::" + fi + echo "::group::Verify pre-installed dependencies" x86_64-w64-mingw32-g++ --version nvcc --version @@ -140,6 +172,7 @@ jobs: # Disable MKL to avoid duplicate target error when conda has multiple MKL installations export USE_MKL=OFF PYTHON_EXECUTABLE=python ./install_executorch.sh + python -c "import torch; v = torch.version.cuda; assert v == '${{ matrix.cuda }}', v" echo "::endgroup::" # Setup Huggingface only for models that need it (not dinov2) @@ -181,15 +214,19 @@ jobs: - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "non-quantized" + cuda: "13.2" - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "quantized-int4-weight-only" + cuda: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "non-quantized" + cuda: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "quantized-int4-weight-only" + cuda: "13.2" # TODO: sortformer produces 0 segments on Windows after D97788666. # Temporarily disabled until root cause is debugged. # - model_repo: "nvidia" @@ -199,16 +236,24 @@ jobs: model_name: "Voxtral-Mini-4B-Realtime-2602" quant: "quantized-int4-tile-packed" mode: "vr-offline" + cuda: "13.2" - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" + cuda: "13.2" + # The other CUDA train PyTorch publishes, on the smallest model: the build and + # link against that toolkit is what differs between trains, not the model. + - model_repo: "facebook" + model_name: "dinov2-small-imagenet1k-1-layer" + quant: "non-quantized" + cuda: "13.4" with: timeout: 240 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda # The same toolkit the export uses. - gpu-arch-version: "13.2" - download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }} + gpu-arch-version: ${{ matrix.cuda }} + download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | git config --global http.sslBackend openssl @@ -219,7 +264,7 @@ jobs: \$ErrorActionPreference = 'Stop' \$PSNativeCommandUseErrorActionPreference = \$true - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda }}' \$env:CUDA_PATH = \$env:CUDA_HOME \$env:PATH = \"\$env:CUDA_HOME\bin;\$env:PATH\" nvcc --version @@ -230,14 +275,14 @@ jobs: throw 'RUNNER_ARTIFACT_DIR is empty. Ensure download-artifact is configured for windows_job.yml.' } - .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '13.2' -Mode '${{ matrix.mode }}' + .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '${{ matrix.cuda }}' -Mode '${{ matrix.mode }}' }" # The counterpart of cuda.yml's unittest-cuda-runtime on Windows. The Windows GPU runners # carry data-center GPUs in TCC mode, which have no memory pools, so this also covers the # allocator's synchronous fallback that the Linux job cannot reach. unittest-cuda-runtime-windows: - name: unittest-cuda-runtime-windows + name: unittest-cuda-runtime-windows-${{ matrix.cuda }} needs: [changed-files, run-decision] if: | contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || @@ -249,12 +294,15 @@ jobs: contains(needs.changed-files.outputs.changed-files, 'tools/cmake/Test.cmake') || needs.run-decision.outputs.is-full-run == 'true' uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + strategy: + fail-fast: false + matrix: + cuda: ["13.2", "13.4"] with: timeout: 120 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - # The same toolkit as test-model-cuda-windows-e2e, for the same reason. - gpu-arch-version: "13.2" + gpu-arch-version: ${{ matrix.cuda }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | git config --global http.sslBackend openssl @@ -263,7 +311,7 @@ jobs: powershell -Command "& { Set-PSDebug -Trace 1 \$ErrorActionPreference = 'Stop' - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda }}' \$env:CUDA_PATH = \$env:CUDA_HOME .ci/scripts/setup-windows.ps1 .ci/scripts/test_cuda_runtime_windows.ps1 From a2e42aba313ad164bd26fb4a8fc33bd2cdc3b8bf Mon Sep 17 00:00:00 2001 From: gasoonjia Date: Thu, 19 Nov 2026 11:14:47 -0800 Subject: [PATCH 2/7] Address review: per-row e2e gate, bin\x64 PATH, int4 13.4 row - test-model-cuda-windows-e2e no longer gates on the export matrix's overall result, so one train failing its export (say an expired torch pin) does not skip every row. Each row fails on its own missing artifact; a skipped export (fork pull request) still skips this. - The e2e job puts CUDA_HOME\bin\x64 ahead of bin, as the runtime and wheel e2e scripts do, so the 13.4 row loads the 13.4 runtime. - Trigger on install_cuda_redist.py, which the export now runs. - Add parakeet-tdt int4 on 13.4: each train also installs its own torch, and the int4 kernels only run for a quantized model. - Use cuda-version as the matrix key, as cuda.yml and the wheel workflow do. - The tests read the matrix key from gpu-arch-version, check that the jobs use the train (gpu-arch-version, CUDA\v path, bin\x64), that every model runs on the image's train, and that the Windows cudart is fetched. --- .ci/scripts/tests/test_cuda_workflow.py | 96 ++++++++++++++++----- .ci/scripts/wheel/install_cuda_redist.py | 9 +- .github/workflows/cuda-windows.yml | 102 +++++++++++++---------- 3 files changed, 137 insertions(+), 70 deletions(-) diff --git a/.ci/scripts/tests/test_cuda_workflow.py b/.ci/scripts/tests/test_cuda_workflow.py index 4ebb1c9bfc1..489feedea9e 100644 --- a/.ci/scripts/tests/test_cuda_workflow.py +++ b/.ci/scripts/tests/test_cuda_workflow.py @@ -26,10 +26,24 @@ def _supported_cuda_versions(): def _matrix_cuda_versions(job): + """The versions a job's gpu-arch-version takes, read from the key it names.""" + expression = re.fullmatch( + r"\$\{\{ matrix\.([\w-]+) \}\}", job["with"]["gpu-arch-version"] + ) + assert expression, job["with"]["gpu-arch-version"] + key = expression.group(1) matrix = job["strategy"]["matrix"] - if "include" in matrix: - return [row["cuda"] for row in matrix["include"]] - return list(matrix.get("cuda") or matrix["cuda-version"]) + if key in matrix: + assert isinstance(matrix[key], list), matrix[key] + return list(matrix[key]) + return [row[key] for row in matrix["include"]] + + +WINDOWS_TRAIN_JOBS = ( + "export-model-cuda-windows-artifact", + "test-model-cuda-windows-e2e", + "unittest-cuda-runtime-windows", +) def _all_keys(value): @@ -63,13 +77,12 @@ def test_every_cuda_job_uses_a_supported_version(self): )["jobs"] for name, job in jobs.items(): with_ = job.get("with", {}) - version = with_.get("gpu-arch-version") if with_.get("gpu-arch-type") != "cuda": continue - if "${{" in str(version): + if "${{" in str(with_["gpu-arch-version"]): versions = _matrix_cuda_versions(job) else: - versions = [version] + versions = [with_["gpu-arch-version"]] for version in versions: checked += 1 with self.subTest(workflow=workflow, job=name, version=version): @@ -80,24 +93,43 @@ def test_windows_jobs_cover_every_supported_train(self): jobs = yaml.safe_load( (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() )["jobs"] - for name in ( - "export-model-cuda-windows-artifact", - "test-model-cuda-windows-e2e", - "unittest-cuda-runtime-windows", - ): + for name in WINDOWS_TRAIN_JOBS: with self.subTest(job=name): + job = jobs[name] self.assertEqual( - sorted(set(_matrix_cuda_versions(jobs[name]))), + job["with"]["gpu-arch-version"], "${{ matrix.cuda-version }}" + ) + self.assertEqual( + sorted(set(_matrix_cuda_versions(job))), sorted(_supported_cuda_versions()), ) - export, e2e = ( - jobs[name]["strategy"]["matrix"]["include"] - for name in ( - "export-model-cuda-windows-artifact", - "test-model-cuda-windows-e2e", - ) + for name in WINDOWS_TRAIN_JOBS[1:]: + with self.subTest(job=name): + script = jobs[name]["with"]["script"] + self.assertIn("CUDA\\v${{ matrix.cuda-version }}'", script) + self.assertNotRegex(script, r"CUDA\\v\d") + e2e_script = jobs["test-model-cuda-windows-e2e"]["with"]["script"] + # The 13.x Windows toolkits keep their DLLs in bin\x64, ahead of bin. + self.assertIn( + r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;", e2e_script + ) + e2e_script = jobs["test-model-cuda-windows-e2e"]["with"]["script"] + # The 13.x Windows toolkits keep their DLLs in bin\x64, ahead of bin. + self.assertIn( + r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;", e2e_script ) - self.assertEqual(export, e2e) + export = jobs["export-model-cuda-windows-artifact"]["strategy"]["matrix"] + e2e = jobs["test-model-cuda-windows-e2e"]["strategy"]["matrix"] + self.assertEqual(export["include"], e2e["include"]) + + def test_one_train_failing_its_export_does_not_skip_the_others(self): + jobs = yaml.safe_load( + (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() + )["jobs"] + gate = jobs["test-model-cuda-windows-e2e"]["if"] + self.assertNotIn("export-model-cuda-windows-artifact.result == 'success'", gate) + self.assertIn("!cancelled()", gate) + self.assertIn("export-model-cuda-windows-artifact.result != 'skipped'", gate) def test_windows_export_image_matches_the_export_job(self): build = (ROOT / ".ci" / "docker" / "build.sh").read_text() @@ -111,10 +143,30 @@ def test_windows_export_image_matches_the_export_job(self): (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() )["jobs"] export = jobs["export-model-cuda-windows-artifact"] - self.assertIn(image_version, _matrix_cuda_versions(export)) + rows = export["strategy"]["matrix"]["include"] + # Every model runs on the image's own train, so only the extra rows download. + models = {(row["model_repo"], row["model_name"], row["quant"]) for row in rows} + on_image = { + (row["model_repo"], row["model_name"], row["quant"]) + for row in rows + if row["cuda-version"] == image_version + } + self.assertEqual(on_image, models) script = export["with"]["script"] - self.assertIn('!= "${{ matrix.cuda }}" ]; then', script) - self.assertIn('install_cuda_redist.py --train "${{ matrix.cuda }}"', script) + self.assertIn('!= "${{ matrix.cuda-version }}" ]; then', script) + for platform, destination in ( + ("linux-x86_64", "cuda-${{ matrix.cuda-version }}"), + ("windows-x86_64", "cuda-windows-${{ matrix.cuda-version }}"), + ): + with self.subTest(platform=platform): + self.assertIn( + f'--platform {platform} --dest "${{TOOLKITS}}/{destination}"', + script, + ) + self.assertIn( + 'export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}"', + script, + ) def test_cuda_builds_take_the_node_driver_on_an_unpinned_v3(self): # This job used to pin linux_job_v2 to a test-infra SHA because that was diff --git a/.ci/scripts/wheel/install_cuda_redist.py b/.ci/scripts/wheel/install_cuda_redist.py index daf0a92073d..1d97a66baae 100644 --- a/.ci/scripts/wheel/install_cuda_redist.py +++ b/.ci/scripts/wheel/install_cuda_redist.py @@ -7,8 +7,8 @@ """Assemble part of a CUDA toolkit from NVIDIA's redistributable component archives. -The Windows CUDA wheel end-to-end jobs cover several CUDA trains while the CI images carry -one toolkit, and NVIDIA publishes no full Windows installer for every release. The redist +The Windows CUDA wheel end-to-end jobs and cuda-windows.yml cover several CUDA trains while +the CI images carry one toolkit, and NVIDIA publishes no full Windows installer for every release. The redist archives exist for every release on both platforms, need no installer or admin rights, and are checksummed in a JSON index. Merging the components a job needs into one directory gives the layout nvcc, CMake's FindCUDAToolkit and torch's cross-compile step expect. @@ -31,8 +31,9 @@ _REDIST = "https://developer.download.nvidia.com/compute/cuda/redist" # The patch release each train resolves to: one entry per train in filter_cuda_matrix.py's -# SUPPORTED_CUDA_VERSIONS, which the end-to-end jobs take their matrix from. A train missing -# here fails the job at argument parsing rather than testing a different release. +# SUPPORTED_CUDA_VERSIONS, which the wheel end-to-end jobs take their matrix from, and in +# cuda-windows.yml's matrices. A train missing here fails the job at argument parsing rather +# than testing a different release. _RELEASES = {"13.2": "13.2.2", "13.4": "13.4.1"} # What a build of the CUDA backend needs. "a|b" names one component a release publishes diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index 4e7fef57afa..b72eb0564fc 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -19,6 +19,7 @@ on: - extension/cuda/** - .ci/scripts/export_model_artifact.sh - .ci/scripts/test_model_e2e_windows.ps1 + - .ci/scripts/wheel/install_cuda_redist.py # Only unittest-cuda-runtime-windows runs for these alone; the other jobs # filter on their own paths and stay skipped. - .ci/scripts/test_cuda_runtime_windows.ps1 @@ -65,6 +66,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-windows.yml') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e_windows.ps1') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/wheel/install_cuda_redist.py') || needs.run-decision.outputs.is-full-run == 'true' ) uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main @@ -79,19 +81,19 @@ jobs: - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "non-quantized" - cuda: "13.2" + cuda-version: "13.2" - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "quantized-int4-weight-only" - cuda: "13.2" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "non-quantized" - cuda: "13.2" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "quantized-int4-weight-only" - cuda: "13.2" + cuda-version: "13.2" # TODO: sortformer produces 0 segments on Windows after D97788666. # Temporarily disabled until root cause is debugged. # - model_repo: "nvidia" @@ -101,26 +103,31 @@ jobs: model_name: "Voxtral-Mini-4B-Realtime-2602" quant: "quantized-int4-tile-packed" mode: "vr-offline" - cuda: "13.2" + cuda-version: "13.2" - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" - cuda: "13.2" - # The other CUDA train PyTorch publishes, on the smallest model: the build and - # link against that toolkit is what differs between trains, not the model. + cuda-version: "13.2" + # Each train installs its own toolkit and its own torch, so the remaining trains + # get the smallest model plus one int4 model, which runs the int4 kernels the + # non-quantized model does not. - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" - cuda: "13.4" + cuda-version: "13.4" + - model_repo: "nvidia" + model_name: "parakeet-tdt" + quant: "quantized-int4-weight-only" + cuda-version: "13.4" with: timeout: 90 secrets-env: EXECUTORCH_HF_TOKEN runner: mt-l-x86aavx2-11-41-a10g gpu-arch-type: cuda - gpu-arch-version: ${{ matrix.cuda }} + gpu-arch-version: ${{ matrix.cuda-version }} docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-cuda-windows-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive - upload-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda }} + upload-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | set -eux @@ -144,20 +151,20 @@ jobs: # The image carries one train's toolkits; another train fetches its own Linux nvcc, # which decides the torch install_executorch picks, and the Windows runtime it links. # Under the job's temp directory: the image runs as ci-user, which cannot write to /opt. - if [ "$(nvcc --version | sed -n 's/.*release \([0-9.]*\),.*/\1/p')" != "${{ matrix.cuda }}" ]; then - echo "::group::Install the CUDA ${{ matrix.cuda }} toolkits" + if [ "$(nvcc --version | sed -n 's/.*release \([0-9.]*\),.*/\1/p')" != "${{ matrix.cuda-version }}" ]; then + echo "::group::Install the CUDA ${{ matrix.cuda-version }} toolkits" TOOLKITS="${RUNNER_TEMP:-/tmp}/cuda-toolkits" mkdir -p "${TOOLKITS}" 2>/dev/null || TOOLKITS=/tmp/cuda-toolkits mkdir -p "${TOOLKITS}" - python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda }}" \ - --platform linux-x86_64 --dest "${TOOLKITS}/cuda-${{ matrix.cuda }}" - export CUDA_HOME="${TOOLKITS}/cuda-${{ matrix.cuda }}" + python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda-version }}" \ + --platform linux-x86_64 --dest "${TOOLKITS}/cuda-${{ matrix.cuda-version }}" + export CUDA_HOME="${TOOLKITS}/cuda-${{ matrix.cuda-version }}" export PATH="${CUDA_HOME}/bin:${PATH}" export CUDACXX="${CUDA_HOME}/bin/nvcc" export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH:-}" - python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda }}" \ - --platform windows-x86_64 --dest "${TOOLKITS}/cuda-windows-${{ matrix.cuda }}" --components cuda_cudart - export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda }}" + python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda-version }}" \ + --platform windows-x86_64 --dest "${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}" --components cuda_cudart + export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}" echo "::endgroup::" fi @@ -172,7 +179,7 @@ jobs: # Disable MKL to avoid duplicate target error when conda has multiple MKL installations export USE_MKL=OFF PYTHON_EXECUTABLE=python ./install_executorch.sh - python -c "import torch; v = torch.version.cuda; assert v == '${{ matrix.cuda }}', v" + python -c "import torch; v = torch.version.cuda; assert v == '${{ matrix.cuda-version }}', v" echo "::endgroup::" # Setup Huggingface only for models that need it (not dinov2) @@ -189,14 +196,15 @@ jobs: test-model-cuda-windows-e2e: name: test-model-cuda-windows-e2e - # Same path filter as the export job above. Also explicitly gated - # on the export job succeeding — when needs: jobs are *skipped* - # (e.g. fork PR), GitHub still evaluates this if:, so without the - # explicit success-check this job would run and then fail trying - # to download an artifact that was never produced. + # Same path filter as the export job above. Not gated on the export's result: that is + # one result for the whole matrix, so one row's failed export (say a train whose torch + # pin expired) would skip every row here. Each row downloads its own artifact and + # fails on the download if its export failed. The skipped check keeps a fork pull + # request, where nothing was exported, from running this and failing on the download. needs: [changed-files, export-model-cuda-windows-artifact, run-decision] if: | - needs.export-model-cuda-windows-artifact.result == 'success' && + !cancelled() && + needs.export-model-cuda-windows-artifact.result != 'skipped' && ( contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || contains(needs.changed-files.outputs.changed-files, 'backends/aoti') || @@ -204,6 +212,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-windows.yml') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e_windows.ps1') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/wheel/install_cuda_redist.py') || needs.run-decision.outputs.is-full-run == 'true' ) uses: pytorch/test-infra/.github/workflows/windows_job.yml@main @@ -214,19 +223,19 @@ jobs: - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "non-quantized" - cuda: "13.2" + cuda-version: "13.2" - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "quantized-int4-weight-only" - cuda: "13.2" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "non-quantized" - cuda: "13.2" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "quantized-int4-weight-only" - cuda: "13.2" + cuda-version: "13.2" # TODO: sortformer produces 0 segments on Windows after D97788666. # Temporarily disabled until root cause is debugged. # - model_repo: "nvidia" @@ -236,24 +245,29 @@ jobs: model_name: "Voxtral-Mini-4B-Realtime-2602" quant: "quantized-int4-tile-packed" mode: "vr-offline" - cuda: "13.2" + cuda-version: "13.2" - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" - cuda: "13.2" - # The other CUDA train PyTorch publishes, on the smallest model: the build and - # link against that toolkit is what differs between trains, not the model. + cuda-version: "13.2" + # Each train installs its own toolkit and its own torch, so the remaining trains + # get the smallest model plus one int4 model, which runs the int4 kernels the + # non-quantized model does not. - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" - cuda: "13.4" + cuda-version: "13.4" + - model_repo: "nvidia" + model_name: "parakeet-tdt" + quant: "quantized-int4-weight-only" + cuda-version: "13.4" with: timeout: 240 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda # The same toolkit the export uses. - gpu-arch-version: ${{ matrix.cuda }} - download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda }} + gpu-arch-version: ${{ matrix.cuda-version }} + download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | git config --global http.sslBackend openssl @@ -264,9 +278,9 @@ jobs: \$ErrorActionPreference = 'Stop' \$PSNativeCommandUseErrorActionPreference = \$true - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda }}' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda-version }}' \$env:CUDA_PATH = \$env:CUDA_HOME - \$env:PATH = \"\$env:CUDA_HOME\bin;\$env:PATH\" + \$env:PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;\$env:PATH\" nvcc --version .ci/scripts/setup-windows.ps1 @@ -275,14 +289,14 @@ jobs: throw 'RUNNER_ARTIFACT_DIR is empty. Ensure download-artifact is configured for windows_job.yml.' } - .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '${{ matrix.cuda }}' -Mode '${{ matrix.mode }}' + .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '${{ matrix.cuda-version }}' -Mode '${{ matrix.mode }}' }" # The counterpart of cuda.yml's unittest-cuda-runtime on Windows. The Windows GPU runners # carry data-center GPUs in TCC mode, which have no memory pools, so this also covers the # allocator's synchronous fallback that the Linux job cannot reach. unittest-cuda-runtime-windows: - name: unittest-cuda-runtime-windows-${{ matrix.cuda }} + name: unittest-cuda-runtime-windows-${{ matrix.cuda-version }} needs: [changed-files, run-decision] if: | contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || @@ -297,12 +311,12 @@ jobs: strategy: fail-fast: false matrix: - cuda: ["13.2", "13.4"] + cuda-version: ["13.2", "13.4"] with: timeout: 120 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - gpu-arch-version: ${{ matrix.cuda }} + gpu-arch-version: ${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | git config --global http.sslBackend openssl @@ -311,7 +325,7 @@ jobs: powershell -Command "& { Set-PSDebug -Trace 1 \$ErrorActionPreference = 'Stop' - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda }}' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda-version }}' \$env:CUDA_PATH = \$env:CUDA_HOME .ci/scripts/setup-windows.ps1 .ci/scripts/test_cuda_runtime_windows.ps1 From 1c3cfb399f9edc1220a45692c3a86d473b958d8e Mon Sep 17 00:00:00 2001 From: gasoonjia Date: Thu, 19 Nov 2026 12:06:48 -0800 Subject: [PATCH 3/7] Address review: tile-packed int4 on 13.4, runtime PATH before setup - The 13.4 int4 row is Voxtral Realtime quantized-int4-tile-packed, the quant that lowers to _weight_int4pack_mm; quantized-int4-weight-only lowers to a dequantize plus matmul and never ran the int4mm shim. It already runs on 13.2 on Windows, so the row differs only in its train. - unittest-cuda-runtime-windows puts the toolkit on PATH before setup, as the e2e job does, so setup sees the row's nvcc. - The tests check every train has a tile-packed row, check the PATH line for both Windows jobs and that it precedes setup, and lose the duplicated assertion. --- .ci/scripts/tests/test_cuda_workflow.py | 26 +++++++++++++++--------- .github/workflows/cuda-windows.yml | 27 ++++++++++++++----------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/.ci/scripts/tests/test_cuda_workflow.py b/.ci/scripts/tests/test_cuda_workflow.py index 489feedea9e..6bdbb51866a 100644 --- a/.ci/scripts/tests/test_cuda_workflow.py +++ b/.ci/scripts/tests/test_cuda_workflow.py @@ -108,19 +108,25 @@ def test_windows_jobs_cover_every_supported_train(self): script = jobs[name]["with"]["script"] self.assertIn("CUDA\\v${{ matrix.cuda-version }}'", script) self.assertNotRegex(script, r"CUDA\\v\d") - e2e_script = jobs["test-model-cuda-windows-e2e"]["with"]["script"] - # The 13.x Windows toolkits keep their DLLs in bin\x64, ahead of bin. - self.assertIn( - r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;", e2e_script - ) - e2e_script = jobs["test-model-cuda-windows-e2e"]["with"]["script"] - # The 13.x Windows toolkits keep their DLLs in bin\x64, ahead of bin. - self.assertIn( - r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;", e2e_script - ) + # The 13.x Windows toolkits keep their DLLs in bin\x64, ahead of bin, and + # the toolkit has to lead PATH before setup picks a torch for its nvcc. + path = script.index( + r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;" + ) + self.assertLess(path, script.index(".ci/scripts/setup-windows.ps1")) export = jobs["export-model-cuda-windows-artifact"]["strategy"]["matrix"] e2e = jobs["test-model-cuda-windows-e2e"]["strategy"]["matrix"] self.assertEqual(export["include"], e2e["include"]) + # Only the tile-packed quant lowers to _weight_int4pack_mm, so each train needs it. + for version in _supported_cuda_versions(): + with self.subTest(train=version): + self.assertTrue( + any( + row["quant"] == "quantized-int4-tile-packed" + and row["cuda-version"] == version + for row in export["include"] + ) + ) def test_one_train_failing_its_export_does_not_skip_the_others(self): jobs = yaml.safe_load( diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index b72eb0564fc..19b2806f02d 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -109,15 +109,16 @@ jobs: quant: "non-quantized" cuda-version: "13.2" # Each train installs its own toolkit and its own torch, so the remaining trains - # get the smallest model plus one int4 model, which runs the int4 kernels the - # non-quantized model does not. + # get the smallest model plus the tile-packed int4 model, the only quant here that + # lowers to _weight_int4pack_mm and so runs the int4mm shim that train's nvcc built. - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" cuda-version: "13.4" - - model_repo: "nvidia" - model_name: "parakeet-tdt" - quant: "quantized-int4-weight-only" + - model_repo: "mistralai" + model_name: "Voxtral-Mini-4B-Realtime-2602" + quant: "quantized-int4-tile-packed" + mode: "vr-offline" cuda-version: "13.4" with: timeout: 90 @@ -196,8 +197,8 @@ jobs: test-model-cuda-windows-e2e: name: test-model-cuda-windows-e2e - # Same path filter as the export job above. Not gated on the export's result: that is - # one result for the whole matrix, so one row's failed export (say a train whose torch + # Same path filter as the export job above. Not gated on the export succeeding: that + # result covers the whole matrix, so one row's failed export (say a train whose torch # pin expired) would skip every row here. Each row downloads its own artifact and # fails on the download if its export failed. The skipped check keeps a fork pull # request, where nothing was exported, from running this and failing on the download. @@ -251,15 +252,16 @@ jobs: quant: "non-quantized" cuda-version: "13.2" # Each train installs its own toolkit and its own torch, so the remaining trains - # get the smallest model plus one int4 model, which runs the int4 kernels the - # non-quantized model does not. + # get the smallest model plus the tile-packed int4 model, the only quant here that + # lowers to _weight_int4pack_mm and so runs the int4mm shim that train's nvcc built. - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" cuda-version: "13.4" - - model_repo: "nvidia" - model_name: "parakeet-tdt" - quant: "quantized-int4-weight-only" + - model_repo: "mistralai" + model_name: "Voxtral-Mini-4B-Realtime-2602" + quant: "quantized-int4-tile-packed" + mode: "vr-offline" cuda-version: "13.4" with: timeout: 240 @@ -327,6 +329,7 @@ jobs: \$ErrorActionPreference = 'Stop' \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda-version }}' \$env:CUDA_PATH = \$env:CUDA_HOME + \$env:PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;\$env:PATH\" .ci/scripts/setup-windows.ps1 .ci/scripts/test_cuda_runtime_windows.ps1 }" From 1c15690a0bcb40224cbecdb5d1b4ae5aeeda56bb Mon Sep 17 00:00:00 2001 From: gasoonjia Date: Thu, 19 Nov 2026 12:44:39 -0800 Subject: [PATCH 4/7] Address review: comment wording, explicit Windows-runner jobs - The PATH comment says what PATH order decides: the nvcc setup.py builds the CUDA extension with. Windows installs CPU torch either way. - Name the two Windows-runner jobs instead of slicing WINDOWS_TRAIN_JOBS. - The artifact cleanup comment says about 21 GB: 18.4 GB measured before this PR, plus a second Voxtral Realtime int4 export (2.9 GB) and a dinov2. --- .ci/scripts/tests/test_cuda_workflow.py | 5 +++-- .github/workflows/cuda-windows.yml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/scripts/tests/test_cuda_workflow.py b/.ci/scripts/tests/test_cuda_workflow.py index 6bdbb51866a..e0b17158899 100644 --- a/.ci/scripts/tests/test_cuda_workflow.py +++ b/.ci/scripts/tests/test_cuda_workflow.py @@ -103,13 +103,14 @@ def test_windows_jobs_cover_every_supported_train(self): sorted(set(_matrix_cuda_versions(job))), sorted(_supported_cuda_versions()), ) - for name in WINDOWS_TRAIN_JOBS[1:]: + for name in ("test-model-cuda-windows-e2e", "unittest-cuda-runtime-windows"): with self.subTest(job=name): script = jobs[name]["with"]["script"] self.assertIn("CUDA\\v${{ matrix.cuda-version }}'", script) self.assertNotRegex(script, r"CUDA\\v\d") # The 13.x Windows toolkits keep their DLLs in bin\x64, ahead of bin, and - # the toolkit has to lead PATH before setup picks a torch for its nvcc. + # the toolkit has to lead PATH before setup, since setup.py builds the + # CUDA extension with the first nvcc on PATH. path = script.index( r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;" ) diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index 19b2806f02d..4e4e592512d 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -337,7 +337,7 @@ jobs: delete-model-cuda-windows-artifacts: name: delete-model-cuda-windows-artifacts # The exports exist only to hand the models to the Windows job above, about - # 18 GB per run, and GitHub bills artifact storage by the hour, so delete + # 21 GB per run, and GitHub bills artifact storage by the hour, so delete # them as soon as the tests have consumed them instead of keeping them for # the repository retention window. They are kept when a test fails so that # "Re-run failed jobs" and debugging still have them; those expire with the From fe17538071d93e56c1e9f65e99880a6205e5c8ba Mon Sep 17 00:00:00 2001 From: gasoonjia Date: Thu, 19 Nov 2026 13:23:26 -0800 Subject: [PATCH 5/7] Address review: train in check names, install_* path filters - The export and e2e jobs name themselves with the train first. Without an explicit name GitHub appends every matrix value and cuts at 100 characters, so the 13.2 and 13.4 Voxtral Realtime rows shared one check name. - Trigger on install_requirements.py and install_utils.py, which hold the torch pins and SUPPORTED_CUDA_VERSIONS these rows install from. - The e2e matrix comment points at the export's instead of repeating a rationale that is about the export, and gpu-arch-version is described as the export's CUDA version, not its toolkit. --- .ci/scripts/tests/test_cuda_workflow.py | 2 ++ .github/workflows/cuda-windows.yml | 16 ++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.ci/scripts/tests/test_cuda_workflow.py b/.ci/scripts/tests/test_cuda_workflow.py index e0b17158899..01fb1a25ec8 100644 --- a/.ci/scripts/tests/test_cuda_workflow.py +++ b/.ci/scripts/tests/test_cuda_workflow.py @@ -99,6 +99,8 @@ def test_windows_jobs_cover_every_supported_train(self): self.assertEqual( job["with"]["gpu-arch-version"], "${{ matrix.cuda-version }}" ) + # GitHub cuts check names at 100 characters, so the train goes first. + self.assertIn(f"{name}-${{{{ matrix.cuda-version }}}}", job["name"]) self.assertEqual( sorted(set(_matrix_cuda_versions(job))), sorted(_supported_cuda_versions()), diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index 4e4e592512d..0103dce7543 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -20,6 +20,8 @@ on: - .ci/scripts/export_model_artifact.sh - .ci/scripts/test_model_e2e_windows.ps1 - .ci/scripts/wheel/install_cuda_redist.py + - install_requirements.py + - install_utils.py # Only unittest-cuda-runtime-windows runs for these alone; the other jobs # filter on their own paths and stay skipped. - .ci/scripts/test_cuda_runtime_windows.ps1 @@ -50,7 +52,7 @@ jobs: uses: ./.github/workflows/_ci-run-decision.yml export-model-cuda-windows-artifact: - name: export-model-cuda-windows-artifact + name: export-model-cuda-windows-artifact-${{ matrix.cuda-version }} (${{ matrix.model_name }}, ${{ matrix.quant }}) # Skip this job if the pull request is from a fork (HuggingFace secrets are not available). # Path-filtered on push: mirrors the workflow-level pull_request `paths:` # filter so push commits that don't touch CUDA-relevant paths skip @@ -67,6 +69,8 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e_windows.ps1') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/wheel/install_cuda_redist.py') || + contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') || + contains(needs.changed-files.outputs.changed-files, 'install_utils.py') || needs.run-decision.outputs.is-full-run == 'true' ) uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main @@ -196,7 +200,7 @@ jobs: source .ci/scripts/export_model_artifact.sh cuda-windows "${{ matrix.model_repo }}/${{ matrix.model_name }}" "${{ matrix.quant }}" "${RUNNER_ARTIFACT_DIR}" "${{ matrix.mode }}" test-model-cuda-windows-e2e: - name: test-model-cuda-windows-e2e + name: test-model-cuda-windows-e2e-${{ matrix.cuda-version }} (${{ matrix.model_name }}, ${{ matrix.quant }}) # Same path filter as the export job above. Not gated on the export succeeding: that # result covers the whole matrix, so one row's failed export (say a train whose torch # pin expired) would skip every row here. Each row downloads its own artifact and @@ -214,6 +218,8 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e_windows.ps1') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/wheel/install_cuda_redist.py') || + contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') || + contains(needs.changed-files.outputs.changed-files, 'install_utils.py') || needs.run-decision.outputs.is-full-run == 'true' ) uses: pytorch/test-infra/.github/workflows/windows_job.yml@main @@ -251,9 +257,7 @@ jobs: model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" cuda-version: "13.2" - # Each train installs its own toolkit and its own torch, so the remaining trains - # get the smallest model plus the tile-packed int4 model, the only quant here that - # lowers to _weight_int4pack_mm and so runs the int4mm shim that train's nvcc built. + # The export matrix's rows, which runs one model per row; see there for why. - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" @@ -267,7 +271,7 @@ jobs: timeout: 240 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - # The same toolkit the export uses. + # The same CUDA version the export used. gpu-arch-version: ${{ matrix.cuda-version }} download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} From fec953f0ada2f3fe4a7eedd1c5b12db8321fd2d2 Mon Sep 17 00:00:00 2001 From: gasoonjia Date: Thu, 19 Nov 2026 14:00:42 -0800 Subject: [PATCH 6/7] Address review: runtime job triggers on install_*, name check uses startswith - unittest-cuda-runtime-windows also runs on install_requirements.py and install_utils.py: it builds through install_executorch.py, and setup.py enables CUDA from SUPPORTED_CUDA_VERSIONS. - The name test checks the train comes first, as its comment says. - Reword the e2e matrix comment. --- .ci/scripts/tests/test_cuda_workflow.py | 5 ++++- .github/workflows/cuda-windows.yml | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/tests/test_cuda_workflow.py b/.ci/scripts/tests/test_cuda_workflow.py index 01fb1a25ec8..15e7870ebf0 100644 --- a/.ci/scripts/tests/test_cuda_workflow.py +++ b/.ci/scripts/tests/test_cuda_workflow.py @@ -100,7 +100,10 @@ def test_windows_jobs_cover_every_supported_train(self): job["with"]["gpu-arch-version"], "${{ matrix.cuda-version }}" ) # GitHub cuts check names at 100 characters, so the train goes first. - self.assertIn(f"{name}-${{{{ matrix.cuda-version }}}}", job["name"]) + self.assertTrue( + job["name"].startswith(f"{name}-${{{{ matrix.cuda-version }}}}"), + job["name"], + ) self.assertEqual( sorted(set(_matrix_cuda_versions(job))), sorted(_supported_cuda_versions()), diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index 0103dce7543..b485d103284 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -257,7 +257,7 @@ jobs: model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" cuda-version: "13.2" - # The export matrix's rows, which runs one model per row; see there for why. + # Same rows as the export matrix; see there for why. - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" @@ -312,6 +312,8 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_cuda_runtime_windows.ps1') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/install_nvidia_driver_windows.ps1') || contains(needs.changed-files.outputs.changed-files, 'tools/cmake/Test.cmake') || + contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') || + contains(needs.changed-files.outputs.changed-files, 'install_utils.py') || needs.run-decision.outputs.is-full-run == 'true' uses: pytorch/test-infra/.github/workflows/windows_job.yml@main strategy: From a874e5ec93eefc61fcb5b59ff38e78e4a05c23e4 Mon Sep 17 00:00:00 2001 From: gasoonjia Date: Thu, 19 Nov 2026 14:54:33 -0800 Subject: [PATCH 7/7] Address review: check rendered names, honor include/exclude, drop LD_LIBRARY_PATH - The test renders each row's check name from the template and requires the names to be unique and at most 100 characters. An explicit name stops GitHub appending matrix values, so a prefix check alone let six rows share a name. - _matrix_cuda_versions adds include values to the axis and removes exclude values, so an exclude that drops a train fails the coverage test. - Drop the LD_LIBRARY_PATH export from the toolkit switch, matching the wheel pipeline's copy: nvcc finds its own libraries, and the export links nothing against the Linux runtime. - Comment wording: bin\x64 goes ahead of bin on PATH; only the export has its own torch; the gate test is named per row. --- .ci/scripts/tests/test_cuda_workflow.py | 35 +++++++++++++++++++------ .github/workflows/cuda-windows.yml | 8 +++--- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/.ci/scripts/tests/test_cuda_workflow.py b/.ci/scripts/tests/test_cuda_workflow.py index 15e7870ebf0..f395a122658 100644 --- a/.ci/scripts/tests/test_cuda_workflow.py +++ b/.ci/scripts/tests/test_cuda_workflow.py @@ -33,10 +33,22 @@ def _matrix_cuda_versions(job): assert expression, job["with"]["gpu-arch-version"] key = expression.group(1) matrix = job["strategy"]["matrix"] - if key in matrix: - assert isinstance(matrix[key], list), matrix[key] - return list(matrix[key]) - return [row[key] for row in matrix["include"]] + versions = list(matrix.get(key, [])) + versions += [row[key] for row in matrix.get("include", []) if key in row] + excluded = {row[key] for row in matrix.get("exclude", []) if key in row} + return [version for version in versions if version not in excluded] + + +def _rendered_names(job): + """The check names GitHub gives a job's include rows, with each row's values filled in.""" + names = [] + for row in job["strategy"]["matrix"]["include"]: + name = job["name"] + for key, value in row.items(): + name = name.replace(f"${{{{ matrix.{key} }}}}", str(value)) + assert "${{" not in name, name + names.append(name) + return names WINDOWS_TRAIN_JOBS = ( @@ -104,6 +116,13 @@ def test_windows_jobs_cover_every_supported_train(self): job["name"].startswith(f"{name}-${{{{ matrix.cuda-version }}}}"), job["name"], ) + # An explicit name stops GitHub appending the matrix values, so the name + # itself has to tell every row apart, within the 100 characters. + if "include" in job["strategy"]["matrix"]: + rendered = _rendered_names(job) + self.assertEqual(len(set(rendered)), len(rendered), rendered) + for check in rendered: + self.assertLessEqual(len(check), 100, check) self.assertEqual( sorted(set(_matrix_cuda_versions(job))), sorted(_supported_cuda_versions()), @@ -113,9 +132,9 @@ def test_windows_jobs_cover_every_supported_train(self): script = jobs[name]["with"]["script"] self.assertIn("CUDA\\v${{ matrix.cuda-version }}'", script) self.assertNotRegex(script, r"CUDA\\v\d") - # The 13.x Windows toolkits keep their DLLs in bin\x64, ahead of bin, and - # the toolkit has to lead PATH before setup, since setup.py builds the - # CUDA extension with the first nvcc on PATH. + # The 13.x Windows toolkits keep their DLLs in bin\x64, so bin\x64 goes + # ahead of bin on PATH. The toolkit has to lead PATH before setup, since + # setup.py builds the CUDA extension with the first nvcc on PATH. path = script.index( r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;" ) @@ -134,7 +153,7 @@ def test_windows_jobs_cover_every_supported_train(self): ) ) - def test_one_train_failing_its_export_does_not_skip_the_others(self): + def test_one_failed_export_row_does_not_skip_the_others(self): jobs = yaml.safe_load( (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() )["jobs"] diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index b485d103284..e2b7a080c01 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -112,9 +112,10 @@ jobs: model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" cuda-version: "13.2" - # Each train installs its own toolkit and its own torch, so the remaining trains - # get the smallest model plus the tile-packed int4 model, the only quant here that - # lowers to _weight_int4pack_mm and so runs the int4mm shim that train's nvcc built. + # Each train installs its own toolkit and, for the export, its own torch, so each + # needs rows of its own. The other trains get only the smallest model and the + # tile-packed int4 model, the only quant here that lowers to _weight_int4pack_mm + # and so runs the int4mm shim that train's nvcc built. - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" @@ -166,7 +167,6 @@ jobs: export CUDA_HOME="${TOOLKITS}/cuda-${{ matrix.cuda-version }}" export PATH="${CUDA_HOME}/bin:${PATH}" export CUDACXX="${CUDA_HOME}/bin/nvcc" - export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH:-}" python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda-version }}" \ --platform windows-x86_64 --dest "${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}" --components cuda_cudart export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}"