diff --git a/.ci/scripts/tests/test_cuda_workflow.py b/.ci/scripts/tests/test_cuda_workflow.py index 4ab845d8fab..f395a122658 100644 --- a/.ci/scripts/tests/test_cuda_workflow.py +++ b/.ci/scripts/tests/test_cuda_workflow.py @@ -25,6 +25,39 @@ def _supported_cuda_versions(): return [f"{major}.{minor}" for major, minor in versions] +def _matrix_cuda_versions(job): + """The versions a job's gpu-arch-version takes, read from the key it names.""" + expression = re.fullmatch( + r"\$\{\{ matrix\.([\w-]+) \}\}", job["with"]["gpu-arch-version"] + ) + assert expression, job["with"]["gpu-arch-version"] + key = expression.group(1) + matrix = job["strategy"]["matrix"] + versions = list(matrix.get(key, [])) + versions += [row[key] for row in matrix.get("include", []) if key in row] + excluded = {row[key] for row in matrix.get("exclude", []) if key in row} + return [version for version in versions if version not in excluded] + + +def _rendered_names(job): + """The check names GitHub gives a job's include rows, with each row's values filled in.""" + names = [] + for row in job["strategy"]["matrix"]["include"]: + name = job["name"] + for key, value in row.items(): + name = name.replace(f"${{{{ matrix.{key} }}}}", str(value)) + assert "${{" not in name, name + names.append(name) + return names + + +WINDOWS_TRAIN_JOBS = ( + "export-model-cuda-windows-artifact", + "test-model-cuda-windows-e2e", + "unittest-cuda-runtime-windows", +) + + def _all_keys(value): if isinstance(value, dict): for key, child in value.items(): @@ -56,14 +89,79 @@ def test_every_cuda_job_uses_a_supported_version(self): )["jobs"] for name, job in jobs.items(): with_ = job.get("with", {}) - version = with_.get("gpu-arch-version") - if with_.get("gpu-arch-type") != "cuda" or "${{" in str(version): + if with_.get("gpu-arch-type") != "cuda": continue - checked += 1 - with self.subTest(workflow=workflow, job=name): - self.assertIn(version, supported) + if "${{" in str(with_["gpu-arch-version"]): + versions = _matrix_cuda_versions(job) + else: + versions = [with_["gpu-arch-version"]] + for version in versions: + checked += 1 + with self.subTest(workflow=workflow, job=name, version=version): + self.assertIn(version, supported) self.assertGreater(checked, 0) + def test_windows_jobs_cover_every_supported_train(self): + jobs = yaml.safe_load( + (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() + )["jobs"] + for name in WINDOWS_TRAIN_JOBS: + with self.subTest(job=name): + job = jobs[name] + self.assertEqual( + job["with"]["gpu-arch-version"], "${{ matrix.cuda-version }}" + ) + # GitHub cuts check names at 100 characters, so the train goes first. + self.assertTrue( + job["name"].startswith(f"{name}-${{{{ matrix.cuda-version }}}}"), + job["name"], + ) + # An explicit name stops GitHub appending the matrix values, so the name + # itself has to tell every row apart, within the 100 characters. + if "include" in job["strategy"]["matrix"]: + rendered = _rendered_names(job) + self.assertEqual(len(set(rendered)), len(rendered), rendered) + for check in rendered: + self.assertLessEqual(len(check), 100, check) + self.assertEqual( + sorted(set(_matrix_cuda_versions(job))), + sorted(_supported_cuda_versions()), + ) + for name in ("test-model-cuda-windows-e2e", "unittest-cuda-runtime-windows"): + with self.subTest(job=name): + script = jobs[name]["with"]["script"] + self.assertIn("CUDA\\v${{ matrix.cuda-version }}'", script) + self.assertNotRegex(script, r"CUDA\\v\d") + # The 13.x Windows toolkits keep their DLLs in bin\x64, so bin\x64 goes + # ahead of bin on PATH. The toolkit has to lead PATH before setup, since + # setup.py builds the CUDA extension with the first nvcc on PATH. + path = script.index( + r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;" + ) + self.assertLess(path, script.index(".ci/scripts/setup-windows.ps1")) + export = jobs["export-model-cuda-windows-artifact"]["strategy"]["matrix"] + e2e = jobs["test-model-cuda-windows-e2e"]["strategy"]["matrix"] + self.assertEqual(export["include"], e2e["include"]) + # Only the tile-packed quant lowers to _weight_int4pack_mm, so each train needs it. + for version in _supported_cuda_versions(): + with self.subTest(train=version): + self.assertTrue( + any( + row["quant"] == "quantized-int4-tile-packed" + and row["cuda-version"] == version + for row in export["include"] + ) + ) + + def test_one_failed_export_row_does_not_skip_the_others(self): + jobs = yaml.safe_load( + (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() + )["jobs"] + gate = jobs["test-model-cuda-windows-e2e"]["if"] + self.assertNotIn("export-model-cuda-windows-artifact.result == 'success'", gate) + self.assertIn("!cancelled()", gate) + self.assertIn("export-model-cuda-windows-artifact.result != 'skipped'", gate) + def test_windows_export_image_matches_the_export_job(self): build = (ROOT / ".ci" / "docker" / "build.sh").read_text() match = re.search( @@ -75,8 +173,31 @@ def test_windows_export_image_matches_the_export_job(self): jobs = yaml.safe_load( (ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text() )["jobs"] - export = jobs["export-model-cuda-windows-artifact"]["with"] - self.assertEqual(export["gpu-arch-version"], image_version) + export = jobs["export-model-cuda-windows-artifact"] + rows = export["strategy"]["matrix"]["include"] + # Every model runs on the image's own train, so only the extra rows download. + models = {(row["model_repo"], row["model_name"], row["quant"]) for row in rows} + on_image = { + (row["model_repo"], row["model_name"], row["quant"]) + for row in rows + if row["cuda-version"] == image_version + } + self.assertEqual(on_image, models) + script = export["with"]["script"] + self.assertIn('!= "${{ matrix.cuda-version }}" ]; then', script) + for platform, destination in ( + ("linux-x86_64", "cuda-${{ matrix.cuda-version }}"), + ("windows-x86_64", "cuda-windows-${{ matrix.cuda-version }}"), + ): + with self.subTest(platform=platform): + self.assertIn( + f'--platform {platform} --dest "${{TOOLKITS}}/{destination}"', + script, + ) + self.assertIn( + 'export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}"', + script, + ) def test_cuda_builds_take_the_node_driver_on_an_unpinned_v3(self): # This job used to pin linux_job_v2 to a test-infra SHA because that was diff --git a/.ci/scripts/wheel/install_cuda_redist.py b/.ci/scripts/wheel/install_cuda_redist.py index daf0a92073d..1d97a66baae 100644 --- a/.ci/scripts/wheel/install_cuda_redist.py +++ b/.ci/scripts/wheel/install_cuda_redist.py @@ -7,8 +7,8 @@ """Assemble part of a CUDA toolkit from NVIDIA's redistributable component archives. -The Windows CUDA wheel end-to-end jobs cover several CUDA trains while the CI images carry -one toolkit, and NVIDIA publishes no full Windows installer for every release. The redist +The Windows CUDA wheel end-to-end jobs and cuda-windows.yml cover several CUDA trains while +the CI images carry one toolkit, and NVIDIA publishes no full Windows installer for every release. The redist archives exist for every release on both platforms, need no installer or admin rights, and are checksummed in a JSON index. Merging the components a job needs into one directory gives the layout nvcc, CMake's FindCUDAToolkit and torch's cross-compile step expect. @@ -31,8 +31,9 @@ _REDIST = "https://developer.download.nvidia.com/compute/cuda/redist" # The patch release each train resolves to: one entry per train in filter_cuda_matrix.py's -# SUPPORTED_CUDA_VERSIONS, which the end-to-end jobs take their matrix from. A train missing -# here fails the job at argument parsing rather than testing a different release. +# SUPPORTED_CUDA_VERSIONS, which the wheel end-to-end jobs take their matrix from, and in +# cuda-windows.yml's matrices. A train missing here fails the job at argument parsing rather +# than testing a different release. _RELEASES = {"13.2": "13.2.2", "13.4": "13.4.1"} # What a build of the CUDA backend needs. "a|b" names one component a release publishes diff --git a/.github/workflows/cuda-windows.yml b/.github/workflows/cuda-windows.yml index 09f7251d909..e2b7a080c01 100644 --- a/.github/workflows/cuda-windows.yml +++ b/.github/workflows/cuda-windows.yml @@ -19,6 +19,9 @@ on: - extension/cuda/** - .ci/scripts/export_model_artifact.sh - .ci/scripts/test_model_e2e_windows.ps1 + - .ci/scripts/wheel/install_cuda_redist.py + - install_requirements.py + - install_utils.py # Only unittest-cuda-runtime-windows runs for these alone; the other jobs # filter on their own paths and stay skipped. - .ci/scripts/test_cuda_runtime_windows.ps1 @@ -49,7 +52,7 @@ jobs: uses: ./.github/workflows/_ci-run-decision.yml export-model-cuda-windows-artifact: - name: export-model-cuda-windows-artifact + name: export-model-cuda-windows-artifact-${{ matrix.cuda-version }} (${{ matrix.model_name }}, ${{ matrix.quant }}) # Skip this job if the pull request is from a fork (HuggingFace secrets are not available). # Path-filtered on push: mirrors the workflow-level pull_request `paths:` # filter so push commits that don't touch CUDA-relevant paths skip @@ -65,6 +68,9 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-windows.yml') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e_windows.ps1') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/wheel/install_cuda_redist.py') || + contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') || + contains(needs.changed-files.outputs.changed-files, 'install_utils.py') || needs.run-decision.outputs.is-full-run == 'true' ) uses: pytorch/test-infra/.github/workflows/linux_job_v3.yml@main @@ -79,15 +85,19 @@ jobs: - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "non-quantized" + cuda-version: "13.2" - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "quantized-int4-weight-only" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "non-quantized" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "quantized-int4-weight-only" + cuda-version: "13.2" # TODO: sortformer produces 0 segments on Windows after D97788666. # Temporarily disabled until root cause is debugged. # - model_repo: "nvidia" @@ -97,18 +107,33 @@ jobs: model_name: "Voxtral-Mini-4B-Realtime-2602" quant: "quantized-int4-tile-packed" mode: "vr-offline" + cuda-version: "13.2" - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" + cuda-version: "13.2" + # Each train installs its own toolkit and, for the export, its own torch, so each + # needs rows of its own. The other trains get only the smallest model and the + # tile-packed int4 model, the only quant here that lowers to _weight_int4pack_mm + # and so runs the int4mm shim that train's nvcc built. + - model_repo: "facebook" + model_name: "dinov2-small-imagenet1k-1-layer" + quant: "non-quantized" + cuda-version: "13.4" + - model_repo: "mistralai" + model_name: "Voxtral-Mini-4B-Realtime-2602" + quant: "quantized-int4-tile-packed" + mode: "vr-offline" + cuda-version: "13.4" with: timeout: 90 secrets-env: EXECUTORCH_HF_TOKEN runner: mt-l-x86aavx2-11-41-a10g gpu-arch-type: cuda - gpu-arch-version: "13.2" + gpu-arch-version: ${{ matrix.cuda-version }} docker-image: ${{ needs.docker-image.outputs.docker-registry }}/ci-image:executorch-ubuntu-22.04-cuda-windows-${{ needs.docker-image.outputs.ci-docker-hash }} submodules: recursive - upload-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }} + upload-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | set -eux @@ -129,6 +154,25 @@ jobs: ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /opt/conda/lib/libstdc++.so.6 echo "::endgroup::" + # The image carries one train's toolkits; another train fetches its own Linux nvcc, + # which decides the torch install_executorch picks, and the Windows runtime it links. + # Under the job's temp directory: the image runs as ci-user, which cannot write to /opt. + if [ "$(nvcc --version | sed -n 's/.*release \([0-9.]*\),.*/\1/p')" != "${{ matrix.cuda-version }}" ]; then + echo "::group::Install the CUDA ${{ matrix.cuda-version }} toolkits" + TOOLKITS="${RUNNER_TEMP:-/tmp}/cuda-toolkits" + mkdir -p "${TOOLKITS}" 2>/dev/null || TOOLKITS=/tmp/cuda-toolkits + mkdir -p "${TOOLKITS}" + python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda-version }}" \ + --platform linux-x86_64 --dest "${TOOLKITS}/cuda-${{ matrix.cuda-version }}" + export CUDA_HOME="${TOOLKITS}/cuda-${{ matrix.cuda-version }}" + export PATH="${CUDA_HOME}/bin:${PATH}" + export CUDACXX="${CUDA_HOME}/bin/nvcc" + python .ci/scripts/wheel/install_cuda_redist.py --train "${{ matrix.cuda-version }}" \ + --platform windows-x86_64 --dest "${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}" --components cuda_cudart + export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}" + echo "::endgroup::" + fi + echo "::group::Verify pre-installed dependencies" x86_64-w64-mingw32-g++ --version nvcc --version @@ -140,6 +184,7 @@ jobs: # Disable MKL to avoid duplicate target error when conda has multiple MKL installations export USE_MKL=OFF PYTHON_EXECUTABLE=python ./install_executorch.sh + python -c "import torch; v = torch.version.cuda; assert v == '${{ matrix.cuda-version }}', v" echo "::endgroup::" # Setup Huggingface only for models that need it (not dinov2) @@ -155,15 +200,16 @@ jobs: source .ci/scripts/export_model_artifact.sh cuda-windows "${{ matrix.model_repo }}/${{ matrix.model_name }}" "${{ matrix.quant }}" "${RUNNER_ARTIFACT_DIR}" "${{ matrix.mode }}" test-model-cuda-windows-e2e: - name: test-model-cuda-windows-e2e - # Same path filter as the export job above. Also explicitly gated - # on the export job succeeding — when needs: jobs are *skipped* - # (e.g. fork PR), GitHub still evaluates this if:, so without the - # explicit success-check this job would run and then fail trying - # to download an artifact that was never produced. + name: test-model-cuda-windows-e2e-${{ matrix.cuda-version }} (${{ matrix.model_name }}, ${{ matrix.quant }}) + # Same path filter as the export job above. Not gated on the export succeeding: that + # result covers the whole matrix, so one row's failed export (say a train whose torch + # pin expired) would skip every row here. Each row downloads its own artifact and + # fails on the download if its export failed. The skipped check keeps a fork pull + # request, where nothing was exported, from running this and failing on the download. needs: [changed-files, export-model-cuda-windows-artifact, run-decision] if: | - needs.export-model-cuda-windows-artifact.result == 'success' && + !cancelled() && + needs.export-model-cuda-windows-artifact.result != 'skipped' && ( contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || contains(needs.changed-files.outputs.changed-files, 'backends/aoti') || @@ -171,6 +217,9 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.github/workflows/cuda-windows.yml') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/export_model_artifact.sh') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_model_e2e_windows.ps1') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/wheel/install_cuda_redist.py') || + contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') || + contains(needs.changed-files.outputs.changed-files, 'install_utils.py') || needs.run-decision.outputs.is-full-run == 'true' ) uses: pytorch/test-infra/.github/workflows/windows_job.yml@main @@ -181,15 +230,19 @@ jobs: - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "non-quantized" + cuda-version: "13.2" - model_repo: "mistralai" model_name: "Voxtral-Mini-3B-2507" quant: "quantized-int4-weight-only" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "non-quantized" + cuda-version: "13.2" - model_repo: "nvidia" model_name: "parakeet-tdt" quant: "quantized-int4-weight-only" + cuda-version: "13.2" # TODO: sortformer produces 0 segments on Windows after D97788666. # Temporarily disabled until root cause is debugged. # - model_repo: "nvidia" @@ -199,16 +252,28 @@ jobs: model_name: "Voxtral-Mini-4B-Realtime-2602" quant: "quantized-int4-tile-packed" mode: "vr-offline" + cuda-version: "13.2" - model_repo: "facebook" model_name: "dinov2-small-imagenet1k-1-layer" quant: "non-quantized" + cuda-version: "13.2" + # Same rows as the export matrix; see there for why. + - model_repo: "facebook" + model_name: "dinov2-small-imagenet1k-1-layer" + quant: "non-quantized" + cuda-version: "13.4" + - model_repo: "mistralai" + model_name: "Voxtral-Mini-4B-Realtime-2602" + quant: "quantized-int4-tile-packed" + mode: "vr-offline" + cuda-version: "13.4" with: timeout: 240 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - # The same toolkit the export uses. - gpu-arch-version: "13.2" - download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }} + # The same CUDA version the export used. + gpu-arch-version: ${{ matrix.cuda-version }} + download-artifact: ${{ matrix.model_repo }}-${{ matrix.model_name }}-cuda-windows-${{ matrix.quant }}-cuda${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | git config --global http.sslBackend openssl @@ -219,9 +284,9 @@ jobs: \$ErrorActionPreference = 'Stop' \$PSNativeCommandUseErrorActionPreference = \$true - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda-version }}' \$env:CUDA_PATH = \$env:CUDA_HOME - \$env:PATH = \"\$env:CUDA_HOME\bin;\$env:PATH\" + \$env:PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;\$env:PATH\" nvcc --version .ci/scripts/setup-windows.ps1 @@ -230,14 +295,14 @@ jobs: throw 'RUNNER_ARTIFACT_DIR is empty. Ensure download-artifact is configured for windows_job.yml.' } - .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '13.2' -Mode '${{ matrix.mode }}' + .ci/scripts/test_model_e2e_windows.ps1 -Device cuda-windows -HfModel '${{ matrix.model_repo }}/${{ matrix.model_name }}' -QuantName '${{ matrix.quant }}' -ModelDir \$artifactDir -ExpectedCudaVersion '${{ matrix.cuda-version }}' -Mode '${{ matrix.mode }}' }" # The counterpart of cuda.yml's unittest-cuda-runtime on Windows. The Windows GPU runners # carry data-center GPUs in TCC mode, which have no memory pools, so this also covers the # allocator's synchronous fallback that the Linux job cannot reach. unittest-cuda-runtime-windows: - name: unittest-cuda-runtime-windows + name: unittest-cuda-runtime-windows-${{ matrix.cuda-version }} needs: [changed-files, run-decision] if: | contains(needs.changed-files.outputs.changed-files, 'backends/cuda') || @@ -247,14 +312,19 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_cuda_runtime_windows.ps1') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/install_nvidia_driver_windows.ps1') || contains(needs.changed-files.outputs.changed-files, 'tools/cmake/Test.cmake') || + contains(needs.changed-files.outputs.changed-files, 'install_requirements.py') || + contains(needs.changed-files.outputs.changed-files, 'install_utils.py') || needs.run-decision.outputs.is-full-run == 'true' uses: pytorch/test-infra/.github/workflows/windows_job.yml@main + strategy: + fail-fast: false + matrix: + cuda-version: ["13.2", "13.4"] with: timeout: 120 runner: windows.g5.4xlarge.nvidia.gpu gpu-arch-type: cuda - # The same toolkit as test-model-cuda-windows-e2e, for the same reason. - gpu-arch-version: "13.2" + gpu-arch-version: ${{ matrix.cuda-version }} ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | git config --global http.sslBackend openssl @@ -263,8 +333,9 @@ jobs: powershell -Command "& { Set-PSDebug -Trace 1 \$ErrorActionPreference = 'Stop' - \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2' + \$env:CUDA_HOME = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${{ matrix.cuda-version }}' \$env:CUDA_PATH = \$env:CUDA_HOME + \$env:PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;\$env:PATH\" .ci/scripts/setup-windows.ps1 .ci/scripts/test_cuda_runtime_windows.ps1 }" @@ -272,7 +343,7 @@ jobs: delete-model-cuda-windows-artifacts: name: delete-model-cuda-windows-artifacts # The exports exist only to hand the models to the Windows job above, about - # 18 GB per run, and GitHub bills artifact storage by the hour, so delete + # 21 GB per run, and GitHub bills artifact storage by the hour, so delete # them as soon as the tests have consumed them instead of keeping them for # the repository retention window. They are kept when a test fails so that # "Re-run failed jobs" and debugging still have them; those expire with the