Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 128 additions & 7 deletions .ci/scripts/tests/test_cuda_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,39 @@ def _supported_cuda_versions():
return [f"{major}.{minor}" for major, minor in versions]


def _matrix_cuda_versions(job):
"""The versions a job's gpu-arch-version takes, read from the key it names."""
expression = re.fullmatch(
r"\$\{\{ matrix\.([\w-]+) \}\}", job["with"]["gpu-arch-version"]
)
assert expression, job["with"]["gpu-arch-version"]
key = expression.group(1)
matrix = job["strategy"]["matrix"]
versions = list(matrix.get(key, []))
versions += [row[key] for row in matrix.get("include", []) if key in row]
excluded = {row[key] for row in matrix.get("exclude", []) if key in row}
return [version for version in versions if version not in excluded]


def _rendered_names(job):
"""The check names GitHub gives a job's include rows, with each row's values filled in."""
names = []
for row in job["strategy"]["matrix"]["include"]:
name = job["name"]
for key, value in row.items():
name = name.replace(f"${{{{ matrix.{key} }}}}", str(value))
assert "${{" not in name, name
names.append(name)
return names


WINDOWS_TRAIN_JOBS = (
"export-model-cuda-windows-artifact",
"test-model-cuda-windows-e2e",
"unittest-cuda-runtime-windows",
)


def _all_keys(value):
if isinstance(value, dict):
for key, child in value.items():
Expand Down Expand Up @@ -56,14 +89,79 @@ def test_every_cuda_job_uses_a_supported_version(self):
)["jobs"]
for name, job in jobs.items():
with_ = job.get("with", {})
version = with_.get("gpu-arch-version")
if with_.get("gpu-arch-type") != "cuda" or "${{" in str(version):
if with_.get("gpu-arch-type") != "cuda":
continue
checked += 1
with self.subTest(workflow=workflow, job=name):
self.assertIn(version, supported)
if "${{" in str(with_["gpu-arch-version"]):
versions = _matrix_cuda_versions(job)
else:
versions = [with_["gpu-arch-version"]]
for version in versions:
checked += 1
with self.subTest(workflow=workflow, job=name, version=version):
self.assertIn(version, supported)
self.assertGreater(checked, 0)

def test_windows_jobs_cover_every_supported_train(self):
jobs = yaml.safe_load(
(ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text()
)["jobs"]
for name in WINDOWS_TRAIN_JOBS:
with self.subTest(job=name):
job = jobs[name]
self.assertEqual(
job["with"]["gpu-arch-version"], "${{ matrix.cuda-version }}"
)
# GitHub cuts check names at 100 characters, so the train goes first.
self.assertTrue(
job["name"].startswith(f"{name}-${{{{ matrix.cuda-version }}}}"),
job["name"],
)
# An explicit name stops GitHub appending the matrix values, so the name
# itself has to tell every row apart, within the 100 characters.
if "include" in job["strategy"]["matrix"]:
rendered = _rendered_names(job)
self.assertEqual(len(set(rendered)), len(rendered), rendered)
for check in rendered:
self.assertLessEqual(len(check), 100, check)
self.assertEqual(
sorted(set(_matrix_cuda_versions(job))),
sorted(_supported_cuda_versions()),
)
for name in ("test-model-cuda-windows-e2e", "unittest-cuda-runtime-windows"):
with self.subTest(job=name):
script = jobs[name]["with"]["script"]
self.assertIn("CUDA\\v${{ matrix.cuda-version }}'", script)
self.assertNotRegex(script, r"CUDA\\v\d")
# The 13.x Windows toolkits keep their DLLs in bin\x64, so bin\x64 goes
# ahead of bin on PATH. The toolkit has to lead PATH before setup, since
# setup.py builds the CUDA extension with the first nvcc on PATH.
path = script.index(
r"PATH = \"\$env:CUDA_HOME\bin\x64;\$env:CUDA_HOME\bin;"
)
self.assertLess(path, script.index(".ci/scripts/setup-windows.ps1"))
export = jobs["export-model-cuda-windows-artifact"]["strategy"]["matrix"]
e2e = jobs["test-model-cuda-windows-e2e"]["strategy"]["matrix"]
self.assertEqual(export["include"], e2e["include"])
# Only the tile-packed quant lowers to _weight_int4pack_mm, so each train needs it.
for version in _supported_cuda_versions():
with self.subTest(train=version):
self.assertTrue(
any(
row["quant"] == "quantized-int4-tile-packed"
and row["cuda-version"] == version
for row in export["include"]
)
)

def test_one_failed_export_row_does_not_skip_the_others(self):
jobs = yaml.safe_load(
(ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text()
)["jobs"]
gate = jobs["test-model-cuda-windows-e2e"]["if"]
self.assertNotIn("export-model-cuda-windows-artifact.result == 'success'", gate)
self.assertIn("!cancelled()", gate)
self.assertIn("export-model-cuda-windows-artifact.result != 'skipped'", gate)

def test_windows_export_image_matches_the_export_job(self):
build = (ROOT / ".ci" / "docker" / "build.sh").read_text()
match = re.search(
Expand All @@ -75,8 +173,31 @@ def test_windows_export_image_matches_the_export_job(self):
jobs = yaml.safe_load(
(ROOT / ".github" / "workflows" / "cuda-windows.yml").read_text()
)["jobs"]
export = jobs["export-model-cuda-windows-artifact"]["with"]
self.assertEqual(export["gpu-arch-version"], image_version)
export = jobs["export-model-cuda-windows-artifact"]
rows = export["strategy"]["matrix"]["include"]
# Every model runs on the image's own train, so only the extra rows download.
models = {(row["model_repo"], row["model_name"], row["quant"]) for row in rows}
on_image = {
(row["model_repo"], row["model_name"], row["quant"])
for row in rows
if row["cuda-version"] == image_version
}
self.assertEqual(on_image, models)
script = export["with"]["script"]
self.assertIn('!= "${{ matrix.cuda-version }}" ]; then', script)
for platform, destination in (
("linux-x86_64", "cuda-${{ matrix.cuda-version }}"),
("windows-x86_64", "cuda-windows-${{ matrix.cuda-version }}"),
):
with self.subTest(platform=platform):
self.assertIn(
f'--platform {platform} --dest "${{TOOLKITS}}/{destination}"',
script,
)
self.assertIn(
'export WINDOWS_CUDA_HOME="${TOOLKITS}/cuda-windows-${{ matrix.cuda-version }}"',
script,
)

def test_cuda_builds_take_the_node_driver_on_an_unpinned_v3(self):
# This job used to pin linux_job_v2 to a test-infra SHA because that was
Expand Down
9 changes: 5 additions & 4 deletions .ci/scripts/wheel/install_cuda_redist.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@

"""Assemble part of a CUDA toolkit from NVIDIA's redistributable component archives.

The Windows CUDA wheel end-to-end jobs cover several CUDA trains while the CI images carry
one toolkit, and NVIDIA publishes no full Windows installer for every release. The redist
The Windows CUDA wheel end-to-end jobs and cuda-windows.yml cover several CUDA trains while
the CI images carry one toolkit, and NVIDIA publishes no full Windows installer for every release. The redist
archives exist for every release on both platforms, need no installer or admin rights, and
are checksummed in a JSON index. Merging the components a job needs into one directory
gives the layout nvcc, CMake's FindCUDAToolkit and torch's cross-compile step expect.
Expand All @@ -31,8 +31,9 @@
_REDIST = "https://developer.download.nvidia.com/compute/cuda/redist"

# The patch release each train resolves to: one entry per train in filter_cuda_matrix.py's
# SUPPORTED_CUDA_VERSIONS, which the end-to-end jobs take their matrix from. A train missing
# here fails the job at argument parsing rather than testing a different release.
# SUPPORTED_CUDA_VERSIONS, which the wheel end-to-end jobs take their matrix from, and in
# cuda-windows.yml's matrices. A train missing here fails the job at argument parsing rather
# than testing a different release.
_RELEASES = {"13.2": "13.2.2", "13.4": "13.4.1"}

# What a build of the CUDA backend needs. "a|b" names one component a release publishes
Expand Down
Loading
Loading