Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/cuda.yml
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ jobs:

merged_dir="${RUNNER_ARTIFACT_DIR}/merged"
mkdir -p "${merged_dir}"
python -m executorch.backends.cuda.merge_ptes \
python -m executorch.backends.cuda.scripts.merge_ptes \
--input-pte "${RUNNER_ARTIFACT_DIR}/native-sm80/linear/linear.pte" \
--input-pte "${RUNNER_ARTIFACT_DIR}/native-sm86/linear/linear.pte" \
--input-ptd "${sm80_ptd}" \
Expand Down
3 changes: 2 additions & 1 deletion backends/cuda/BUCK
Original file line number Diff line number Diff line change
Expand Up @@ -152,11 +152,12 @@ fbcode_target(
_kind = runtime.python_library,
name = "merge_ptes",
srcs = [
"merge_ptes.py",
"scripts/merge_ptes.py",
],
visibility = ["PUBLIC"],
deps = [
":cuda_backend",
":scripts_atomic_output",
"//caffe2:torch",
"//executorch/exir/_serialize:lib",
"//executorch/exir:schema",
Expand Down
43 changes: 23 additions & 20 deletions backends/cuda/merge_ptes.py → backends/cuda/scripts/merge_ptes.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,6 @@
import argparse
import copy
import hashlib
import os
import shutil
import tempfile
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Sequence, Tuple
Expand All @@ -23,7 +20,8 @@
decode_cuda_aoti_metadata,
encode_cuda_aoti_metadata,
)
from executorch.exir._serialize._cord import Cord
from executorch.backends.cuda.scripts._atomic_output import write_atomically
from executorch.exir._serialize._cord import Cord, FileBackedData
from executorch.exir._serialize._named_data_store import (
NamedDataStore,
NamedDataStoreOutput,
Expand All @@ -44,7 +42,6 @@
FlatTensorHeader,
)


CUDA_BACKEND_ID = "CudaBackend"


Expand Down Expand Up @@ -280,14 +277,16 @@ def _validate_shared_weights(
!= reference_entry.storage_nbytes
):
raise ValueError(
f"CUDA weight {fqn!r} has an invalid size in {reference.source.pte_path}"
f"CUDA weight {fqn!r} has an invalid size in "
f"{reference.source.pte_path}"
)
if (
candidate.blob_size(candidate_entry.storage_key)
!= candidate_entry.storage_nbytes
):
raise ValueError(
f"CUDA weight {fqn!r} has an invalid size in {candidate.source.pte_path}"
f"CUDA weight {fqn!r} has an invalid size in "
f"{candidate.source.pte_path}"
)
if reference.blob_sha256(reference_entry.storage_key) != candidate.blob_sha256(
candidate_entry.storage_key
Expand Down Expand Up @@ -559,7 +558,9 @@ def _prepare_merged_output(reference: _Artifact) -> Tuple[Program, NamedDataStor
for key, entry in entries.items()
if key not in cuda_so_keys
}
for filename, entries in reference.pte.named_data.external_data.items()
for filename, entries in (
reference.pte.named_data.external_data.items()
)
},
)
)
Expand Down Expand Up @@ -700,6 +701,8 @@ def main() -> None:
raise ValueError("--input-ptd must be provided once per --input-pte")
if args.input_ptd and args.output_ptd is None:
raise ValueError("--output-ptd is required when --input-ptd is provided")
if not args.input_ptd and args.output_ptd is not None:
raise ValueError("--output-ptd requires --input-ptd")
if args.fallback_ptd is not None and args.fallback_pte is None:
raise ValueError("--fallback-ptd requires --fallback-pte")
sources = [
Expand All @@ -718,20 +721,20 @@ def main() -> None:
else None
)
result = merge_cuda_pte_files_with_provenance(sources, fallback)
args.output_pte.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(
dir=args.output_pte.parent, prefix=f".{args.output_pte.name}.", delete=False
) as temporary:
temporary_path = Path(temporary.name)
result.pte.write_to_file(temporary)
os.replace(temporary_path, args.output_pte)

# The (much larger) PTD goes first, so a failed copy leaves the
# existing output PTE untouched instead of pairing a new PTE with
# a stale weights blob.
if args.output_ptd is not None:
if not args.input_ptd:
raise ValueError("--output-ptd requires --input-ptd")
args.output_ptd.parent.mkdir(parents=True, exist_ok=True)
if args.input_ptd[0].resolve() != args.output_ptd.resolve():
shutil.copyfile(args.input_ptd[0], args.output_ptd)
write_atomically(
args.output_ptd,
FileBackedData(str(args.input_ptd[0])).write_to_file,
mode_from=args.input_ptd[0],
)

write_atomically(
args.output_pte, result.pte.write_to_file, mode_from=sources[0].pte_path
)

print("Merged CUDA code provenance:")
print("delegate\tkind\ttarget\tsource PTE")
Expand Down
20 changes: 20 additions & 0 deletions backends/cuda/tests/targets.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -601,3 +601,23 @@ def define_common_targets(is_fbcode = False):
"//executorch/exir/backend/test:device_util",
],
)

python_unittest(
name = "test_merge_ptes",
srcs = [
"test_merge_ptes.py",
],
visibility = [
"//executorch/...",
],
deps = [
"//caffe2:torch",
"//executorch/backends/cuda:cuda_backend",
"//executorch/backends/cuda:merge_ptes",
"//executorch/exir:lib",
"//executorch/exir/_serialize:lib",
"//executorch/exir:schema",
"//executorch/exir/backend:compile_spec_schema",
"//executorch/extension/flat_tensor/serialize:serialize",
],
)
98 changes: 93 additions & 5 deletions backends/cuda/tests/test_merge_ptes.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
decode_cuda_aoti_metadata,
encode_cuda_aoti_metadata,
)
from executorch.backends.cuda.merge_ptes import (
from executorch.backends.cuda.scripts.merge_ptes import (
CudaPteInput,
main as merge_main,
merge_cuda_pte_files,
Expand All @@ -46,7 +46,7 @@
from executorch.extension.flat_tensor.serialize.serialize import FlatTensorSerializer


@patch("executorch.backends.cuda.merge_ptes.torch.version.hip", None)
@patch("executorch.backends.cuda.scripts.merge_ptes.torch.version.hip", None)
class TestMergeCudaPtes(unittest.TestCase):
def _write_artifact(
self,
Expand Down Expand Up @@ -585,7 +585,7 @@ def test_rejects_fallback_without_ptx(self) -> None:

def test_rejects_rocm(self) -> None:
with patch(
"executorch.backends.cuda.merge_ptes.torch.version.hip", "6.3"
"executorch.backends.cuda.scripts.merge_ptes.torch.version.hip", "6.3"
), self.assertRaisesRegex(RuntimeError, "only NVIDIA CUDA"):
merge_cuda_pte_files([])

Expand All @@ -602,7 +602,9 @@ def test_cli_writes_merged_pte_and_reuses_base_ptd(self) -> None:
first = self._write_artifact(sm80, 80, b"sm80-so", b"weight")
second = self._write_artifact(sm120, 120, b"sm120-so", b"weight")
fallback = self._write_artifact(fallback_dir, 75, b"fallback-so", b"weight")
self.assertIsNotNone(first.ptd_path)
assert first.ptd_path is not None
first.pte_path.chmod(0o640)
first.ptd_path.chmod(0o604)
output_pte = output / "model.pte"
output_ptd = output / "aoti_cuda_blob.ptd"

Expand Down Expand Up @@ -631,7 +633,12 @@ def test_cli_writes_merged_pte_and_reuses_base_ptd(self) -> None:
merge_main()

self.assertTrue(output_pte.is_file())
assert first.ptd_path is not None
self.assertEqual(output_pte.stat().st_mode & 0o777, 0o640)
self.assertEqual(output_ptd.stat().st_mode & 0o777, 0o604)
self.assertEqual(
{entry.name for entry in output.iterdir()},
{"model.pte", "aoti_cuda_blob.ptd"},
)
self.assertEqual(output_ptd.read_bytes(), first.ptd_path.read_bytes())
report = stdout.getvalue()
self.assertIn(f"cubin\tsm80\t{first.pte_path}", report)
Expand All @@ -642,6 +649,87 @@ def test_cli_writes_merged_pte_and_reuses_base_ptd(self) -> None:
report,
)

def test_cli_output_ptd_without_input_ptd_writes_nothing(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
source = root / "sm80"
source.mkdir()
input_pte = source / "model.pte"
input_pte.write_bytes(b"unused")
output_pte = root / "output" / "model.pte"
output_ptd = root / "output" / "aoti_cuda_blob.ptd"
with patch(
"sys.argv",
[
"merge_ptes",
"--input-pte",
str(input_pte),
"--output-pte",
str(output_pte),
"--output-ptd",
str(output_ptd),
],
), patch(
"executorch.backends.cuda.scripts.merge_ptes."
"merge_cuda_pte_files_with_provenance"
) as merge, self.assertRaisesRegex(
ValueError, "requires --input-ptd"
):
merge_main()
merge.assert_not_called()
self.assertFalse(output_pte.exists())
self.assertFalse(output_ptd.exists())

def test_cli_failed_ptd_copy_leaves_no_output(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
sm80 = root / "sm80"
sm120 = root / "sm120"
output = root / "output"
sm80.mkdir()
sm120.mkdir()
inputs = [
self._write_artifact(sm80, 80, b"sm80-so", b"weight", ptx_compute=0),
self._write_artifact(sm120, 120, b"sm120-so", b"weight", ptx_compute=0),
]
output.mkdir()
output_pte = output / "model.pte"
output_pte.write_bytes(b"previous pte")
output_ptd = output / "aoti_cuda_blob.ptd"
output_ptd.write_bytes(b"previous ptd")
argv = ["merge_ptes"]
for artifact in inputs:
argv += ["--input-pte", str(artifact.pte_path)]
for artifact in inputs:
assert artifact.ptd_path is not None
argv += ["--input-ptd", str(artifact.ptd_path)]
argv += [
"--output-pte",
str(output_pte),
"--output-ptd",
str(output_ptd),
]

class FailingFileData:
def __init__(self, path: str) -> None:
pass

def write_to_file(self, stream) -> None:
stream.write(b"partial")
raise OSError("copy failed")

with patch("sys.argv", argv), patch(
"executorch.backends.cuda.scripts.merge_ptes.FileBackedData",
FailingFileData,
), self.assertRaisesRegex(OSError, "copy failed"):
merge_main()
self.assertEqual(output_pte.read_bytes(), b"previous pte")
self.assertEqual(output_ptd.read_bytes(), b"previous ptd")
self.assertEqual(
{entry.name for entry in output.iterdir()},
{"model.pte", "aoti_cuda_blob.ptd"},
)


if __name__ == "__main__":
unittest.main()
2 changes: 1 addition & 1 deletion examples/cuda/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ and weights can be combined so the runtime selects an exactly matching native
AOTI library, or uses one explicitly designated PTX fallback.

```bash
python -m executorch.backends.cuda.merge_ptes \
python -m executorch.backends.cuda.scripts.merge_ptes \
--input-pte a100/model.pte \
--input-pte rtx5090/model.pte \
--input-ptd a100/aoti_cuda_blob.ptd \
Expand Down
Loading