diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index d0bc499aae9..4e7dd0a8c2f 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -365,7 +365,7 @@ jobs: merged_dir="${RUNNER_ARTIFACT_DIR}/merged" mkdir -p "${merged_dir}" - python -m executorch.backends.cuda.merge_ptes \ + python -m executorch.backends.cuda.scripts.merge_ptes \ --input-pte "${RUNNER_ARTIFACT_DIR}/native-sm80/linear/linear.pte" \ --input-pte "${RUNNER_ARTIFACT_DIR}/native-sm86/linear/linear.pte" \ --input-ptd "${sm80_ptd}" \ diff --git a/backends/cuda/BUCK b/backends/cuda/BUCK index 2180d57c9d8..2e831cb3729 100644 --- a/backends/cuda/BUCK +++ b/backends/cuda/BUCK @@ -152,11 +152,12 @@ fbcode_target( _kind = runtime.python_library, name = "merge_ptes", srcs = [ - "merge_ptes.py", + "scripts/merge_ptes.py", ], visibility = ["PUBLIC"], deps = [ ":cuda_backend", + ":scripts_atomic_output", "//caffe2:torch", "//executorch/exir/_serialize:lib", "//executorch/exir:schema", diff --git a/backends/cuda/merge_ptes.py b/backends/cuda/scripts/merge_ptes.py similarity index 95% rename from backends/cuda/merge_ptes.py rename to backends/cuda/scripts/merge_ptes.py index 184bbfd3b86..002164fa5e2 100644 --- a/backends/cuda/merge_ptes.py +++ b/backends/cuda/scripts/merge_ptes.py @@ -7,9 +7,6 @@ import argparse import copy import hashlib -import os -import shutil -import tempfile from dataclasses import dataclass, replace from pathlib import Path from typing import Dict, Iterable, List, Optional, Sequence, Tuple @@ -23,7 +20,8 @@ decode_cuda_aoti_metadata, encode_cuda_aoti_metadata, ) -from executorch.exir._serialize._cord import Cord +from executorch.backends.cuda.scripts._atomic_output import write_atomically +from executorch.exir._serialize._cord import Cord, FileBackedData from executorch.exir._serialize._named_data_store import ( NamedDataStore, NamedDataStoreOutput, @@ -44,7 +42,6 @@ FlatTensorHeader, ) - CUDA_BACKEND_ID = "CudaBackend" @@ -280,14 +277,16 @@ def _validate_shared_weights( != reference_entry.storage_nbytes ): raise ValueError( - f"CUDA weight {fqn!r} has an invalid size in {reference.source.pte_path}" + f"CUDA weight {fqn!r} has an invalid size in " + f"{reference.source.pte_path}" ) if ( candidate.blob_size(candidate_entry.storage_key) != candidate_entry.storage_nbytes ): raise ValueError( - f"CUDA weight {fqn!r} has an invalid size in {candidate.source.pte_path}" + f"CUDA weight {fqn!r} has an invalid size in " + f"{candidate.source.pte_path}" ) if reference.blob_sha256(reference_entry.storage_key) != candidate.blob_sha256( candidate_entry.storage_key @@ -559,7 +558,9 @@ def _prepare_merged_output(reference: _Artifact) -> Tuple[Program, NamedDataStor for key, entry in entries.items() if key not in cuda_so_keys } - for filename, entries in reference.pte.named_data.external_data.items() + for filename, entries in ( + reference.pte.named_data.external_data.items() + ) }, ) ) @@ -700,6 +701,8 @@ def main() -> None: raise ValueError("--input-ptd must be provided once per --input-pte") if args.input_ptd and args.output_ptd is None: raise ValueError("--output-ptd is required when --input-ptd is provided") + if not args.input_ptd and args.output_ptd is not None: + raise ValueError("--output-ptd requires --input-ptd") if args.fallback_ptd is not None and args.fallback_pte is None: raise ValueError("--fallback-ptd requires --fallback-pte") sources = [ @@ -718,20 +721,20 @@ def main() -> None: else None ) result = merge_cuda_pte_files_with_provenance(sources, fallback) - args.output_pte.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile( - dir=args.output_pte.parent, prefix=f".{args.output_pte.name}.", delete=False - ) as temporary: - temporary_path = Path(temporary.name) - result.pte.write_to_file(temporary) - os.replace(temporary_path, args.output_pte) - + # The (much larger) PTD goes first, so a failed copy leaves the + # existing output PTE untouched instead of pairing a new PTE with + # a stale weights blob. if args.output_ptd is not None: - if not args.input_ptd: - raise ValueError("--output-ptd requires --input-ptd") - args.output_ptd.parent.mkdir(parents=True, exist_ok=True) if args.input_ptd[0].resolve() != args.output_ptd.resolve(): - shutil.copyfile(args.input_ptd[0], args.output_ptd) + write_atomically( + args.output_ptd, + FileBackedData(str(args.input_ptd[0])).write_to_file, + mode_from=args.input_ptd[0], + ) + + write_atomically( + args.output_pte, result.pte.write_to_file, mode_from=sources[0].pte_path + ) print("Merged CUDA code provenance:") print("delegate\tkind\ttarget\tsource PTE") diff --git a/backends/cuda/tests/targets.bzl b/backends/cuda/tests/targets.bzl index 42eccb404e9..0d85ef8e3e7 100644 --- a/backends/cuda/tests/targets.bzl +++ b/backends/cuda/tests/targets.bzl @@ -601,3 +601,23 @@ def define_common_targets(is_fbcode = False): "//executorch/exir/backend/test:device_util", ], ) + + python_unittest( + name = "test_merge_ptes", + srcs = [ + "test_merge_ptes.py", + ], + visibility = [ + "//executorch/...", + ], + deps = [ + "//caffe2:torch", + "//executorch/backends/cuda:cuda_backend", + "//executorch/backends/cuda:merge_ptes", + "//executorch/exir:lib", + "//executorch/exir/_serialize:lib", + "//executorch/exir:schema", + "//executorch/exir/backend:compile_spec_schema", + "//executorch/extension/flat_tensor/serialize:serialize", + ], + ) diff --git a/backends/cuda/tests/test_merge_ptes.py b/backends/cuda/tests/test_merge_ptes.py index 8ca8b8ecee5..8bfb6c9f946 100644 --- a/backends/cuda/tests/test_merge_ptes.py +++ b/backends/cuda/tests/test_merge_ptes.py @@ -20,7 +20,7 @@ decode_cuda_aoti_metadata, encode_cuda_aoti_metadata, ) -from executorch.backends.cuda.merge_ptes import ( +from executorch.backends.cuda.scripts.merge_ptes import ( CudaPteInput, main as merge_main, merge_cuda_pte_files, @@ -46,7 +46,7 @@ from executorch.extension.flat_tensor.serialize.serialize import FlatTensorSerializer -@patch("executorch.backends.cuda.merge_ptes.torch.version.hip", None) +@patch("executorch.backends.cuda.scripts.merge_ptes.torch.version.hip", None) class TestMergeCudaPtes(unittest.TestCase): def _write_artifact( self, @@ -585,7 +585,7 @@ def test_rejects_fallback_without_ptx(self) -> None: def test_rejects_rocm(self) -> None: with patch( - "executorch.backends.cuda.merge_ptes.torch.version.hip", "6.3" + "executorch.backends.cuda.scripts.merge_ptes.torch.version.hip", "6.3" ), self.assertRaisesRegex(RuntimeError, "only NVIDIA CUDA"): merge_cuda_pte_files([]) @@ -602,7 +602,9 @@ def test_cli_writes_merged_pte_and_reuses_base_ptd(self) -> None: first = self._write_artifact(sm80, 80, b"sm80-so", b"weight") second = self._write_artifact(sm120, 120, b"sm120-so", b"weight") fallback = self._write_artifact(fallback_dir, 75, b"fallback-so", b"weight") - self.assertIsNotNone(first.ptd_path) + assert first.ptd_path is not None + first.pte_path.chmod(0o640) + first.ptd_path.chmod(0o604) output_pte = output / "model.pte" output_ptd = output / "aoti_cuda_blob.ptd" @@ -631,7 +633,12 @@ def test_cli_writes_merged_pte_and_reuses_base_ptd(self) -> None: merge_main() self.assertTrue(output_pte.is_file()) - assert first.ptd_path is not None + self.assertEqual(output_pte.stat().st_mode & 0o777, 0o640) + self.assertEqual(output_ptd.stat().st_mode & 0o777, 0o604) + self.assertEqual( + {entry.name for entry in output.iterdir()}, + {"model.pte", "aoti_cuda_blob.ptd"}, + ) self.assertEqual(output_ptd.read_bytes(), first.ptd_path.read_bytes()) report = stdout.getvalue() self.assertIn(f"cubin\tsm80\t{first.pte_path}", report) @@ -642,6 +649,87 @@ def test_cli_writes_merged_pte_and_reuses_base_ptd(self) -> None: report, ) + def test_cli_output_ptd_without_input_ptd_writes_nothing(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "sm80" + source.mkdir() + input_pte = source / "model.pte" + input_pte.write_bytes(b"unused") + output_pte = root / "output" / "model.pte" + output_ptd = root / "output" / "aoti_cuda_blob.ptd" + with patch( + "sys.argv", + [ + "merge_ptes", + "--input-pte", + str(input_pte), + "--output-pte", + str(output_pte), + "--output-ptd", + str(output_ptd), + ], + ), patch( + "executorch.backends.cuda.scripts.merge_ptes." + "merge_cuda_pte_files_with_provenance" + ) as merge, self.assertRaisesRegex( + ValueError, "requires --input-ptd" + ): + merge_main() + merge.assert_not_called() + self.assertFalse(output_pte.exists()) + self.assertFalse(output_ptd.exists()) + + def test_cli_failed_ptd_copy_leaves_no_output(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + sm80 = root / "sm80" + sm120 = root / "sm120" + output = root / "output" + sm80.mkdir() + sm120.mkdir() + inputs = [ + self._write_artifact(sm80, 80, b"sm80-so", b"weight", ptx_compute=0), + self._write_artifact(sm120, 120, b"sm120-so", b"weight", ptx_compute=0), + ] + output.mkdir() + output_pte = output / "model.pte" + output_pte.write_bytes(b"previous pte") + output_ptd = output / "aoti_cuda_blob.ptd" + output_ptd.write_bytes(b"previous ptd") + argv = ["merge_ptes"] + for artifact in inputs: + argv += ["--input-pte", str(artifact.pte_path)] + for artifact in inputs: + assert artifact.ptd_path is not None + argv += ["--input-ptd", str(artifact.ptd_path)] + argv += [ + "--output-pte", + str(output_pte), + "--output-ptd", + str(output_ptd), + ] + + class FailingFileData: + def __init__(self, path: str) -> None: + pass + + def write_to_file(self, stream) -> None: + stream.write(b"partial") + raise OSError("copy failed") + + with patch("sys.argv", argv), patch( + "executorch.backends.cuda.scripts.merge_ptes.FileBackedData", + FailingFileData, + ), self.assertRaisesRegex(OSError, "copy failed"): + merge_main() + self.assertEqual(output_pte.read_bytes(), b"previous pte") + self.assertEqual(output_ptd.read_bytes(), b"previous ptd") + self.assertEqual( + {entry.name for entry in output.iterdir()}, + {"model.pte", "aoti_cuda_blob.ptd"}, + ) + if __name__ == "__main__": unittest.main() diff --git a/examples/cuda/README.md b/examples/cuda/README.md index 022d7256618..e4662a0baee 100644 --- a/examples/cuda/README.md +++ b/examples/cuda/README.md @@ -43,7 +43,7 @@ and weights can be combined so the runtime selects an exactly matching native AOTI library, or uses one explicitly designated PTX fallback. ```bash -python -m executorch.backends.cuda.merge_ptes \ +python -m executorch.backends.cuda.scripts.merge_ptes \ --input-pte a100/model.pte \ --input-pte rtx5090/model.pte \ --input-ptd a100/aoti_cuda_blob.ptd \