diff --git a/backends/cuda/runtime/cuda_backend.cpp b/backends/cuda/runtime/cuda_backend.cpp index 933a9e25df4..25319cdbc4e 100644 --- a/backends/cuda/runtime/cuda_backend.cpp +++ b/backends/cuda/runtime/cuda_backend.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,38 @@ std::string_view trim(std::string_view s) { return s.substr(start, end - start); } +// The CUDA device index named by a "cuda" or "cuda:N" device string, as +// written by CudaPartitioner into the target_device compile spec and accepted +// from the target_device load option. +Result parse_cuda_device_string(std::string_view value) { + while (!value.empty() && value.back() == '\0') { + value.remove_suffix(1); + } + value = trim(value); + constexpr std::string_view kCuda = "cuda"; + if (value == kCuda) { + return 0; + } + const bool has_prefix = value.size() > kCuda.size() + 1 && + value.substr(0, kCuda.size()) == kCuda && value[kCuda.size()] == ':'; + int index = -1; + bool parsed = false; + if (has_prefix) { + const std::string_view digits = value.substr(kCuda.size() + 1); + const auto result = + std::from_chars(digits.data(), digits.data() + digits.size(), index); + parsed = result.ec == std::errc() && + result.ptr == digits.data() + digits.size() && index >= 0; + } + ET_CHECK_OR_RETURN_ERROR( + parsed, + InvalidArgument, + "init: target_device '%.*s' is not of the form cuda[:N]", + static_cast(value.size()), + value.data()); + return index; +} + // The CUDA device index named by the target_device compile spec, in the // "cuda" or "cuda:N" form CudaPartitioner writes, or -1 without one. Result parse_cuda_target_device(ArrayRef compile_specs) { @@ -119,32 +152,7 @@ Result parse_cuda_target_device(ArrayRef compile_specs) { value = std::string_view( static_cast(spec.value.buffer), spec.value.nbytes); } - while (!value.empty() && value.back() == '\0') { - value.remove_suffix(1); - } - value = trim(value); - constexpr std::string_view kCuda = "cuda"; - if (value == kCuda) { - return 0; - } - const bool has_prefix = value.size() > kCuda.size() + 1 && - value.substr(0, kCuda.size()) == kCuda && value[kCuda.size()] == ':'; - int index = -1; - bool parsed = false; - if (has_prefix) { - const std::string_view digits = value.substr(kCuda.size() + 1); - const auto result = - std::from_chars(digits.data(), digits.data() + digits.size(), index); - parsed = result.ec == std::errc() && - result.ptr == digits.data() + digits.size() && index >= 0; - } - ET_CHECK_OR_RETURN_ERROR( - parsed, - InvalidArgument, - "init: target_device '%.*s' is not of the form cuda[:N]", - static_cast(value.size()), - value.data()); - return index; + return parse_cuda_device_string(value); } return -1; } @@ -559,6 +567,72 @@ class ET_EXPERIMENTAL CudaBackend final device_index, device_count); } + + // A target_device load option (the compile spec's key, passed as a + // runtime spec at load) remaps the method to another device: the one + // device every delegate and planned buffer of the method was lowered + // for is replaced by the one the caller names, so a program exported + // on whichever GPU was free (cuda:3, say) loads on a single-GPU box as + // cuda:0. The remap is only defined for a method whose delegates all + // share that one exported device; the registry below refuses a method + // whose delegates were lowered for different devices, and the loaders + // refuse one whose planned buffers span devices. Without the option, + // the exported device stands, exactly as before. + const auto device_override = + context.get_runtime_spec(kTargetDeviceSpec); + ET_CHECK_OR_RETURN_ERROR( + device_override.ok() || device_override.error() == Error::NotFound, + InvalidArgument, + "init: the '%s' load option for method '%s' must be a string of " + "the form cuda[:N]", + kTargetDeviceSpec, + method_name.c_str()); + const int exported_device_index = device_index; + const void* remap_registry_key = nullptr; + if (device_override.ok()) { + const auto parsed_override = parse_cuda_device_string( + device_override.get() == nullptr ? "" : device_override.get()); + ET_CHECK_OK_OR_RETURN_ERROR(parsed_override.error()); + int device_count = 0; + ET_CUDA_CHECK_OR_RETURN_ERROR(cudaGetDeviceCount(&device_count)); + ET_CHECK_OR_RETURN_ERROR( + parsed_override.get() < device_count, + InvalidArgument, + "init: method '%s' remapped to cuda:%d by the '%s' load option, " + "but this process sees %d CUDA device(s)", + method_name.c_str(), + parsed_override.get(), + kTargetDeviceSpec, + device_count); + device_index = parsed_override.get(); + remap_registry_key = context.get_named_data_map(); + if (device_index != exported_device_index) { + ET_LOG( + Info, + "Method '%s': target_device load option remaps cuda:%d to " + "cuda:%d", + method_name.c_str(), + exported_device_index, + device_index); + } + if (remap_registry_key != nullptr) { + std::lock_guard guard(remapped_methods_mutex_); + const auto it = remapped_methods_.find( + RemappedMethodKey{remap_registry_key, method_name}); + ET_CHECK_OR_RETURN_ERROR( + it == remapped_methods_.end() || + it->second.exported_device == exported_device_index, + InvalidArgument, + "init: method '%s' has delegates lowered for cuda:%d and " + "cuda:%d, so the '%s' load option cannot remap it; remap " + "needs a method lowered for a single device", + method_name.c_str(), + it == remapped_methods_.end() ? exported_device_index + : it->second.exported_device, + exported_device_index, + kTargetDeviceSpec); + } + } const auto device_guard = CUDAGuard::create(device_index); ET_CHECK_OK_OR_RETURN_ERROR(device_guard.error()); @@ -808,6 +882,28 @@ class ET_EXPERIMENTAL CudaBackend final mutable_state_note_handle(handle); + if (remap_registry_key != nullptr) { + std::lock_guard guard(remapped_methods_mutex_); + const RemappedMethodKey key{remap_registry_key, method_name}; + const auto it = + remapped_methods_ + .try_emplace(key, RemappedMethod{exported_device_index, 0}) + .first; + ET_CHECK_OR_RETURN_ERROR( + it->second.exported_device == exported_device_index, + InvalidArgument, + "init: method '%s' has delegates lowered for cuda:%d and " + "cuda:%d, so the '%s' load option cannot remap it; remap " + "needs a method lowered for a single device", + method_name.c_str(), + it->second.exported_device, + exported_device_index, + kTargetDeviceSpec); + ++it->second.delegates; + handle->exported_device_index = exported_device_index; + handle->remap_registry_key = remap_registry_key; + } + { std::lock_guard guard(live_handles_mutex_); ++live_handles_[device_index]; @@ -1109,6 +1205,15 @@ class ET_EXPERIMENTAL CudaBackend final remove_error.message().c_str()); } + if (handle->remap_registry_key != nullptr) { + std::lock_guard guard(remapped_methods_mutex_); + const auto it = remapped_methods_.find( + RemappedMethodKey{handle->remap_registry_key, handle->method_name}); + if (it != remapped_methods_.end() && --it->second.delegates == 0) { + remapped_methods_.erase(it); + } + } + const int device_index = handle->device_index; delete handle; @@ -1590,6 +1695,32 @@ class ET_EXPERIMENTAL CudaBackend final mutable std::mutex live_handles_mutex_; mutable std::unordered_map live_handles_; + // Methods a target_device load option remapped, by (named data map, method + // name): the device the method's delegates were lowered for, and how many + // of them are alive. Every delegate of the method must agree on the + // exported device, or the remap would move only part of the method. + struct RemappedMethod { + int exported_device; + size_t delegates; + }; + struct RemappedMethodKey { + const void* data_map; + std::string method_name; + bool operator==(const RemappedMethodKey& other) const { + return data_map == other.data_map && method_name == other.method_name; + } + }; + struct RemappedMethodKeyHash { + size_t operator()(const RemappedMethodKey& key) const { + return std::hash{}(key.data_map) ^ + std::hash{}(key.method_name); + } + }; + mutable std::mutex remapped_methods_mutex_; + mutable std:: + unordered_map + remapped_methods_; + // --------------------------------------------------------------- // Per-weight constant cache. // diff --git a/backends/cuda/runtime/cuda_delegate_handle.h b/backends/cuda/runtime/cuda_delegate_handle.h index 4a16e804566..54ce812de1d 100644 --- a/backends/cuda/runtime/cuda_delegate_handle.h +++ b/backends/cuda/runtime/cuda_delegate_handle.h @@ -330,6 +330,13 @@ struct CudaDelegateHandle : public aoti::AOTIDelegateHandle { // The CUDA device the container, its weights and its IO buffers live on. int device_index{0}; + // When a target_device load option remapped this delegate, the device the + // program was lowered for, and the named data map that identifies the + // method in the backend's remap registry. exported_device_index stays -1 + // and remap_registry_key stays null when no remap was requested. + int exported_device_index{-1}; + const void* remap_registry_key{nullptr}; + // CUDA graph state (warmup, capture, replay, static buffers) CudaGraphState cuda_graph_state; diff --git a/backends/cuda/tests/test_target_device.py b/backends/cuda/tests/test_target_device.py index f5c87a3a99c..2346a121a6c 100644 --- a/backends/cuda/tests/test_target_device.py +++ b/backends/cuda/tests/test_target_device.py @@ -112,6 +112,30 @@ def _export(kind: str, directory: str, target: int) -> None: torch.save(expected, Path(directory) / "expected.pt") +# A target_device load option moves the program to the named device +# instead: exported for one device, run on the other. +_RUN_REMAP = """ +import sys + +import torch +from executorch.runtime import Runtime + +directory, current, remap = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]) +torch.cuda.set_device(current) +method = Runtime.get().load_program( + directory + "/model.pte", + data_path=directory + "/aoti_cuda_blob.ptd", + backend_options={"CudaBackend": {"target_device": f"cuda:{remap}"}}, +).load_method("forward") +inputs = torch.load(directory + "/inputs.pt") +expected = torch.load(directory + "/expected.pt") +for _ in range(2): + (output,) = method.execute(inputs) + torch.testing.assert_close(output.cpu(), expected, atol=2e-2, rtol=2e-2) +assert torch.cuda.current_device() == current, torch.cuda.current_device() +""" + + # Programs on both devices in one process, with cross-method sharing enabled # as a runner would: each runs on its own device, and freeing both trims each # device's pool. @@ -192,3 +216,51 @@ def test_programs_on_both_devices_in_one_process(self): def test_int4_matmul_runs_on_the_target_device(self): self._check("int4") + + def test_load_option_remaps_to_the_named_device(self): + for target, remap in ((0, 1), (1, 0)): + with tempfile.TemporaryDirectory() as directory: + _export("dense", directory, target) + with self.subTest(target=target, remap=remap): + result = subprocess.run( + [ + sys.executable, + "-c", + _RUN_REMAP, + directory, + str(target), + str(remap), + ], + capture_output=True, + text=True, + timeout=600, + ) + self.assertEqual( + result.returncode, + 0, + f"lowered for cuda:{target}, remapped to cuda:{remap}:\n" + f"{result.stdout}\n{result.stderr}", + ) + + def test_load_option_remap_to_a_missing_device_fails(self): + with tempfile.TemporaryDirectory() as directory: + _export("dense", directory, 0) + result = subprocess.run( + [ + sys.executable, + "-c", + _RUN_REMAP, + directory, + "0", + str(torch.cuda.device_count()), + ], + capture_output=True, + text=True, + timeout=600, + ) + self.assertNotEqual( + result.returncode, + 0, + f"remap to a missing device loaded:\n" + f"{result.stdout}\n{result.stderr}", + ) diff --git a/extension/module/module.cpp b/extension/module/module.cpp index 6a6fd143949..58dbb0471e8 100644 --- a/extension/module/module.cpp +++ b/extension/module/module.cpp @@ -8,7 +8,9 @@ #include +#include #include +#include #include #include @@ -120,6 +122,51 @@ runtime::Result> make_data_loader( } return data_loader; } + +// The CUDA backend's target_device load option: the device a method's +// planned buffers move to at load, replacing the one they were planned +// for. Named here because Module allocates the buffers before any +// delegate initializes; the CUDA delegate reads the same option (under +// its own backend id) and moves the rest of the method. Returns -1 when +// the option is not set. +runtime::Result cuda_target_device_override( + const runtime::LoadBackendOptionsMap* backend_options) { + if (backend_options == nullptr) { + return -1; + } + for (const auto& option : backend_options->get_options("CudaBackend")) { + if (std::strcmp(option.key, "target_device") != 0) { + continue; + } + const auto* text = + std::get_if>( + &option.value); + if (text == nullptr) { + return runtime::Error::InvalidArgument; + } + std::string_view value(text->data()); + while (!value.empty() && value.back() == '\0') { + value.remove_suffix(1); + } + if (value == "cuda") { + return 0; + } + int index = -1; + bool parsed = false; + if (value.size() > 5 && value.substr(0, 5) == "cuda:") { + const std::string_view digits = value.substr(5); + const auto result = + std::from_chars(digits.data(), digits.data() + digits.size(), index); + parsed = result.ec == std::errc() && + result.ptr == digits.data() + digits.size() && index >= 0; + } + if (!parsed) { + return runtime::Error::InvalidArgument; + } + return index; + } + return -1; +} } // namespace Module::Module( @@ -450,7 +497,8 @@ Module::make_planned_memory_with_shared_arenas( runtime::Result> Module::make_planned_memory_with_devices( - const ET_RUNTIME_NAMESPACE::MethodMeta& method_meta) { + const ET_RUNTIME_NAMESPACE::MethodMeta& method_meta, + int target_device_override) { auto planned = std::make_unique(); const size_t num_buffers = method_meta.num_memory_planned_buffers(); planned->planned_buffers.reserve(num_buffers); @@ -458,6 +506,10 @@ Module::make_planned_memory_with_devices( planned->device_buffers.reserve(num_buffers); planned->planned_devices.reserve(num_buffers); + // The device the method's device buffers were planned for, once one is + // seen. A target_device override moves every buffer together, so a + // method planned across devices has no single device to move to. + int planned_device = -1; for (size_t i = 0; i < num_buffers; ++i) { auto size = method_meta.memory_planned_buffer_size(i); ET_CHECK_OK_OR_RETURN_ERROR( @@ -465,6 +517,29 @@ Module::make_planned_memory_with_devices( auto device = method_meta.memory_planned_buffer_device(i); ET_CHECK_OK_OR_RETURN_ERROR( device.error(), "Failed to get buffer device for index %zu", i); + if (target_device_override >= 0 && !device->is_cpu()) { + ET_CHECK_OR_RETURN_ERROR( + device->type() == runtime::etensor::DeviceType::CUDA, + InvalidArgument, + "target_device load option cannot move buffer %zu: it is on " + "device type %d, not CUDA", + i, + static_cast(device->type())); + ET_CHECK_OR_RETURN_ERROR( + planned_device < 0 || planned_device == device->index(), + InvalidArgument, + "target_device load option cannot move this method: its " + "buffers are planned on cuda:%d and cuda:%d, and a remap " + "needs a single device", + planned_device, + static_cast(device->index())); + planned_device = device->index(); + device = + runtime::Result(runtime::etensor::Device( + device->type(), + static_cast( + target_device_override))); + } planned->planned_devices.push_back(device.get()); if (device->is_cpu()) { @@ -598,7 +673,11 @@ runtime::Error Module::load_method( // Device-aware path: allocate CPU and device buffers. The device // span is owned by the HierarchicalAllocator inside PlannedMemory. - auto planned_res = make_planned_memory_with_devices(meta); + const auto device_override = + cuda_target_device_override(effective_backend_options); + ET_CHECK_OK_OR_RETURN_ERROR(device_override.error()); + auto planned_res = + make_planned_memory_with_devices(meta, device_override.get()); ET_CHECK_OK_OR_RETURN_ERROR(planned_res.error()); method_holder.planned_memory = std::move(planned_res.get()); planned_memory = method_holder.planned_memory->planned_memory.get(); diff --git a/extension/module/module.h b/extension/module/module.h index df66df0c871..b2417a65239 100644 --- a/extension/module/module.h +++ b/extension/module/module.h @@ -744,7 +744,8 @@ class Module { std::vector>& shared_arenas); runtime::Result> make_planned_memory_with_devices( - const ET_RUNTIME_NAMESPACE::MethodMeta& method_meta); + const ET_RUNTIME_NAMESPACE::MethodMeta& method_meta, + int target_device_override = -1); runtime::Result> get_mem_planned_buffer_sizes( const std::string& method_name); runtime::Result> get_max_mem_planned_buffer_sizes(); diff --git a/extension/pybindings/pybindings.cpp b/extension/pybindings/pybindings.cpp index 44eaa4f686a..13b28bfff81 100644 --- a/extension/pybindings/pybindings.cpp +++ b/extension/pybindings/pybindings.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -4381,12 +4383,58 @@ bool has_device_buffers(const MethodMeta& method_meta) { return false; } +/// The CUDA backend's target_device load option: the device a method's +/// planned buffers move to at load, replacing the one they were planned +/// for. Named here because this loader allocates the buffers before any +/// delegate initializes; the CUDA delegate reads the same option (under +/// its own backend id) and moves the rest of the method, and +/// extension/module does the same for the C++ loader. Returns -1 when +/// the option is not set. +Result cuda_target_device_override( + const LoadBackendOptionsMap* backend_options) { + if (backend_options == nullptr) { + return -1; + } + for (const auto& option : backend_options->get_options("CudaBackend")) { + if (std::strcmp(option.key, "target_device") != 0) { + continue; + } + const auto* text = + std::get_if>(&option.value); + if (text == nullptr) { + return Error::InvalidArgument; + } + std::string_view value(text->data()); + while (!value.empty() && value.back() == '\0') { + value.remove_suffix(1); + } + if (value == "cuda") { + return 0; + } + int index = -1; + bool parsed = false; + if (value.size() > 5 && value.substr(0, 5) == "cuda:") { + const std::string_view digits = value.substr(5); + const auto result = + std::from_chars(digits.data(), digits.data() + digits.size(), index); + parsed = result.ec == std::errc() && + result.ptr == digits.data() + digits.size() && index >= 0; + } + if (!parsed) { + return Error::InvalidArgument; + } + return index; + } + return -1; +} + /// Arenas sized and placed for a single method, used when that method's /// buffers cannot come from the program-wide host arenas. Returns nullptr when /// every buffer is on the host, so one pass over the metadata answers both /// whether the method needs its own arenas and how big they are. std::shared_ptr make_method_memory( - const MethodMeta& method_meta) { + const MethodMeta& method_meta, + const LoadBackendOptionsMap* backend_options = nullptr) { const size_t num_buffers = method_meta.num_memory_planned_buffers(); std::vector sizes; std::vector devices; @@ -4406,6 +4454,40 @@ std::shared_ptr make_method_memory( if (!needs_device_memory) { return nullptr; } + const auto device_override = cuda_target_device_override(backend_options); + THROW_IF_ERROR( + device_override.error(), + "target_device load option must be a string of the form cuda[:N]"); + if (device_override.get() >= 0) { + // A remap moves every device buffer together, so a method planned + // across devices has no single device to move to. + int planned_device = -1; + for (auto& device : devices) { + if (device.is_cpu()) { + continue; + } + THROW_IF_ERROR( + device.type() == runtime::etensor::DeviceType::CUDA + ? Error::Ok + : Error::InvalidArgument, + "target_device load option cannot move a buffer on device " + "type %d, not CUDA", + static_cast(device.type())); + THROW_IF_ERROR( + planned_device < 0 || planned_device == device.index() + ? Error::Ok + : Error::InvalidArgument, + "target_device load option cannot move this method: its " + "buffers are planned on cuda:%d and cuda:%d, and a remap " + "needs a single device", + planned_device, + static_cast(device.index())); + planned_device = device.index(); + device = runtime::etensor::Device( + device.type(), + static_cast(device_override.get())); + } + } return std::make_shared(std::move(sizes), std::move(devices)); } @@ -5566,7 +5648,8 @@ struct PyProgram final { // accelerator method cannot make the rest of the program unloadable. A // host-only method keeps sharing the program-wide arenas, so its planned // memory is not isolated from the other host-only methods of this program. - auto method_memory = make_method_memory(meta.get()); + auto method_memory = + make_method_memory(meta.get(), &state_->backend_options_); auto memory = method_memory ? std::move(method_memory) : memory_; Result res = state_->program_->load_method( method_name.c_str(),