Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions backends/cuda/batching/cuda_executor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,22 @@ Result<std::unique_ptr<CudaExecutor>> CudaExecutor::create(
declared_seqs,
metadata::detail::read_int_method(*module, kMaxSeqsMethod));
const bool sequence_layout = declared_seqs.has_value();
ET_ASSIGN_OR_RETURN(
sequence_info_abi,
metadata::detail::read_int_method(
*module, "get_offgraph_kv_sequence_info_abi"));
ET_CHECK_OR_RETURN_ERROR(
sequence_info_abi.value_or(1) == 1 ||
(sequence_layout && sequence_info_abi == 2),
InvalidProgram,
"CudaExecutor: unsupported sequence-info ABI");
Comment thread
Gasoonjia marked this conversation as resolved.
Resolved
// Runtime columns have not been qualified with draft (DFlash) programs:
// column moves interact with pending draft owners, and the combination
// is rejected by the factory as well as here.
ET_CHECK_OR_RETURN_ERROR(
!(sequence_info_abi == 2 && geometry.dflash_block_size),
InvalidProgram,
"CudaExecutor: runtime sequence columns do not support DFlash");
std::int64_t max_cells = 0;
if (sequence_layout) {
ET_CHECK_OR_RETURN_ERROR(
Expand Down Expand Up @@ -527,6 +543,7 @@ Result<std::unique_ptr<CudaExecutor>> CudaExecutor::create(
static_cast<int>(sequence_layout ? max_context_length : max_cells);
cfg.kv_dtype = kv_dtype;
cfg.max_write = max_step_tokens;
cfg.runtime_sequence_columns = sequence_info_abi == 2;
if (sequence_layout) {
cfg.max_seqs = static_cast<int>(*declared_seqs);
}
Expand Down Expand Up @@ -1089,6 +1106,12 @@ OffGraphKVMetrics CudaExecutor::kv_metrics() const {
return kv_->metrics();
}

Error CudaExecutor::compact_kv_columns(int spare_columns) {
ET_CHECK_OR_RETURN_ERROR(
!raw_executing_, InvalidState, "CudaExecutor: compact between executes");
return kv_->compact_columns(spare_columns);
}

std::vector<ForwardMethodCalls> CudaExecutor::method_calls() const {
std::vector<ForwardMethodCalls> calls;
calls.reserve(methods_.size() + 1);
Expand Down
3 changes: 2 additions & 1 deletion backends/cuda/batching/cuda_executor.h
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ class ET_EXPERIMENTAL CudaExecutor : public llm_batching::Executor {

// The cache's pool usage, for memory reporting.
OffGraphKVMetrics kv_metrics() const;
::executorch::runtime::Error compact_kv_columns(int spare_columns = 1);

// Every method, narrowest first, with how many forwards it ran. Read it
// between executes, not while one runs.
Expand Down Expand Up @@ -273,7 +274,7 @@ class ET_EXPERIMENTAL CudaExecutor : public llm_batching::Executor {
llm_cache::InstallGuard install_guard_;
std::unique_ptr<::executorch::extension::Module> module_;
llm_cache::BatchControl* const ctl_;
const CudaKVCache* const kv_;
CudaKVCache* const kv_;
std::string backend_id_;
std::int32_t vocab_size_;
int max_session_tokens_;
Expand Down
28 changes: 24 additions & 4 deletions backends/cuda/passes/lower_offgraph_kv.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ def ring_physical_capacity(window: int, max_write: int) -> int:

def parse_offgraph_kv_manifest(value: bytes) -> dict[str, Any]: # noqa: C901
manifest = json.loads(value.decode("utf-8"))
if manifest.get("version") != 1:
raise ValueError("off-graph KV manifest version must be 1")
if manifest.get("version") not in (1, 2):
raise ValueError("off-graph KV manifest version must be 1 or 2")
maximum_capacity = manifest.get("maximum_capacity")
if not isinstance(maximum_capacity, int) or maximum_capacity <= 0:
raise ValueError("off-graph maximum_capacity must be positive")
Expand All @@ -95,6 +95,16 @@ def parse_offgraph_kv_manifest(value: bytes) -> dict[str, Any]: # noqa: C901
layout = manifest.setdefault("layout", "sequence")
if layout not in ("sequence", *PACKED_LAYOUTS):
raise ValueError(f"invalid off-graph KV layout {layout!r}")
runtime_columns = manifest.setdefault("runtime_columns", False)
if not isinstance(runtime_columns, bool):
raise ValueError("off-graph runtime_columns must be a bool")
if manifest["version"] == 2:
if layout != "batched-sequence" or not runtime_columns:
raise ValueError(
"off-graph version 2 requires batched-sequence runtime columns"
)
elif runtime_columns:
raise ValueError("off-graph runtime columns require manifest version 2")
if layout == "batched-sequence":
for key in ("max_seqs", "max_segments"):
if not isinstance(manifest.get(key), int) or manifest[key] <= 0:
Expand Down Expand Up @@ -297,6 +307,7 @@ def batched_sequence_step(
causal: bool,
out_dtype: torch.dtype | None = None,
context_window: bool = False,
runtime_columns: bool = False,
) -> torch.Tensor:
"""Write this step's K/V into each sequence's column, then attend.

Expand All @@ -311,9 +322,12 @@ def batched_sequence_step(
K/V-width entries. ``k``/``v`` are BHSD; the storage is BSHD.
``context_window`` gives every draft block query the same retained context
before the block; ordinary attention uses each query's own window.
With ``runtime_columns``, ABI-5 ``seq_info[:, 4]`` supplies the physical
pitch for writes and attention; ``columns`` remains the declared maximum.
"""
position = position.reshape(-1)
slots = (position % rows) * columns + seq_cols[: k.shape[2]]
pitch = seq_info[0, 4].to(torch.int64) if runtime_columns else columns
slots = (position % rows) * pitch + seq_cols[: k.shape[2]]
_write(k_storage, slots, k)
_write(v_storage, slots, v)
heads, dim = k_storage.shape[2], k_storage.shape[3]
Expand Down Expand Up @@ -342,6 +356,7 @@ def _batched_sequence_step_fn(
causal: bool,
out_dtype,
context_window: bool = False,
runtime_columns: bool = False,
):
def fn(q, k, v, position, k_storage, v_storage, seq_cols, seq_info):
return batched_sequence_step(
Expand All @@ -361,6 +376,7 @@ def fn(q, k, v, position, k_storage, v_storage, seq_cols, seq_info):
causal,
out_dtype,
context_window,
runtime_columns,
)

return fn
Expand Down Expand Up @@ -737,7 +753,10 @@ def _lower_batched_sequence(
seq_info = self._cell_buffer(
exported_program,
OFFGRAPH_KV_SEQ_INFO_FQN,
(self._manifest["max_segments"], 4),
(
self._manifest["max_segments"],
5 if self._manifest.get("runtime_columns", False) else 4,
),
torch.int32,
device,
)
Expand All @@ -753,6 +772,7 @@ def _lower_batched_sequence(
self._manifest["causal"],
out_dtype,
self._manifest.get("context_window", False),
self._manifest.get("runtime_columns", False),
)
return fn, call_args, example_args

Expand Down
Loading
Loading