From 7ee6c466e0a06963f7b6797e4ba4d92d2f4d5fda Mon Sep 17 00:00:00 2001 From: Stephen Jia Date: Thu, 19 Nov 2026 14:18:41 -0800 Subject: [PATCH] [ET-VK][runtime] Write one vec4 texel per thread in buffer permute and view Pull Request resolved: https://github.com/pytorch/executorch/pull/25305 ## Ulterior Motive Buffer tensors pad the packed dim to a multiple of four. This commit recovers the latency that padding costs a model with decomposed pixel shuffles, while keeping the padding. ## Rationale **What**: `permute` and `view_copy` get vec4 shader variants for buffer outputs whose packed dim has stride 1. Each thread writes one texel of four packed-dim elements. **Why**: Exporting with decompositions turns `pixel_shuffle` into `view -> permute -> view`. The permute output's packed dim has size 2, so padding doubles the buffer and half of its elements are padding. The scalar shaders spend one thread on each of those elements, and the following view loses its same-physical-order fast path. ## Details - Dispatch - **Before**: `permute_buffer` and `view_buffer` launch one thread per padded output element and do scalar writes. - **After**: if `is_packed_dim_vec4_contiguous(out)` holds, `permute_vec4_buffer` / `view_vec4_buffer` launch one thread per texel (`default_pick_vec4_gwg`) and write a vec4. All other outputs keep the scalar shaders. - `is_packed_dim_vec4_contiguous` (`TensorUtils`) returns true for a buffer tensor that meets all of these: - elements of at most 4 bytes; - packed block size 1; - packed dim aligned to 4; - packed dim innermost in the dim order. - Both shaders select the vec4 path with a `VEC4_OUTPUT` macro and `#ifdef` inside `main()`. Only the output binding declaration differs per variant. - `permute_vec4_buffer`: - It computes the tensor index once per texel. - Lane `i` reads `inp_bufi + i * stride`, where `stride` is the input stride of the permuted packed dim. - Padding lanes are written as zero. - `view_vec4_buffer`: - If input and output have the same physical element order, it copies four consecutive elements. - Otherwise, lane 0 maps through the contiguous index. Later lanes either advance the input index by one contiguous step, when every dim below the packed dim has size 1, or recompute it. - `cases.py` changes: - New pixel-shuffle-shaped 6D cases for buffer `permute` (float and int32, all layouts, zero-padding checks) and for `view_copy`. - The `view_copy` highdim suite now also runs int32 and checks zero padding. Authored with Claude Code. Differential Revision: [D133233612](https://our.internmc.facebook.com/intern/diff/D133233612/) **NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D133233612/)! ghstack-source-id: 478906400 --- .../graph/ops/glsl/permute_buffer.glsl | 37 ++++++++- .../graph/ops/glsl/permute_buffer.yaml | 3 + .../runtime/graph/ops/glsl/view_buffer.glsl | 81 ++++++++++++++++++- .../runtime/graph/ops/glsl/view_buffer.yaml | 11 +++ .../vulkan/runtime/graph/ops/impl/Permute.cpp | 8 +- .../vulkan/runtime/graph/ops/impl/View.cpp | 11 ++- .../graph/ops/impl/utils/TensorUtils.cpp | 21 +++++ .../graph/ops/impl/utils/TensorUtils.h | 7 ++ backends/vulkan/test/op_tests/cases.py | 15 +++- 9 files changed, 185 insertions(+), 9 deletions(-) diff --git a/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.glsl b/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.glsl index a490cee736e..c6d5207b328 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.glsl @@ -12,16 +12,22 @@ ${define_required_extensions("buffer", DTYPE)} #define PRECISION ${PRECISION} -#define VEC4_T ${texel_type(DTYPE)} +#define VEC4_T ${texel_load_type(DTYPE, "buffer")} #define T ${buffer_scalar_type(DTYPE)} ${define_active_storage_type("buffer")} +$if VEC4: + #define VEC4_OUTPUT + layout(std430) buffer; #include "indexing.glslh" -${layout_declare_tensor(B, "w", "t_outp", DTYPE, "buffer")} +$if VEC4: + ${layout_declare_tensor(B, "w", "t_outp", DTYPE, "buffer", is_scalar_array=False)} +$else: + ${layout_declare_tensor(B, "w", "t_outp", DTYPE, "buffer")} ${layout_declare_tensor(B, "r", "t_inp", DTYPE, "buffer")} ${layout_declare_ubo(B, "BufferMetadata", "outp")} @@ -60,6 +66,32 @@ TensorIndex unpermute(const TensorIndex outp_tidx) { } void main() { +#ifdef VEC4_OUTPUT + // Each thread writes one texel of four consecutive elements along the + // output's packed dim, which must have stride 1. + const int outp_packed_dim = get_packed_dim(outp_layout); + const uint texel_idx = linear_idx_from_gid(); + if (texel_idx >= div_4(numel(outp))) { + return; + } + + const TensorIndex outp_tidx = + linear_idx_to_tensor_idx(outp, mul_4(texel_idx), outp_layout); + VEC4_T texel = VEC4_T(0); + if (!out_of_bounds(outp_tidx, outp)) { + const TensorIndex inp_tidx = unpermute(outp_tidx); + const uint inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx); + const uint inp_lane_stride = + stride_at(inp, extract_4b(permute_order, outp_packed_dim)); + const int valid_lanes = num_valid_lanes(outp_tidx, outp, outp_packed_dim); + [[unroll]] for (int i = 0; i < 4; ++i) { + if (i < valid_lanes) { + texel[i] = t_inp[inp_bufi + i * inp_lane_stride]; + } + } + } + t_outp[texel_idx] = texel; +#else const uint outp_bufi = linear_idx_from_gid(); if (outp_bufi >= numel(outp)) { return; @@ -77,4 +109,5 @@ void main() { // Copy data from input to output t_outp[outp_bufi] = t_inp[inp_bufi]; +#endif } diff --git a/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.yaml index 6fe5a67c286..837838e1fc6 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.yaml +++ b/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.yaml @@ -1,6 +1,7 @@ permute_buffer: parameter_names_with_default_values: DTYPE: float + VEC4: false generate_variant_forall: DTYPE: - VALUE: half @@ -9,3 +10,5 @@ permute_buffer: - VALUE: uint8 shader_variants: - NAME: permute_buffer + - NAME: permute_vec4_buffer + VEC4: true diff --git a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.glsl b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.glsl index 9687a599678..444d4ddd267 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.glsl @@ -3,14 +3,21 @@ #define PRECISION ${PRECISION} #define T ${buffer_scalar_type(DTYPE)} +#define VEC4_T ${texel_load_type(DTYPE, STORAGE)} ${define_required_extensions(STORAGE, DTYPE)} +$if VEC4: + #define VEC4_OUTPUT + layout(std430) buffer; #include "indexing.glslh" -${layout_declare_tensor(B, "w", "t_outp", DTYPE, STORAGE)} +$if VEC4: + ${layout_declare_tensor(B, "w", "t_outp", DTYPE, STORAGE, is_scalar_array=False)} +$else: + ${layout_declare_tensor(B, "w", "t_outp", DTYPE, STORAGE)} ${layout_declare_tensor(B, "r", "t_inp", DTYPE, STORAGE)} ${layout_declare_ubo(B, "BufferMetadata", "outp")} @@ -23,11 +30,82 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "outp_layout", "CONTIG_LAYOUT_INT")} ${layout_declare_spec_const(C, "int", "inp_layout", "CONTIG_LAYOUT_INT")} +/* + * Advances tidx to the next element in contiguous order and moves bufi to that + * element's buffer index. + */ +void advance_contiguous( + inout TensorIndex tidx, + inout uint bufi, + const BufferMetadata meta) { + [[unroll]] for (int d = 0; d < DIMLIMIT; ++d) { + tidx.data[div_4(d)][mod_4(d)]++; + bufi += stride_at(meta, d); + if (idx_at(tidx, d) < size_at(meta, d)) { + return; + } + bufi -= size_at(meta, d) * stride_at(meta, d); + tidx.data[div_4(d)][mod_4(d)] = 0; + } +} + /* * The insight behind the view operation is that the contiguous index of each * tensor element in the input and output tensors are the same. */ void main() { +#ifdef VEC4_OUTPUT + // Each thread writes one texel of four consecutive elements along the + // output's packed dim, which must have stride 1. + const int outp_packed_dim = get_packed_dim(outp_layout); + const uint texel_idx = linear_idx_from_gid(); + if (texel_idx >= div_4(numel(outp))) { + return; + } + + const uint outp_bufi = mul_4(texel_idx); + if (has_same_physical_element_order(outp, inp, outp_layout, inp_layout)) { + t_outp[texel_idx] = VEC4_T( + t_inp[outp_bufi], + t_inp[outp_bufi + 1], + t_inp[outp_bufi + 2], + t_inp[outp_bufi + 3]); + return; + } + + VEC4_T texel = VEC4_T(0); + const TensorIndex outp_tidx = linear_idx_to_tensor_idx(outp, outp_bufi); + if (!out_of_bounds(outp_tidx, outp)) { + const uint contig_idx = tensor_idx_to_contiguous_idx(outp, outp_tidx); + uint contig_lane_stride = 1; + [[unroll]] for (int d = 0; d < DIMLIMIT; ++d) { + if (d < outp_packed_dim) { + contig_lane_stride *= size_at(outp, d); + } + } + const int valid_lanes = num_valid_lanes(outp_tidx, outp, outp_packed_dim); + + TensorIndex inp_tidx = contiguous_idx_to_tensor_idx(inp, contig_idx); + uint inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx); + texel[0] = t_inp[inp_bufi]; + [[unroll]] for (int i = 1; i < 4; ++i) { + if (i < valid_lanes) { + // When every dim below the packed dim has size 1, consecutive lanes + // are consecutive in contiguous order, so the input index can be + // advanced instead of recomputed. + if (contig_lane_stride == 1) { + advance_contiguous(inp_tidx, inp_bufi, inp); + } else { + inp_tidx = contiguous_idx_to_tensor_idx( + inp, contig_idx + i * contig_lane_stride); + inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx); + } + texel[i] = t_inp[inp_bufi]; + } + } + } + t_outp[texel_idx] = texel; +#else const uint outp_bufi = linear_idx_from_gid(); if (outp_bufi >= numel(outp)) { return; @@ -49,4 +127,5 @@ void main() { const uint inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx); t_outp[outp_bufi] = t_inp[inp_bufi]; +#endif } diff --git a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml index ec92bf483c8..40ce4602a60 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml +++ b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml @@ -8,6 +8,7 @@ view_buffer: parameter_names_with_default_values: DTYPE: float STORAGE: buffer + VEC4: false generate_variant_forall: DTYPE: - VALUE: half @@ -18,3 +19,13 @@ view_buffer: - VALUE: int32 shader_variants: - NAME: view_buffer + # Double buffers hold 8-byte elements, which do not form vec4 texels. + - NAME: view_vec4_buffer + VEC4: true + generate_variant_forall: + DTYPE: + - VALUE: half + - VALUE: float + - VALUE: int8 + - VALUE: uint8 + - VALUE: int32 diff --git a/backends/vulkan/runtime/graph/ops/impl/Permute.cpp b/backends/vulkan/runtime/graph/ops/impl/Permute.cpp index 694f12879e5..dfca2c42355 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Permute.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/Permute.cpp @@ -125,7 +125,11 @@ void add_permute_node( const ValueRef out) { check_args(graph, in, permute_dims, out); - std::string kernel_name = "permute"; + // Buffer outputs whose packed dim has stride 1 are written one vec4 texel + // per thread. + const bool use_vec4 = is_packed_dim_vec4_contiguous(graph, out); + + std::string kernel_name = use_vec4 ? "permute_vec4" : "permute"; kernel_name.reserve(kShaderNameReserve); add_storage_type_suffix(kernel_name, graph.storage_type_of(out)); add_dtype_suffix(kernel_name, graph.dtype_of(out)); @@ -174,7 +178,7 @@ void add_permute_node( graph.execute_nodes().emplace_back(new DynamicDispatchNode( graph, VK_KERNEL_FROM_STR(kernel_name), - default_pick_gwg, + use_vec4 ? default_pick_vec4_gwg : default_pick_gwg, pick_balanced_lwg, {{out, vkapi::kWrite}, {in, vkapi::kRead}}, param_ubos, diff --git a/backends/vulkan/runtime/graph/ops/impl/View.cpp b/backends/vulkan/runtime/graph/ops/impl/View.cpp index 72cf47105e9..06b51ebee07 100644 --- a/backends/vulkan/runtime/graph/ops/impl/View.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/View.cpp @@ -76,8 +76,13 @@ void add_view_copy_node( ValueRef out, const std::vector& resize_args, const ExecuteNode::ResizeFunction& resize_fn) { - std::string kernel_name = - graph.is_buffer_storage(out) ? "view_buffer" : "view_texture"; + // Buffer outputs whose packed dim has stride 1 are written one vec4 texel + // per thread. + const bool use_vec4 = is_packed_dim_vec4_contiguous(graph, out); + + std::string kernel_name = use_vec4 ? "view_vec4_buffer" + : graph.is_buffer_storage(out) ? "view_buffer" + : "view_texture"; kernel_name.reserve(kShaderNameReserve); add_dtype_suffix(kernel_name, graph.dtype_of(out)); @@ -93,7 +98,7 @@ void add_view_copy_node( graph.execute_nodes().emplace_back(new DynamicDispatchNode( graph, VK_KERNEL_FROM_STR(kernel_name), - default_pick_gwg, + use_vec4 ? default_pick_vec4_gwg : default_pick_gwg, pick_balanced_lwg, // Inputs and Outputs {{out, vkapi::kWrite}, {in, vkapi::kRead}}, diff --git a/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.cpp b/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.cpp index 536ee07775b..59d6d099f86 100644 --- a/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.cpp @@ -44,6 +44,27 @@ bool check_same_packed_dim( return graph.packed_dim_of(in) == graph.packed_dim_of(out); } +bool is_packed_dim_vec4_contiguous(ComputeGraph& graph, const ValueRef tensor) { + if (!graph.is_buffer_storage(tensor)) { + return false; + } + // Elements wider than 4 bytes do not form vec4 texels. + if (vkapi::element_size(graph.dtype_of(tensor)) > 4) { + return false; + } + const api::PackedDimInfo& packed_dim_info = graph.packed_dim_info_of(tensor); + if (packed_dim_info.packed_dim_block_size != 1 || + packed_dim_info.outer_packed_dim_block_size != 1 || + packed_dim_info.packed_dim_align % 4 != 0) { + return false; + } + const std::vector dim_order = graph.dim_order_of(tensor); + const int64_t ndim = utils::safe_downcast(dim_order.size()); + const int64_t packed_dim = packed_dim_info.packed_dim; + // A packed dim beyond the tensor's rank is placed innermost. + return packed_dim >= ndim || dim_order.back() == ndim - 1 - packed_dim; +} + // // Work group size calculation functions // diff --git a/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h b/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h index ce007f91489..ba45f87c92d 100644 --- a/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h +++ b/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h @@ -30,6 +30,13 @@ bool check_same_packed_dim( const ValueRef in, const ValueRef out); +/* + * Returns true if a buffer tensor's packed dim has stride 1, so that each group + * of four elements along it, starting at a multiple of four, forms one vec4 + * texel in the buffer. + */ +bool is_packed_dim_vec4_contiguous(ComputeGraph& graph, const ValueRef tensor); + // // Work group size calculation functions // diff --git a/backends/vulkan/test/op_tests/cases.py b/backends/vulkan/test/op_tests/cases.py index dc0e31399aa..08625e4c234 100644 --- a/backends/vulkan/test/op_tests/cases.py +++ b/backends/vulkan/test/op_tests/cases.py @@ -1145,7 +1145,17 @@ def get_permute_inputs(): "at::kFloat", ] test_suite.check_zero_padding = True - return test_suite + + # Pixel shuffle decomposed into view -> permute -> view, which leaves a + # packed dim of size 2. + highdim_test_suite = VkTestSuite([((1, 3, 2, 2, 5, 6), (0, 1, 4, 2, 5, 3))]) + highdim_test_suite.layouts = test_suite.layouts + highdim_test_suite.storage_types = ["utils::kBuffer"] + highdim_test_suite.dtypes = ["at::kFloat", "at::kInt"] + highdim_test_suite.data_gen = "make_seq_tensor" + highdim_test_suite.check_zero_padding = True + highdim_test_suite.test_name_suffix = "highdim" + return [test_suite, highdim_test_suite] @register_test_suite("aten.view_copy.default") @@ -1183,11 +1193,14 @@ def get_view_inputs(): ((1, 1, 3, 3, 3), (9, 3)), ((2, 3, 4, 6, 5, 4), (6, 4, 6, 5, 4)), ((2, 3, 3, 7, 8), (2, 3, 3, 8 * 7)), + ((1, 3, 5, 2, 6, 2), (1, 3, 10, 12)), ] ) highdim_test_suite.storage_types = [ "utils::kBuffer", ] + highdim_test_suite.dtypes = ["at::kFloat", "at::kHalf", "at::kInt"] + highdim_test_suite.check_zero_padding = True highdim_test_suite.test_name_suffix = "highdim" highdim_test_suite.data_gen = "make_seq_tensor"