diff --git a/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.glsl b/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.glsl index a490cee736e..c6d5207b328 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.glsl @@ -12,16 +12,22 @@ ${define_required_extensions("buffer", DTYPE)} #define PRECISION ${PRECISION} -#define VEC4_T ${texel_type(DTYPE)} +#define VEC4_T ${texel_load_type(DTYPE, "buffer")} #define T ${buffer_scalar_type(DTYPE)} ${define_active_storage_type("buffer")} +$if VEC4: + #define VEC4_OUTPUT + layout(std430) buffer; #include "indexing.glslh" -${layout_declare_tensor(B, "w", "t_outp", DTYPE, "buffer")} +$if VEC4: + ${layout_declare_tensor(B, "w", "t_outp", DTYPE, "buffer", is_scalar_array=False)} +$else: + ${layout_declare_tensor(B, "w", "t_outp", DTYPE, "buffer")} ${layout_declare_tensor(B, "r", "t_inp", DTYPE, "buffer")} ${layout_declare_ubo(B, "BufferMetadata", "outp")} @@ -60,6 +66,32 @@ TensorIndex unpermute(const TensorIndex outp_tidx) { } void main() { +#ifdef VEC4_OUTPUT + // Each thread writes one texel of four consecutive elements along the + // output's packed dim, which must have stride 1. + const int outp_packed_dim = get_packed_dim(outp_layout); + const uint texel_idx = linear_idx_from_gid(); + if (texel_idx >= div_4(numel(outp))) { + return; + } + + const TensorIndex outp_tidx = + linear_idx_to_tensor_idx(outp, mul_4(texel_idx), outp_layout); + VEC4_T texel = VEC4_T(0); + if (!out_of_bounds(outp_tidx, outp)) { + const TensorIndex inp_tidx = unpermute(outp_tidx); + const uint inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx); + const uint inp_lane_stride = + stride_at(inp, extract_4b(permute_order, outp_packed_dim)); + const int valid_lanes = num_valid_lanes(outp_tidx, outp, outp_packed_dim); + [[unroll]] for (int i = 0; i < 4; ++i) { + if (i < valid_lanes) { + texel[i] = t_inp[inp_bufi + i * inp_lane_stride]; + } + } + } + t_outp[texel_idx] = texel; +#else const uint outp_bufi = linear_idx_from_gid(); if (outp_bufi >= numel(outp)) { return; @@ -77,4 +109,5 @@ void main() { // Copy data from input to output t_outp[outp_bufi] = t_inp[inp_bufi]; +#endif } diff --git a/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.yaml index 6fe5a67c286..837838e1fc6 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.yaml +++ b/backends/vulkan/runtime/graph/ops/glsl/permute_buffer.yaml @@ -1,6 +1,7 @@ permute_buffer: parameter_names_with_default_values: DTYPE: float + VEC4: false generate_variant_forall: DTYPE: - VALUE: half @@ -9,3 +10,5 @@ permute_buffer: - VALUE: uint8 shader_variants: - NAME: permute_buffer + - NAME: permute_vec4_buffer + VEC4: true diff --git a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.glsl b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.glsl index 9687a599678..444d4ddd267 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.glsl @@ -3,14 +3,21 @@ #define PRECISION ${PRECISION} #define T ${buffer_scalar_type(DTYPE)} +#define VEC4_T ${texel_load_type(DTYPE, STORAGE)} ${define_required_extensions(STORAGE, DTYPE)} +$if VEC4: + #define VEC4_OUTPUT + layout(std430) buffer; #include "indexing.glslh" -${layout_declare_tensor(B, "w", "t_outp", DTYPE, STORAGE)} +$if VEC4: + ${layout_declare_tensor(B, "w", "t_outp", DTYPE, STORAGE, is_scalar_array=False)} +$else: + ${layout_declare_tensor(B, "w", "t_outp", DTYPE, STORAGE)} ${layout_declare_tensor(B, "r", "t_inp", DTYPE, STORAGE)} ${layout_declare_ubo(B, "BufferMetadata", "outp")} @@ -23,11 +30,82 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in; ${layout_declare_spec_const(C, "int", "outp_layout", "CONTIG_LAYOUT_INT")} ${layout_declare_spec_const(C, "int", "inp_layout", "CONTIG_LAYOUT_INT")} +/* + * Advances tidx to the next element in contiguous order and moves bufi to that + * element's buffer index. + */ +void advance_contiguous( + inout TensorIndex tidx, + inout uint bufi, + const BufferMetadata meta) { + [[unroll]] for (int d = 0; d < DIMLIMIT; ++d) { + tidx.data[div_4(d)][mod_4(d)]++; + bufi += stride_at(meta, d); + if (idx_at(tidx, d) < size_at(meta, d)) { + return; + } + bufi -= size_at(meta, d) * stride_at(meta, d); + tidx.data[div_4(d)][mod_4(d)] = 0; + } +} + /* * The insight behind the view operation is that the contiguous index of each * tensor element in the input and output tensors are the same. */ void main() { +#ifdef VEC4_OUTPUT + // Each thread writes one texel of four consecutive elements along the + // output's packed dim, which must have stride 1. + const int outp_packed_dim = get_packed_dim(outp_layout); + const uint texel_idx = linear_idx_from_gid(); + if (texel_idx >= div_4(numel(outp))) { + return; + } + + const uint outp_bufi = mul_4(texel_idx); + if (has_same_physical_element_order(outp, inp, outp_layout, inp_layout)) { + t_outp[texel_idx] = VEC4_T( + t_inp[outp_bufi], + t_inp[outp_bufi + 1], + t_inp[outp_bufi + 2], + t_inp[outp_bufi + 3]); + return; + } + + VEC4_T texel = VEC4_T(0); + const TensorIndex outp_tidx = linear_idx_to_tensor_idx(outp, outp_bufi); + if (!out_of_bounds(outp_tidx, outp)) { + const uint contig_idx = tensor_idx_to_contiguous_idx(outp, outp_tidx); + uint contig_lane_stride = 1; + [[unroll]] for (int d = 0; d < DIMLIMIT; ++d) { + if (d < outp_packed_dim) { + contig_lane_stride *= size_at(outp, d); + } + } + const int valid_lanes = num_valid_lanes(outp_tidx, outp, outp_packed_dim); + + TensorIndex inp_tidx = contiguous_idx_to_tensor_idx(inp, contig_idx); + uint inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx); + texel[0] = t_inp[inp_bufi]; + [[unroll]] for (int i = 1; i < 4; ++i) { + if (i < valid_lanes) { + // When every dim below the packed dim has size 1, consecutive lanes + // are consecutive in contiguous order, so the input index can be + // advanced instead of recomputed. + if (contig_lane_stride == 1) { + advance_contiguous(inp_tidx, inp_bufi, inp); + } else { + inp_tidx = contiguous_idx_to_tensor_idx( + inp, contig_idx + i * contig_lane_stride); + inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx); + } + texel[i] = t_inp[inp_bufi]; + } + } + } + t_outp[texel_idx] = texel; +#else const uint outp_bufi = linear_idx_from_gid(); if (outp_bufi >= numel(outp)) { return; @@ -49,4 +127,5 @@ void main() { const uint inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx); t_outp[outp_bufi] = t_inp[inp_bufi]; +#endif } diff --git a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml index ec92bf483c8..40ce4602a60 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml +++ b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml @@ -8,6 +8,7 @@ view_buffer: parameter_names_with_default_values: DTYPE: float STORAGE: buffer + VEC4: false generate_variant_forall: DTYPE: - VALUE: half @@ -18,3 +19,13 @@ view_buffer: - VALUE: int32 shader_variants: - NAME: view_buffer + # Double buffers hold 8-byte elements, which do not form vec4 texels. + - NAME: view_vec4_buffer + VEC4: true + generate_variant_forall: + DTYPE: + - VALUE: half + - VALUE: float + - VALUE: int8 + - VALUE: uint8 + - VALUE: int32 diff --git a/backends/vulkan/runtime/graph/ops/impl/Permute.cpp b/backends/vulkan/runtime/graph/ops/impl/Permute.cpp index 694f12879e5..dfca2c42355 100644 --- a/backends/vulkan/runtime/graph/ops/impl/Permute.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/Permute.cpp @@ -125,7 +125,11 @@ void add_permute_node( const ValueRef out) { check_args(graph, in, permute_dims, out); - std::string kernel_name = "permute"; + // Buffer outputs whose packed dim has stride 1 are written one vec4 texel + // per thread. + const bool use_vec4 = is_packed_dim_vec4_contiguous(graph, out); + + std::string kernel_name = use_vec4 ? "permute_vec4" : "permute"; kernel_name.reserve(kShaderNameReserve); add_storage_type_suffix(kernel_name, graph.storage_type_of(out)); add_dtype_suffix(kernel_name, graph.dtype_of(out)); @@ -174,7 +178,7 @@ void add_permute_node( graph.execute_nodes().emplace_back(new DynamicDispatchNode( graph, VK_KERNEL_FROM_STR(kernel_name), - default_pick_gwg, + use_vec4 ? default_pick_vec4_gwg : default_pick_gwg, pick_balanced_lwg, {{out, vkapi::kWrite}, {in, vkapi::kRead}}, param_ubos, diff --git a/backends/vulkan/runtime/graph/ops/impl/View.cpp b/backends/vulkan/runtime/graph/ops/impl/View.cpp index 72cf47105e9..06b51ebee07 100644 --- a/backends/vulkan/runtime/graph/ops/impl/View.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/View.cpp @@ -76,8 +76,13 @@ void add_view_copy_node( ValueRef out, const std::vector& resize_args, const ExecuteNode::ResizeFunction& resize_fn) { - std::string kernel_name = - graph.is_buffer_storage(out) ? "view_buffer" : "view_texture"; + // Buffer outputs whose packed dim has stride 1 are written one vec4 texel + // per thread. + const bool use_vec4 = is_packed_dim_vec4_contiguous(graph, out); + + std::string kernel_name = use_vec4 ? "view_vec4_buffer" + : graph.is_buffer_storage(out) ? "view_buffer" + : "view_texture"; kernel_name.reserve(kShaderNameReserve); add_dtype_suffix(kernel_name, graph.dtype_of(out)); @@ -93,7 +98,7 @@ void add_view_copy_node( graph.execute_nodes().emplace_back(new DynamicDispatchNode( graph, VK_KERNEL_FROM_STR(kernel_name), - default_pick_gwg, + use_vec4 ? default_pick_vec4_gwg : default_pick_gwg, pick_balanced_lwg, // Inputs and Outputs {{out, vkapi::kWrite}, {in, vkapi::kRead}}, diff --git a/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.cpp b/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.cpp index 536ee07775b..59d6d099f86 100644 --- a/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.cpp @@ -44,6 +44,27 @@ bool check_same_packed_dim( return graph.packed_dim_of(in) == graph.packed_dim_of(out); } +bool is_packed_dim_vec4_contiguous(ComputeGraph& graph, const ValueRef tensor) { + if (!graph.is_buffer_storage(tensor)) { + return false; + } + // Elements wider than 4 bytes do not form vec4 texels. + if (vkapi::element_size(graph.dtype_of(tensor)) > 4) { + return false; + } + const api::PackedDimInfo& packed_dim_info = graph.packed_dim_info_of(tensor); + if (packed_dim_info.packed_dim_block_size != 1 || + packed_dim_info.outer_packed_dim_block_size != 1 || + packed_dim_info.packed_dim_align % 4 != 0) { + return false; + } + const std::vector dim_order = graph.dim_order_of(tensor); + const int64_t ndim = utils::safe_downcast(dim_order.size()); + const int64_t packed_dim = packed_dim_info.packed_dim; + // A packed dim beyond the tensor's rank is placed innermost. + return packed_dim >= ndim || dim_order.back() == ndim - 1 - packed_dim; +} + // // Work group size calculation functions // diff --git a/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h b/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h index ce007f91489..ba45f87c92d 100644 --- a/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h +++ b/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h @@ -30,6 +30,13 @@ bool check_same_packed_dim( const ValueRef in, const ValueRef out); +/* + * Returns true if a buffer tensor's packed dim has stride 1, so that each group + * of four elements along it, starting at a multiple of four, forms one vec4 + * texel in the buffer. + */ +bool is_packed_dim_vec4_contiguous(ComputeGraph& graph, const ValueRef tensor); + // // Work group size calculation functions // diff --git a/backends/vulkan/test/op_tests/cases.py b/backends/vulkan/test/op_tests/cases.py index dc0e31399aa..08625e4c234 100644 --- a/backends/vulkan/test/op_tests/cases.py +++ b/backends/vulkan/test/op_tests/cases.py @@ -1145,7 +1145,17 @@ def get_permute_inputs(): "at::kFloat", ] test_suite.check_zero_padding = True - return test_suite + + # Pixel shuffle decomposed into view -> permute -> view, which leaves a + # packed dim of size 2. + highdim_test_suite = VkTestSuite([((1, 3, 2, 2, 5, 6), (0, 1, 4, 2, 5, 3))]) + highdim_test_suite.layouts = test_suite.layouts + highdim_test_suite.storage_types = ["utils::kBuffer"] + highdim_test_suite.dtypes = ["at::kFloat", "at::kInt"] + highdim_test_suite.data_gen = "make_seq_tensor" + highdim_test_suite.check_zero_padding = True + highdim_test_suite.test_name_suffix = "highdim" + return [test_suite, highdim_test_suite] @register_test_suite("aten.view_copy.default") @@ -1183,11 +1193,14 @@ def get_view_inputs(): ((1, 1, 3, 3, 3), (9, 3)), ((2, 3, 4, 6, 5, 4), (6, 4, 6, 5, 4)), ((2, 3, 3, 7, 8), (2, 3, 3, 8 * 7)), + ((1, 3, 5, 2, 6, 2), (1, 3, 10, 12)), ] ) highdim_test_suite.storage_types = [ "utils::kBuffer", ] + highdim_test_suite.dtypes = ["at::kFloat", "at::kHalf", "at::kInt"] + highdim_test_suite.check_zero_padding = True highdim_test_suite.test_name_suffix = "highdim" highdim_test_suite.data_gen = "make_seq_tensor"