Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 35 additions & 2 deletions backends/vulkan/runtime/graph/ops/glsl/permute_buffer.glsl
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,22 @@ ${define_required_extensions("buffer", DTYPE)}

#define PRECISION ${PRECISION}

#define VEC4_T ${texel_type(DTYPE)}
#define VEC4_T ${texel_load_type(DTYPE, "buffer")}
#define T ${buffer_scalar_type(DTYPE)}

${define_active_storage_type("buffer")}

$if VEC4:
#define VEC4_OUTPUT

layout(std430) buffer;

#include "indexing.glslh"

${layout_declare_tensor(B, "w", "t_outp", DTYPE, "buffer")}
$if VEC4:
${layout_declare_tensor(B, "w", "t_outp", DTYPE, "buffer", is_scalar_array=False)}
$else:
${layout_declare_tensor(B, "w", "t_outp", DTYPE, "buffer")}
${layout_declare_tensor(B, "r", "t_inp", DTYPE, "buffer")}

${layout_declare_ubo(B, "BufferMetadata", "outp")}
Expand Down Expand Up @@ -60,6 +66,32 @@ TensorIndex unpermute(const TensorIndex outp_tidx) {
}

void main() {
#ifdef VEC4_OUTPUT
// Each thread writes one texel of four consecutive elements along the
// output's packed dim, which must have stride 1.
const int outp_packed_dim = get_packed_dim(outp_layout);
const uint texel_idx = linear_idx_from_gid();
if (texel_idx >= div_4(numel(outp))) {
return;
}

const TensorIndex outp_tidx =
linear_idx_to_tensor_idx(outp, mul_4(texel_idx), outp_layout);
VEC4_T texel = VEC4_T(0);
if (!out_of_bounds(outp_tidx, outp)) {
const TensorIndex inp_tidx = unpermute(outp_tidx);
const uint inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx);
const uint inp_lane_stride =
stride_at(inp, extract_4b(permute_order, outp_packed_dim));
const int valid_lanes = num_valid_lanes(outp_tidx, outp, outp_packed_dim);
[[unroll]] for (int i = 0; i < 4; ++i) {
if (i < valid_lanes) {
texel[i] = t_inp[inp_bufi + i * inp_lane_stride];
}
}
}
t_outp[texel_idx] = texel;
#else
const uint outp_bufi = linear_idx_from_gid();
if (outp_bufi >= numel(outp)) {
return;
Expand All @@ -77,4 +109,5 @@ void main() {

// Copy data from input to output
t_outp[outp_bufi] = t_inp[inp_bufi];
#endif
}
3 changes: 3 additions & 0 deletions backends/vulkan/runtime/graph/ops/glsl/permute_buffer.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
permute_buffer:
parameter_names_with_default_values:
DTYPE: float
VEC4: false
generate_variant_forall:
DTYPE:
- VALUE: half
Expand All @@ -9,3 +10,5 @@ permute_buffer:
- VALUE: uint8
shader_variants:
- NAME: permute_buffer
- NAME: permute_vec4_buffer
VEC4: true
81 changes: 80 additions & 1 deletion backends/vulkan/runtime/graph/ops/glsl/view_buffer.glsl
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,21 @@
#define PRECISION ${PRECISION}

#define T ${buffer_scalar_type(DTYPE)}
#define VEC4_T ${texel_load_type(DTYPE, STORAGE)}

${define_required_extensions(STORAGE, DTYPE)}

$if VEC4:
#define VEC4_OUTPUT

layout(std430) buffer;

#include "indexing.glslh"

${layout_declare_tensor(B, "w", "t_outp", DTYPE, STORAGE)}
$if VEC4:
${layout_declare_tensor(B, "w", "t_outp", DTYPE, STORAGE, is_scalar_array=False)}
$else:
${layout_declare_tensor(B, "w", "t_outp", DTYPE, STORAGE)}
${layout_declare_tensor(B, "r", "t_inp", DTYPE, STORAGE)}

${layout_declare_ubo(B, "BufferMetadata", "outp")}
Expand All @@ -23,11 +30,82 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
${layout_declare_spec_const(C, "int", "outp_layout", "CONTIG_LAYOUT_INT")}
${layout_declare_spec_const(C, "int", "inp_layout", "CONTIG_LAYOUT_INT")}

/*
* Advances tidx to the next element in contiguous order and moves bufi to that
* element's buffer index.
*/
void advance_contiguous(
inout TensorIndex tidx,
inout uint bufi,
const BufferMetadata meta) {
[[unroll]] for (int d = 0; d < DIMLIMIT; ++d) {
tidx.data[div_4(d)][mod_4(d)]++;
bufi += stride_at(meta, d);
if (idx_at(tidx, d) < size_at(meta, d)) {
return;
}
bufi -= size_at(meta, d) * stride_at(meta, d);
tidx.data[div_4(d)][mod_4(d)] = 0;
}
}

/*
* The insight behind the view operation is that the contiguous index of each
* tensor element in the input and output tensors are the same.
*/
void main() {
#ifdef VEC4_OUTPUT
// Each thread writes one texel of four consecutive elements along the
// output's packed dim, which must have stride 1.
const int outp_packed_dim = get_packed_dim(outp_layout);
const uint texel_idx = linear_idx_from_gid();
if (texel_idx >= div_4(numel(outp))) {
return;
}

const uint outp_bufi = mul_4(texel_idx);
if (has_same_physical_element_order(outp, inp, outp_layout, inp_layout)) {
t_outp[texel_idx] = VEC4_T(
t_inp[outp_bufi],
t_inp[outp_bufi + 1],
t_inp[outp_bufi + 2],
t_inp[outp_bufi + 3]);
return;
}

VEC4_T texel = VEC4_T(0);
const TensorIndex outp_tidx = linear_idx_to_tensor_idx(outp, outp_bufi);
if (!out_of_bounds(outp_tidx, outp)) {
const uint contig_idx = tensor_idx_to_contiguous_idx(outp, outp_tidx);
uint contig_lane_stride = 1;
[[unroll]] for (int d = 0; d < DIMLIMIT; ++d) {
if (d < outp_packed_dim) {
contig_lane_stride *= size_at(outp, d);
}
}
const int valid_lanes = num_valid_lanes(outp_tidx, outp, outp_packed_dim);

TensorIndex inp_tidx = contiguous_idx_to_tensor_idx(inp, contig_idx);
uint inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx);
texel[0] = t_inp[inp_bufi];
[[unroll]] for (int i = 1; i < 4; ++i) {
if (i < valid_lanes) {
// When every dim below the packed dim has size 1, consecutive lanes
// are consecutive in contiguous order, so the input index can be
// advanced instead of recomputed.
if (contig_lane_stride == 1) {
advance_contiguous(inp_tidx, inp_bufi, inp);
} else {
inp_tidx = contiguous_idx_to_tensor_idx(
inp, contig_idx + i * contig_lane_stride);
inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx);
}
texel[i] = t_inp[inp_bufi];
}
}
}
t_outp[texel_idx] = texel;
#else
const uint outp_bufi = linear_idx_from_gid();
if (outp_bufi >= numel(outp)) {
return;
Expand All @@ -49,4 +127,5 @@ void main() {
const uint inp_bufi = tensor_idx_to_linear_idx(inp, inp_tidx);

t_outp[outp_bufi] = t_inp[inp_bufi];
#endif
}
11 changes: 11 additions & 0 deletions backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ view_buffer:
parameter_names_with_default_values:
DTYPE: float
STORAGE: buffer
VEC4: false
generate_variant_forall:
DTYPE:
- VALUE: half
Expand All @@ -18,3 +19,13 @@ view_buffer:
- VALUE: int32
shader_variants:
- NAME: view_buffer
# Double buffers hold 8-byte elements, which do not form vec4 texels.
- NAME: view_vec4_buffer
VEC4: true
generate_variant_forall:
DTYPE:
- VALUE: half
- VALUE: float
- VALUE: int8
- VALUE: uint8
- VALUE: int32
8 changes: 6 additions & 2 deletions backends/vulkan/runtime/graph/ops/impl/Permute.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,11 @@ void add_permute_node(
const ValueRef out) {
check_args(graph, in, permute_dims, out);

std::string kernel_name = "permute";
// Buffer outputs whose packed dim has stride 1 are written one vec4 texel
// per thread.
const bool use_vec4 = is_packed_dim_vec4_contiguous(graph, out);

std::string kernel_name = use_vec4 ? "permute_vec4" : "permute";
kernel_name.reserve(kShaderNameReserve);
add_storage_type_suffix(kernel_name, graph.storage_type_of(out));
add_dtype_suffix(kernel_name, graph.dtype_of(out));
Expand Down Expand Up @@ -174,7 +178,7 @@ void add_permute_node(
graph.execute_nodes().emplace_back(new DynamicDispatchNode(
graph,
VK_KERNEL_FROM_STR(kernel_name),
default_pick_gwg,
use_vec4 ? default_pick_vec4_gwg : default_pick_gwg,
pick_balanced_lwg,
{{out, vkapi::kWrite}, {in, vkapi::kRead}},
param_ubos,
Expand Down
11 changes: 8 additions & 3 deletions backends/vulkan/runtime/graph/ops/impl/View.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,13 @@ void add_view_copy_node(
ValueRef out,
const std::vector<ValueRef>& resize_args,
const ExecuteNode::ResizeFunction& resize_fn) {
std::string kernel_name =
graph.is_buffer_storage(out) ? "view_buffer" : "view_texture";
// Buffer outputs whose packed dim has stride 1 are written one vec4 texel
// per thread.
const bool use_vec4 = is_packed_dim_vec4_contiguous(graph, out);

std::string kernel_name = use_vec4 ? "view_vec4_buffer"
: graph.is_buffer_storage(out) ? "view_buffer"
: "view_texture";
kernel_name.reserve(kShaderNameReserve);
add_dtype_suffix(kernel_name, graph.dtype_of(out));

Expand All @@ -93,7 +98,7 @@ void add_view_copy_node(
graph.execute_nodes().emplace_back(new DynamicDispatchNode(
graph,
VK_KERNEL_FROM_STR(kernel_name),
default_pick_gwg,
use_vec4 ? default_pick_vec4_gwg : default_pick_gwg,
pick_balanced_lwg,
// Inputs and Outputs
{{out, vkapi::kWrite}, {in, vkapi::kRead}},
Expand Down
21 changes: 21 additions & 0 deletions backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,27 @@ bool check_same_packed_dim(
return graph.packed_dim_of(in) == graph.packed_dim_of(out);
}

bool is_packed_dim_vec4_contiguous(ComputeGraph& graph, const ValueRef tensor) {
if (!graph.is_buffer_storage(tensor)) {
return false;
}
// Elements wider than 4 bytes do not form vec4 texels.
if (vkapi::element_size(graph.dtype_of(tensor)) > 4) {
return false;
}
const api::PackedDimInfo& packed_dim_info = graph.packed_dim_info_of(tensor);
if (packed_dim_info.packed_dim_block_size != 1 ||
packed_dim_info.outer_packed_dim_block_size != 1 ||
packed_dim_info.packed_dim_align % 4 != 0) {
return false;
}
const std::vector<int64_t> dim_order = graph.dim_order_of(tensor);
const int64_t ndim = utils::safe_downcast<int64_t>(dim_order.size());
const int64_t packed_dim = packed_dim_info.packed_dim;
// A packed dim beyond the tensor's rank is placed innermost.
return packed_dim >= ndim || dim_order.back() == ndim - 1 - packed_dim;
}

//
// Work group size calculation functions
//
Expand Down
7 changes: 7 additions & 0 deletions backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ bool check_same_packed_dim(
const ValueRef in,
const ValueRef out);

/*
* Returns true if a buffer tensor's packed dim has stride 1, so that each group
* of four elements along it, starting at a multiple of four, forms one vec4
* texel in the buffer.
*/
bool is_packed_dim_vec4_contiguous(ComputeGraph& graph, const ValueRef tensor);

//
// Work group size calculation functions
//
Expand Down
15 changes: 14 additions & 1 deletion backends/vulkan/test/op_tests/cases.py
Original file line number Diff line number Diff line change
Expand Up @@ -1145,7 +1145,17 @@ def get_permute_inputs():
"at::kFloat",
]
test_suite.check_zero_padding = True
return test_suite

# Pixel shuffle decomposed into view -> permute -> view, which leaves a
# packed dim of size 2.
highdim_test_suite = VkTestSuite([((1, 3, 2, 2, 5, 6), (0, 1, 4, 2, 5, 3))])
highdim_test_suite.layouts = test_suite.layouts
highdim_test_suite.storage_types = ["utils::kBuffer"]
highdim_test_suite.dtypes = ["at::kFloat", "at::kInt"]
highdim_test_suite.data_gen = "make_seq_tensor"
highdim_test_suite.check_zero_padding = True
highdim_test_suite.test_name_suffix = "highdim"
return [test_suite, highdim_test_suite]


@register_test_suite("aten.view_copy.default")
Expand Down Expand Up @@ -1183,11 +1193,14 @@ def get_view_inputs():
((1, 1, 3, 3, 3), (9, 3)),
((2, 3, 4, 6, 5, 4), (6, 4, 6, 5, 4)),
((2, 3, 3, 7, 8), (2, 3, 3, 8 * 7)),
((1, 3, 5, 2, 6, 2), (1, 3, 10, 12)),
]
)
highdim_test_suite.storage_types = [
"utils::kBuffer",
]
highdim_test_suite.dtypes = ["at::kFloat", "at::kHalf", "at::kInt"]
highdim_test_suite.check_zero_padding = True
highdim_test_suite.test_name_suffix = "highdim"
highdim_test_suite.data_gen = "make_seq_tensor"

Expand Down
Loading