Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions backends/vulkan/custom_ops_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -1110,6 +1110,36 @@ def q8ta_pixel_shuffle_impl(
lib.impl(name, q8ta_pixel_shuffle_impl, "CompositeExplicitAutograd")
q8ta_pixel_shuffle_op = getattr(getattr(torch.ops, namespace), name)


def q8ta_pixel_shuffle_with_activation_impl(
input: torch.Tensor,
input_scale: float,
input_zero_point: int,
output_inv_scale: float,
output_zero_point: int,
upscale_factor: int,
activation: str,
):
output_scale = 1.0 / output_inv_scale
dequant = torch.ops.quantized_decomposed.dequantize_per_tensor(
input, input_scale, input_zero_point, -128, 127, input.dtype
)
out = torch.nn.functional.pixel_shuffle(dequant, upscale_factor)
if activation == "relu":
out = torch.nn.functional.relu(out)
else:
assert activation == "none", f"Unsupported activation: {activation}"
return torch.ops.quantized_decomposed.quantize_per_tensor(
out, output_scale, output_zero_point, -128, 127, torch.int8
)


name = "q8ta_pixel_shuffle.with_activation"
lib.define(
f"{name}(Tensor input, float input_scale, int input_zero_point, float output_inv_scale, int output_zero_point, int upscale_factor, str activation) -> Tensor"
)
lib.impl(name, q8ta_pixel_shuffle_with_activation_impl, "CompositeExplicitAutograd")

########################
## embedding_q4gsw ##
########################
Expand Down
7 changes: 6 additions & 1 deletion backends/vulkan/op_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -670,7 +670,12 @@ def register_q8ta_relu():
# =============================================================================


@update_features(exir_ops.edge.et_vk.q8ta_pixel_shuffle.default)
@update_features(
[
exir_ops.edge.et_vk.q8ta_pixel_shuffle.default,
exir_ops.edge.et_vk.q8ta_pixel_shuffle.with_activation,
]
)
def register_q8ta_pixel_shuffle():
# The fused kernel is restricted to the channels-packed family
# (PACKED_INT8_4W4C, PACKED_INT8_4C1W, PACKED_INT8_CONV2D), all of which
Expand Down
46 changes: 30 additions & 16 deletions backends/vulkan/patterns/quantized_pixel_shuffle.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

from typing import List, Optional, Set
from typing import Dict, List, Optional, Set

import executorch.backends.vulkan.utils as utils

Expand All @@ -30,6 +30,12 @@
exir_ops.edge.dim_order_ops._clone_dim_order.default,
}

# Activations that can follow the pixel_shuffle, mapped to the activation name
# passed to q8ta_pixel_shuffle.with_activation.
_ACTIVATIONS: Dict[object, str] = {
exir_ops.edge.aten.relu.default: "relu",
}


def _is_noop_passthrough(node: torch.fx.Node) -> bool:
return node.op == "call_function" and node.target in _NOOP_PASSTHROUGH_TARGETS
Expand Down Expand Up @@ -61,6 +67,8 @@ class QuantizedPixelShuffleMatch(PatternMatch):
[optional] clone / _clone_dim_order
aten.pixel_shuffle.default (upscale_factor = r)
[optional] clone / _clone_dim_order
[optional] activation (aten.relu.default)
[optional] clone / _clone_dim_order
q8ta_quantize_per_tensor (fp32 -> int8)

The anchor is the dequantize node since it is a unique entry point.
Expand Down Expand Up @@ -88,8 +96,14 @@ def __init__(self, dequantize_node: torch.fx.Node) -> None:
if pixel_shuffle_node.target != exir_ops.edge.aten.pixel_shuffle.default:
return

# Walk forward to the quantize node (skipping any clones).
# Walk forward to the quantize node (skipping any clones), through an
# optional activation.
quantize_node = _skip_passthrough_user(pixel_shuffle_node, self.all_nodes)
self.activation_node: Optional[torch.fx.Node] = None
if quantize_node is not None and quantize_node.target in _ACTIVATIONS:
self.activation_node = quantize_node
self.all_nodes.append(self.activation_node)
quantize_node = _skip_passthrough_user(self.activation_node, self.all_nodes)
if quantize_node is None or not utils.is_quant_node(quantize_node):
return

Expand Down Expand Up @@ -139,8 +153,6 @@ def make_quantized_pixel_shuffle_custom_op(
graph_module: torch.fx.GraphModule,
match: QuantizedPixelShuffleMatch,
) -> None:
op_target = exir_ops.edge.et_vk.q8ta_pixel_shuffle.default

# The fused op takes the *inverse* of the output scale to match the
# runtime kernel's expectation.
output_scale = match.output_scales_node
Expand All @@ -162,19 +174,21 @@ def make_quantized_pixel_shuffle_custom_op(
"quantized_pixel_shuffle pattern only supports scalar output scales"
)

args = (
match.input_int8_node,
match.input_scales_node,
match.input_zeros_node,
inv_output_scale,
match.output_zeros_node,
match.upscale_factor,
)
op_target = exir_ops.edge.et_vk.q8ta_pixel_shuffle.default
if match.activation_node is not None:
op_target = exir_ops.edge.et_vk.q8ta_pixel_shuffle.with_activation
args = (*args, _ACTIVATIONS[match.activation_node.target])

with graph_module.graph.inserting_before(match.output_node):
new_node = graph_module.graph.create_node(
"call_function",
op_target,
args=(
match.input_int8_node,
match.input_scales_node,
match.input_zeros_node,
inv_output_scale,
match.output_zeros_node,
match.upscale_factor,
),
)
new_node = graph_module.graph.create_node("call_function", op_target, args)

new_node.meta["val"] = match.output_node.meta["val"]
match.quantize_output_node.replace_all_uses_with(new_node)
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,16 @@ layout(push_constant) uniform restrict Block {
float inv_scale_out;
int zp_in;
int zp_out;
// Lower clamp bound of requantized values; raised to zp_out to apply relu.
int q_min;
};

layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;

${layout_declare_spec_const(C, "int", "inp_layout", "CONTIG_LAYOUT_INT")}
${layout_declare_spec_const(C, "int", "outp_layout", "CONTIG_LAYOUT_INT")}
// Set by the host when input and output quantization params match, making the
// shader a pure byte shuffle.
// Set by the host when input and output quantization params match and no
// activation is applied, making the shader a pure byte shuffle.
${layout_declare_spec_const(C, "int", "passthrough", "0")}

/*
Expand All @@ -66,7 +68,7 @@ int requantize_word(const int packed) {
const int q_in = (packed << (24 - 8 * i)) >> 24;
const float dq = float(q_in - zp_in) * scale_in;
const int q_out =
clamp(int(round(dq * inv_scale_out)) + zp_out, -128, 127);
clamp(int(round(dq * inv_scale_out)) + zp_out, q_min, 127);
result |= (q_out & 0xFF) << (8 * i);
}
return result;
Expand Down
22 changes: 21 additions & 1 deletion backends/vulkan/runtime/graph/ops/impl/Q8taPixelShuffle.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,16 @@
* LICENSE file in the root directory of this source tree.
*/

#include <executorch/backends/vulkan/runtime/graph/ops/impl/Q8taPixelShuffle.h>

#include <executorch/backends/vulkan/runtime/graph/ops/OperatorRegistry.h>

#include <executorch/backends/vulkan/runtime/graph/ops/impl/Common.h>
#include <executorch/backends/vulkan/runtime/graph/ops/utils/ShaderNameUtils.h>

#include <executorch/backends/vulkan/runtime/graph/Logging.h>

#include <algorithm>
#include <cmath>
#include <cstdint>

Expand Down Expand Up @@ -100,6 +103,7 @@ void add_q8ta_pixel_shuffle_node(
const ValueRef output_inv_scale,
const ValueRef output_zp,
const ValueRef upscale_factor,
const ActivationType activation,
const ValueRef packed_int8_output) {
// dtype must be the int8x4 packed type
VK_CHECK_COND(graph.dtype_of(packed_int8_input) == vkapi::kInt8x4);
Expand Down Expand Up @@ -145,14 +149,21 @@ void add_q8ta_pixel_shuffle_node(
float inv_scale_out = graph.extract_scalar<float>(output_inv_scale);
int32_t zp_in = graph.extract_scalar<int32_t>(input_zp);
int32_t zp_out = graph.extract_scalar<int32_t>(output_zp);
// quantize(relu(x)) == max(quantize(x), zp_out), so relu only raises the
// lower clamp bound of the requantized value.
int32_t q_min = -128;
if (activation == ActivationType::kRelu) {
q_min = std::clamp(zp_out, -128, 127);
}

// Detect the pure-byte-shuffle case: same scale & same zero-point. In that
// case the shader can skip the requantize math entirely.
// Use a small relative tolerance on the scales.
const float scale_diff = std::abs(scale_in - scale_out_actual);
const float scale_thresh = 1e-7f * std::max(std::abs(scale_in), 1e-7f);
int32_t passthrough = 0;
if (scale_diff <= scale_thresh && zp_in == zp_out) {
if (scale_diff <= scale_thresh && zp_in == zp_out &&
activation == ActivationType::kNone) {
passthrough = 1;
}

Expand All @@ -161,6 +172,7 @@ void add_q8ta_pixel_shuffle_node(
PushConstantDataInfo(&inv_scale_out, sizeof(inv_scale_out)),
PushConstantDataInfo(&zp_in, sizeof(zp_in)),
PushConstantDataInfo(&zp_out, sizeof(zp_out)),
PushConstantDataInfo(&q_min, sizeof(q_min)),
};

// UBOs
Expand Down Expand Up @@ -209,6 +221,12 @@ void q8ta_pixel_shuffle(
const ValueRef output_inv_scale = args.at(idx++);
const ValueRef output_zp = args.at(idx++);
const ValueRef upscale_factor = args.at(idx++);
// q8ta_pixel_shuffle.with_activation passes an activation before the output.
ActivationType activation = ActivationType::kNone;
if (args.size() == 8) {
activation =
activation_type_from_string(graph.extract_string(args.at(idx++)));
}
const ValueRef packed_int8_output = args.at(idx);

add_q8ta_pixel_shuffle_node(
Expand All @@ -219,11 +237,13 @@ void q8ta_pixel_shuffle(
output_inv_scale,
output_zp,
upscale_factor,
activation,
packed_int8_output);
}

REGISTER_OPERATORS {
VK_REGISTER_OP(et_vk.q8ta_pixel_shuffle.default, q8ta_pixel_shuffle);
VK_REGISTER_OP(et_vk.q8ta_pixel_shuffle.with_activation, q8ta_pixel_shuffle);
}

} // namespace vkcompute
8 changes: 6 additions & 2 deletions backends/vulkan/runtime/graph/ops/impl/Q8taPixelShuffle.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,17 @@

#include <executorch/backends/vulkan/runtime/graph/ComputeGraph.h>

#include <executorch/backends/vulkan/runtime/graph/ops/impl/convolution/conv2d/q8ta/Q8taConv2dCommon.h>

namespace vkcompute {

//
// Fused PixelShuffle operating on int8x4 packed tensors.
//
// Replaces the decomposed chain:
// q8ta_dequantize -> view -> permute -> view -> q8ta_quantize
// with a single byte-shuffle (and optional requantize when scales differ).
// q8ta_dequantize -> view -> permute -> view -> [relu] -> q8ta_quantize
// with a single byte-shuffle (and optional requantize when scales differ or an
// activation is applied).
//

void add_q8ta_pixel_shuffle_node(
Expand All @@ -28,6 +31,7 @@ void add_q8ta_pixel_shuffle_node(
const ValueRef output_inv_scale,
const ValueRef output_zp,
const ValueRef upscale_factor,
const ActivationType activation,
const ValueRef packed_int8_output);

} // namespace vkcompute
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#include <executorch/backends/vulkan/test/ops/benchmark_v2/framework/ReferenceRegistry.h>
#include <executorch/backends/vulkan/test/ops/benchmark_v2/operators/quantize/q8ta/Q8taReferenceCommon.h>

#include <algorithm>

namespace vkcompute::shader_profiling::reference {
static Tensor q8ta_pixel_shuffle_reference(
const Json& spec,
Expand All @@ -18,6 +20,9 @@ static Tensor q8ta_pixel_shuffle_reference(
inverse_out_scale = q8ta::scale(spec, 3);
const int64_t iz = q8ta::integer(spec, 2), oz = q8ta::integer(spec, 4),
r = q8ta::integer(spec, 5);
// q8ta_pixel_shuffle.with_activation passes an activation before the output.
const bool relu = spec.at("args").size() == 8 &&
spec.at("args").at(6).at("string") == "relu";
const float actual_out_scale = 1.0f / inverse_out_scale;
const bool passthrough = std::abs(is - actual_out_scale) <=
1e-7f * std::max(std::abs(is), 1e-7f) &&
Expand All @@ -34,16 +39,22 @@ static Tensor q8ta_pixel_shuffle_reference(
input.sizes[3] +
x / r;
const int64_t value = int64_t(input.values[source]);
out.values[i] = passthrough
? double(value)
: double(q8ta::requantize(
q8ta::dequantize(value, is, iz), inverse_out_scale, oz));
int64_t q = passthrough
? value
: q8ta::requantize(
q8ta::dequantize(value, is, iz), inverse_out_scale, oz);
if (relu) {
q = std::max(q, oz);
}
out.values[i] = double(q);
});
return out;
}
REGISTER_REFERENCE_FUNCTIONS {
REGISTER_REFERENCE_FN(q8ta_pixel_shuffle, q8ta_pixel_shuffle_reference);
REGISTER_OPERATOR_REFERENCE(
et_vk.q8ta_pixel_shuffle.default, q8ta_pixel_shuffle);
REGISTER_OPERATOR_REFERENCE(
et_vk.q8ta_pixel_shuffle.with_activation, q8ta_pixel_shuffle);
}
} // namespace vkcompute::shader_profiling::reference
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ REGISTER_THROUGHPUT_FUNCTIONS {
et_vk.q8ta_pixel_shuffle.default, throughput::tensor_traffic);
REGISTER_OPERATOR_COMPUTE_FLOPS(
et_vk.q8ta_pixel_shuffle.default, pixel_shuffle_flops);
REGISTER_OPERATOR_MEMORY_TRAFFIC(
et_vk.q8ta_pixel_shuffle.with_activation, throughput::tensor_traffic);
REGISTER_OPERATOR_COMPUTE_FLOPS(
et_vk.q8ta_pixel_shuffle.with_activation, pixel_shuffle_flops);
}
} // namespace
} // namespace vkcompute::shader_profiling
Loading
Loading