diff --git a/backends/qualcomm/_passes/recompose_rms_norm.py b/backends/qualcomm/_passes/recompose_rms_norm.py index f9cdd676345..5296ee5477c 100644 --- a/backends/qualcomm/_passes/recompose_rms_norm.py +++ b/backends/qualcomm/_passes/recompose_rms_norm.py @@ -3,12 +3,51 @@ # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import logging +import math +import numbers +import operator + import torch -from executorch.backends.qualcomm._passes.utils import find_patterns +from executorch.backends.qualcomm._passes.utils import copy_meta, find_patterns from executorch.backends.qualcomm.builders.node_visitor import dq_ops +from executorch.backends.qualcomm.builders.utils import is_parameter from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.dialects.edge._ops import EdgeOpOverload from executorch.exir.pass_base import ExportPass, PassResult from executorch.exir.passes import dead_code_elimination_pass +from torch._functorch._aot_autograd.descriptors import BufferAOTInput, ParamAOTInput +from torch._subclasses.fake_tensor import FakeTensor + +logger = logging.getLogger(__name__) + +# Export wraps lifted scalar tensor constants before they reach Tensor_Tensor ops. +SCALAR_WRAPPER_TARGETS = { + exir_ops.edge.aten.alias_copy.default, + exir_ops.edge.aten.detach.default, + exir_ops.edge.aten.detach_.default, + exir_ops.edge.aten.lift_fresh_copy.default, + torch.ops.aten.alias_copy.default, + torch.ops.aten.detach.default, + torch.ops.aten.detach_.default, + torch.ops.aten.lift_fresh_copy.default, +} + +TO_DTYPE_TARGETS = { + exir_ops.edge.aten.to.dtype, + torch.ops.aten.to.dtype, +} + +DIM_ORDER_COPY_TARGETS = { + exir_ops.edge.dim_order_ops._to_dim_order_copy.default, +} + +VIEW_TARGETS = { + exir_ops.edge.aten.view_copy.default, + torch.ops.aten.view.default, + torch.ops.aten.view_as.default, + torch.ops.aten.view_copy.default, +} def _is_node(node): @@ -19,10 +58,6 @@ def _is_call(node): return _is_node(node) and node.op == "call_function" -def _is_placeholder(node): - return _is_node(node) and node.op == "placeholder" - - def _is_get_attr(node): return _is_node(node) and node.op == "get_attr" @@ -58,6 +93,7 @@ def _is_pow(node): return _is_call(node) and node.target in [ exir_ops.edge.aten.pow.Tensor_Tensor, exir_ops.edge.aten.pow.Tensor_Scalar, + torch.ops.aten.pow.Tensor_Tensor, torch.ops.aten.pow.Tensor_Scalar, ] @@ -69,102 +105,403 @@ def _is_rsqrt(node): ] +def _tensor_dtype(node): + value = node.meta.get("val") if _is_node(node) else None + return value.dtype if value is not None else None + + +def _is_cast_wrapper(node): + if not _is_call(node) or node.target not in ( + TO_DTYPE_TARGETS | DIM_ORDER_COPY_TARGETS + ): + return False + source = node.args[0] + source_value = source.meta.get("val") if _is_node(source) else None + result_value = node.meta.get("val") + if source_value is None or result_value is None: + return False + if source_value.shape != result_value.shape: + return False + if node.target in DIM_ORDER_COPY_TARGETS: + dim_order = node.kwargs.get("dim_order") + if dim_order is not None and list(dim_order) != list(range(source_value.dim())): + return False + return True + + +def _get_scalar(value, lifted_tensor_constants=None): + while _is_node(value) and value.target in SCALAR_WRAPPER_TARGETS: + value = value.args[0] + if _is_node(value): + node = value + value = ( + lifted_tensor_constants.get(node.name) + if lifted_tensor_constants is not None + else None + ) + if value is None: + node_val = node.meta.get("val") + if isinstance(node_val, FakeTensor): + value = node_val.constant + if isinstance(value, torch.Tensor): + if value.numel() != 1: + return None + value = value.item() + if isinstance(value, numbers.Real) and not isinstance(value, bool): + return value + return None + + +def _writes_only_fresh_activations(node): + """Recognize writes that cannot change a scalar's state or alias.""" + schema = node.target._schema + for index, argument in enumerate(schema.arguments): + if argument.alias_info is None or not argument.alias_info.is_write: + continue + value = ( + node.args[index] + if index < len(node.args) + else node.kwargs.get(argument.name) + ) + if ( + not _is_call(value) + or value.target in SCALAR_WRAPPER_TARGETS + or not isinstance(value.target, (torch._ops.OpOverload, EdgeOpOverload)) + ): + return False + producer = value.target._schema + tensor = value.meta.get("val") + if ( + producer.is_mutable + or len(producer.returns) != 1 + or producer.returns[0].alias_info is not None + or not isinstance(tensor, FakeTensor) + or tensor.constant is not None + ): + return False + return True + + class RecomposeRmsNorm(ExportPass): """ Merge decomposed operators back to one super node. """ - def __init__(self, quantization_capture=False): + def __init__(self, quantization_capture=False, edge_program=None): super(RecomposeRmsNorm, self).__init__() self.rms_norm_target = exir_ops.edge.aten.rms_norm.default - self.skip_targets = [ - exir_ops.edge.aten.to.dtype, - ] self.quantization_capture = quantization_capture + self.edge_program = edge_program + self.lifted_tensor_constants = {} + self.allow_buffer_scalars = False + if edge_program is not None: + # FP export keeps lifted tensor literals in ExportedProgram.constants, + # rather than attaching them to the placeholder's FakeTensor metadata. + for ( + node_name, + constant_name, + ) in edge_program.graph_signature.inputs_to_lifted_tensor_constants.items(): + constant = edge_program.constants.get(constant_name) + if constant is not None: + self.lifted_tensor_constants[node_name] = constant if quantization_capture: self.rms_norm_target = torch.ops.aten.rms_norm.default - self.skip_targets = [ - torch.ops.aten.to.dtype, - ] - def _get_input_node(self, node): - input_node = node.args[0] - while input_node.target in self.skip_targets: - input_node = input_node.args[0] - return input_node + def _get_rms_scalar(self, operand, *, exponent=False): + while _is_node(operand) and operand.target in SCALAR_WRAPPER_TARGETS: + operand = operand.args[0] + if not _is_node(operand): + return _get_scalar(operand) + descriptor = operand.meta.get("desc") + if not self.allow_buffer_scalars or isinstance(descriptor, ParamAOTInput): + return None + if operand.op == "placeholder" and self.edge_program is not None: + signature = self.edge_program.graph_signature + if ( + operand.name in signature.user_inputs + or operand.name in signature.inputs_to_parameters + ): + return None + scalar = _get_scalar(operand, self.lifted_tensor_constants) + if scalar is not None: + return scalar + if operand.op == "placeholder" and self.edge_program is not None: + target = self.edge_program.graph_signature.inputs_to_buffers.get( + operand.name, + descriptor.target if isinstance(descriptor, BufferAOTInput) else None, + ) + value = self.edge_program.state_dict.get( + target, self.edge_program.constants.get(target) + ) + elif operand.op == "get_attr": + try: + value = operand.graph.owning_module.get_buffer(operand.target) + except AttributeError: + return None + else: + return None + if ( + isinstance(value, torch.Tensor) + and value.dim() <= (1 if exponent else 0) + and not value.requires_grad + ): + return _get_scalar(value) + return None + + def _is_fp32_upcast(self, node): + if not _is_cast_wrapper(node): + return False + return ( + _tensor_dtype(node.args[0]) in {torch.float16, torch.bfloat16} + and _tensor_dtype(node) == torch.float32 + ) + + def _get_wildcard_path(self, last_mul, outer_mul): + def find_path(node, remaining): + if node is outer_mul: + return [] + if remaining == 0 or not _is_cast_wrapper(node): + return None + path = find_path(node.args[0], remaining - 1) + return None if path is None else [node, *path] + + for arg in last_mul.args: + path = find_path(arg, 3) + if path is not None: + return path + return None + + def _is_known_output_cast_path(self, path, square): + normalized_input = square.args[0] + if not self._is_fp32_upcast(normalized_input): + return all( + _tensor_dtype(node.args[0]) == _tensor_dtype(node) for node in path + ) + input_dtype = _tensor_dtype(normalized_input.args[0]) + if not path or _tensor_dtype(path[0]) != input_dtype: + return False + return all( + _tensor_dtype(node.args[0]) in {torch.float32, input_dtype} + and _tensor_dtype(node) in {torch.float32, input_dtype} + for node in path + ) + + def _is_last_dim_reduction(self, mean, input_node): + input_val = input_node.meta.get("val") + if input_val is None or input_val.dim() == 0: + return False + + dim = mean.args[1] if len(mean.args) > 1 else mean.kwargs.get("dim") + keepdim = ( + mean.args[2] if len(mean.args) > 2 else mean.kwargs.get("keepdim", False) + ) + if isinstance(dim, int): + dim = [dim] + elif dim is None or dim == [] or dim == (): + dim = range(input_val.dim()) + elif not isinstance(dim, (list, tuple)): + return False + return ( + len(dim) == 1 + and dim[0] in (-1, input_val.dim() - 1) + and (keepdim is True or input_val.dim() == 1) + ) + + def _validate_rms_norm(self, pattern): + last_mul, _, outer_mul, inverse_sqrt, add, mean, square = pattern + if not ( + _is_rsqrt(inverse_sqrt) + or ( + _is_pow(inverse_sqrt) + and self._get_rms_scalar(inverse_sqrt.args[1], exponent=True) == -0.5 + ) + ): + return None + + if _is_mul(square): + if len(square.args) != 2 or square.args[0] is not square.args[1]: + return None + elif not ( + _is_pow(square) and self._get_rms_scalar(square.args[1], exponent=True) == 2 + ): + return None + + outer_inputs = [arg for arg in outer_mul.args if arg is not inverse_sqrt] + if len(outer_inputs) != 1 or outer_inputs[0] is not square.args[0]: + return None + + wildcard_path = self._get_wildcard_path(last_mul, outer_mul) + if wildcard_path is None or not self._is_known_output_cast_path( + wildcard_path, square + ): + return None + + if not self._is_last_dim_reduction(mean, square.args[0]): + return None + if any( + _tensor_dtype(node) != _tensor_dtype(square.args[0]) + for node in (square, mean, add, inverse_sqrt) + ): + return None + alpha = add.kwargs.get("alpha", add.args[2] if len(add.args) > 2 else 1) + if alpha != 1: + return None + + if len(add.args) < 2 or mean not in add.args[:2]: + return None + eps_arg = add.args[1] if add.args[0] is mean else add.args[0] + eps = self._get_rms_scalar(eps_arg) + if eps is None or not math.isfinite(eps) or eps < 0: + return None + return eps + + def _is_valid_rms_norm(self, pattern): + return self._validate_rms_norm(pattern) is not None + + def _find_rms_norm_pattern(self, node, patterns): + for matched_patterns in find_patterns(node, patterns): + if matched_patterns is None: + continue + for candidate in matched_patterns: + eps = self._validate_rms_norm(candidate) + if eps is not None: + return candidate, eps + logger.debug( + "Rejected RMS norm candidate %s: invalid normalization semantics", + candidate[0].name, + ) + return None + + def _is_static_weight(self, node): + source = node.args[0] if _is_dq(node) else node + return ( + _is_get_attr(source) + or isinstance(source.meta.get("desc"), (ParamAOTInput, BufferAOTInput)) + or ( + self.edge_program is not None + and is_parameter(source, self.edge_program) + ) + ) + + def _get_static_weight(self, node): + if self._is_static_weight(node): + return node + if not _is_call(node) or node.target not in VIEW_TARGETS: + return None + + source = node.args[0] + source_val = source.meta.get("val") if _is_node(source) else None + node_val = node.meta.get("val") + if ( + source_val is None + or node_val is None + or source_val.shape != node_val.shape + or not self._is_static_weight(source) + ): + return None + return source + + def _get_rms_norm_weight(self, last_mul_node, outer_mul, input_node): + for arg in last_mul_node.args: + if arg is outer_mul or not _is_node(arg): + continue + weight = self._get_static_weight(arg) + if weight is None: + continue + arg_val = weight.meta.get("val") + input_val = input_node.meta.get("val") + if ( + arg_val is not None + and input_val is not None + and len(arg_val.shape) == 1 + and list(arg_val.shape) == [input_val.shape[-1]] + ): + return weight, list(arg_val.shape) + return None, None def call(self, graph_module: torch.fx.GraphModule): graph = graph_module.graph + graphs = ( + (graph,) if self.edge_program is None else (graph, self.edge_program.graph) + ) + # A write can change a scalar through an alias. Keep checking the + # original graph too, but allow writes to fresh activation results + # (e.g. Gemma's final hidden_states *= layer_scalar). + self.allow_buffer_scalars = not ( + self.edge_program is not None + and self.edge_program.graph_signature.buffers_to_mutate + ) and all( + node.target in SCALAR_WRAPPER_TARGETS + or node.target is operator.getitem + or ( + isinstance(node.target, (torch._ops.OpOverload, EdgeOpOverload)) + and ( + not node.target._schema.is_mutable + or _writes_only_fresh_activations(node) + ) + ) + for source_graph in graphs + for node in source_graph.nodes + if node.op not in {"placeholder", "get_attr", "output"} + ) # Root Mean Square normalization math equivalent implementation patterns = [ - # transformers.models.qwen2.modeling_qwen2.Qwen2RMSNorm + # transformers.models.qwen2.modeling_qwen2.Qwen2RMSNorm and + # transformers.models.qwen3.modeling_qwen3.Qwen3RMSNorm [_is_mul, "*", _is_mul, _is_rsqrt, _is_add, _is_mean, _is_pow], # executorch.examples.models.llama.norm.RMSNorm [_is_mul, "*", _is_mul, _is_rsqrt, _is_add, _is_mean, _is_mul], + # transformers.models.gemma4.modeling_gemma4.Gemma4RMSNorm + [_is_mul, "*", _is_mul, _is_pow, _is_add, _is_mean, _is_pow], ] for node in graph.nodes: if not _is_mul(node): continue - rms_norm_patterns = [ - pattern - for pattern in find_patterns(node, patterns) - if pattern is not None - ] - - if len(rms_norm_patterns) > 0: - # Use first matched pattern - rms_norm_pattern = rms_norm_patterns[0][0] - last_mul_node = rms_norm_pattern[0] - gamma_node = None - # weight should be a constant - for arg in last_mul_node.args: - if _is_get_attr(arg) or _is_placeholder(arg) or _is_dq(arg): - gamma_node = arg - - if gamma_node is None: - continue - - input_node = self._get_input_node(rms_norm_pattern[6]) - # Only the last-dim form the RmsNorm builder supports is recomposed. - # Anything else, e.g. a 0-d scale, is left decomposed. - input_shape = list(input_node.meta["val"].shape) - rank = len(input_shape) - mean_node = rms_norm_pattern[5] - mean_dims = mean_node.args[1] or range(rank) - keepdim = len(mean_node.args) > 2 and mean_node.args[2] - if ( - rank == 0 - or [dim % rank for dim in mean_dims] != [rank - 1] - or not (keepdim or rank == 1) - or list(gamma_node.meta["val"].shape) != input_shape[-1:] - ): - continue - - eps = rms_norm_pattern[4].args[1] - if isinstance(eps, torch.fx.Node): - eps = eps.meta["val"].constant.item() - - with graph.inserting_before(last_mul_node): - # args schema - # (Tensor input, int[] normalized_shape, Tensor? weight=None, float? eps=None) -> Tensor - rms_node = graph.create_node( - "call_function", - self.rms_norm_target, - ( - input_node, - input_shape[-1:], - gamma_node, - eps, - ), - ) - users = last_mul_node.users.copy() - for user in users: - user.replace_input_with(last_mul_node, rms_node) - # copy metadata - rms_node.meta = last_mul_node.meta + match = self._find_rms_norm_pattern(node, patterns) + if match is None: + continue + rms_norm_pattern, eps = match + + last_mul_node = rms_norm_pattern[0] + outer_mul = rms_norm_pattern[2] + normalized_input = rms_norm_pattern[-1].args[0] + input_node = ( + normalized_input.args[0] + if self._is_fp32_upcast(normalized_input) + else normalized_input + ) + if _tensor_dtype(last_mul_node) != _tensor_dtype(input_node): + continue + gamma_node, gamma_shape = self._get_rms_norm_weight( + last_mul_node, outer_mul, input_node + ) + if gamma_node is None: + logger.debug( + "Rejected RMS norm candidate %s: no non-scalar weight", + last_mul_node.name, + ) + continue + + with graph.inserting_before(last_mul_node): + # args schema + # (Tensor input, int[] normalized_shape, Tensor? weight=None, float? eps=None) -> Tensor + rms_node = graph.create_node( + "call_function", + self.rms_norm_target, + ( + input_node, + gamma_shape, + gamma_node, + eps, + ), + ) + rms_node.meta = copy_meta(last_mul_node.meta) + users = last_mul_node.users.copy() + for user in users: + user.replace_input_with(last_mul_node, rms_node) dead_code_elimination_pass(graph_module) return PassResult(graph_module, True) diff --git a/backends/qualcomm/_passes/remove_redundancy.py b/backends/qualcomm/_passes/remove_redundancy.py index 7589d40b745..2bbcaf17853 100644 --- a/backends/qualcomm/_passes/remove_redundancy.py +++ b/backends/qualcomm/_passes/remove_redundancy.py @@ -4,6 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import torch +from executorch.backends.qualcomm.builders.utils import is_parameter from executorch.exir.dialects._ops import ops as exir_ops from executorch.exir.pass_base import ExportPass, PassResult from executorch.exir.passes import dead_code_elimination_pass @@ -14,8 +15,9 @@ class RemoveRedundancy(ExportPass): Trim certain operators to reduce unnecessary overhead. """ - def __init__(self, quantization_capture=False): + def __init__(self, quantization_capture=False, edge_program=None): super(RemoveRedundancy, self).__init__() + self.edge_program = edge_program self.redundant_ops_general = { torch.clone: self._default_condition, torch.ops.aten.clone.default: self._default_condition, @@ -26,6 +28,7 @@ def __init__(self, quantization_capture=False): exir_ops.edge.aten.expand_copy.default: self._same_shape_condition, exir_ops.edge.aten.lift_fresh_copy.default: self._default_condition, exir_ops.edge.aten.repeat.default: self._same_shape_condition, + exir_ops.edge.aten.view_copy.default: self._rms_norm_weight_view_condition, # remove this target if '_skip_dim_order' is set to False exir_ops.edge.dim_order_ops._to_dim_order_copy.default: self._dim_order_op_condition, # remove channel_last / contiguous _to_copy if '_skip_dim_order' is set to True @@ -48,6 +51,22 @@ def _dim_order_op_condition(self, node): def _same_shape_condition(self, node): return node.args[0].meta["val"].shape == node.meta["val"].shape + def _rms_norm_weight_view_condition(self, node): + source = node.args[0] + is_static = source.op == "get_attr" or ( + self.edge_program is not None and is_parameter(source, self.edge_program) + ) + if not is_static or not self._same_shape_condition(node): + return False + return all( + user.target == exir_ops.edge.aten.rms_norm.default + and ( + (len(user.args) > 2 and user.args[2] is node) + or user.kwargs.get("weight") is node + ) + for user in node.users + ) + def _to_copy_op_condition(self, node): return "memory_format" in node.kwargs diff --git a/backends/qualcomm/builders/op_rms_norm.py b/backends/qualcomm/builders/op_rms_norm.py index e34efed61f3..d9cebe8a5f3 100644 --- a/backends/qualcomm/builders/op_rms_norm.py +++ b/backends/qualcomm/builders/op_rms_norm.py @@ -19,7 +19,7 @@ ) from executorch.exir.dialects._ops import ops as exir_ops -from .node_visitor import NodeVisitor +from .node_visitor import NodeVisitor, q_ops from .node_visitor_manager import register_node_visitor from .qnn_constants import OpRmsNorm, QNN_OP_PACKAGE_NAME_QTI_AISW @@ -51,7 +51,7 @@ def define_node( normalized_shapes = node.args[1] if ( len(normalized_shapes) != 1 - and normalized_shapes[0] != input_tensor.shape[-1] + or normalized_shapes[0] != input_tensor.shape[-1] ): warnings.warn( "[QNN Delegate Op Builder]: Only supports normalization with last input dimension.", @@ -61,9 +61,10 @@ def define_node( axes = [node.args[0].meta["val"].dim() - 1] axes_shape = [len(axes)] - has_weight = len(node.args) > 2 and node.args[2] is not None + weight = node.args[2] if len(node.args) > 2 else node.kwargs.get("weight") + has_weight = weight is not None if has_weight: - weight_node = self.get_node(node.args[2]) + weight_node = self.get_node(weight) weight_tensor = get_parameter(weight_node, self.edge_program) else: # elementwise_affine=False: use all-ones weight as identity @@ -110,9 +111,17 @@ def define_node( nodes_to_wrappers, ) - epsilon = torch.finfo(torch.float32).eps - if len(node.args) > 3: - epsilon = node.args[3] + eps = node.args[3] if len(node.args) > 3 else node.kwargs.get("eps") + if eps is not None: + epsilon = eps + else: + logical_input = node.args[0] + input_dtype = logical_input.meta["val"].dtype + if logical_input.target in q_ops: + input_dtype = logical_input.args[0].meta["val"].dtype + if not input_dtype.is_floating_point: + input_dtype = node.meta["val"].dtype + epsilon = torch.finfo(input_dtype).eps output_tensor = self.get_tensor(node, node) output_tensor_wrapper = self.define_tensor( node, diff --git a/backends/qualcomm/tests/models.py b/backends/qualcomm/tests/models.py index a11f201338c..b3b07f570f3 100644 --- a/backends/qualcomm/tests/models.py +++ b/backends/qualcomm/tests/models.py @@ -2526,6 +2526,18 @@ def forward(self, x): return self.rms(x) +class Fp32RmsNorm(torch.nn.Module): + def __init__(self, dim): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(dim)) + + def forward(self, x): + input_dtype = x.dtype + x = x.float() + x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-6) + return x.to(input_dtype) * self.weight.to(input_dtype) + + class Roll(torch.nn.Module): def __init__(self, shifts, dims=None): super().__init__() diff --git a/backends/qualcomm/tests/rework/passes/test.py b/backends/qualcomm/tests/rework/passes/test.py index ad38429733b..eeab93a3f0a 100644 --- a/backends/qualcomm/tests/rework/passes/test.py +++ b/backends/qualcomm/tests/rework/passes/test.py @@ -407,6 +407,12 @@ def test_recompose_rms_norm(request, kwargs): RecomposeRmsNorm.test(request, kwargs) # noqa: F405 +@enumerate_backends_quantized() +@repack_pass_fixtures +def test_recompose_rms_norm_annotation(request, kwargs): + RecomposeRmsNorm.test_annotation(request, kwargs) # noqa: F405 + + @enumerate_backends() @repack_pass_fixtures def test_remove_0d_tensor(request, kwargs): diff --git a/backends/qualcomm/tests/rework/src/pattern.py b/backends/qualcomm/tests/rework/src/pattern.py index d5efb37cbfe..e249624c917 100644 --- a/backends/qualcomm/tests/rework/src/pattern.py +++ b/backends/qualcomm/tests/rework/src/pattern.py @@ -7,6 +7,7 @@ from __future__ import annotations import inspect +import io import math import operator from typing import TYPE_CHECKING @@ -38,6 +39,7 @@ QCOM_QUANT_ATTRS, QCOM_REQUANTIZE, ) +from executorch.exir import EdgeCompileConfig, to_edge from executorch.exir.delegate import executorch_call_delegate from executorch.exir.dialects._ops import ops as exir_ops from torch.fx.passes.utils.source_matcher_utils import get_source_partitions @@ -4909,16 +4911,286 @@ def test( class RecomposeRmsNorm: + class _BufferRmsNorm(torch.nn.Module): + def __init__( + self, + square=2.0, + eps=9.765625e-10, + mutate=None, + keepdim=True, + detach_exponent=False, + ): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(8)) + self.mutate = mutate + self.keepdim = keepdim + self.detach_exponent = detach_exponent + if isinstance(square, torch.nn.Parameter): + self.square = square + else: + self.register_buffer("square", torch.as_tensor(square)) + self.register_buffer("eps", torch.as_tensor(eps)) + if mutate == "buffer": + self.register_buffer("alias", self.square.view(())) + + def forward(self, x): + square, eps = self.square, self.eps + if self.mutate == "literal": + square, eps = 2, torch.tensor(1e-6) + eps.view(()).add_(0.1) + elif self.mutate == "buffer": + self.alias.add_(2.0) + elif self.mutate == "computed_epsilon": + eps = x.sum() + eps.add_(0.1) + if self.detach_exponent: + square = square.detach() + statistic = x.pow(square).mean(-1, keepdim=self.keepdim) + norm = x * torch.rsqrt(statistic + eps) + output = norm * self.weight + if self.mutate == "epsilon": + self.eps.add_(0.1) + elif self.mutate == "activation": + output.mul_(0.75) + elif self.mutate == "activation_reads_epsilon": + output.mul_(self.eps) + elif self.mutate == "scalar_activation": + scalar = x.sum() + scalar.add_(0.1) + output = output + scalar + return output + + @staticmethod + def _test_saved(options, expected_count, assertions): + model = RecomposeRmsNorm._BufferRmsNorm(**options) + inputs = (torch.rand(2, 4, 8) + 0.5 if model.keepdim else torch.zeros(8),) + archive = io.BytesIO() + torch.export.save(torch.export.export(model, inputs), archive) + archive.seek(0) + source = torch.export.load(archive) + # Exercise Edge re-export renaming and keep the original mutation graph. + for node, spec in zip( + source.graph.find_nodes(op="placeholder"), + source.graph_signature.input_specs, + ): + if spec.target is not None: + node.name = node.target = spec.target.replace(".", "_") + spec.arg.name = node.name + source.graph_module.recompile() + # PyTorch cannot re-export the saved shared-buffer mutation graph. + edge = options.get("mutate") != "buffer" + program = ( + to_edge( + source, compile_config=EdgeCompileConfig(_check_ir_validity=False) + ).exported_program() + if edge + else source + ) + for node in program.graph.nodes: + if program.graph_signature.inputs_to_buffers.get(node.name) in { + "square", + "eps", + "alias", + }: + assert node.meta["val"].constant is None + gm = program.graph_module + _passes.RecomposeRmsNorm( + quantization_capture=not edge, edge_program=source + ).call(gm) + gm.graph.lint() + target = ( + exir_ops.edge.aten.rms_norm.default + if edge + else torch.ops.aten.rms_norm.default + ) + assertions.assert_target_count(gm, target, expected_count) + actual = program.module() + for _ in range(2 if options.get("mutate") else 1): + torch.testing.assert_close(actual(*inputs), model(*inputs)) + if expected_count: + rms = next(node for node in gm.graph.nodes if node.target == target) + assert rms.args[3] == model.eps.item() + if not options: + x = inputs[0].detach().requires_grad_() + actual_grad = torch.autograd.grad(actual(x).sum(), x)[0] + expected_grad = torch.autograd.grad(model(x).sum(), x)[0] + torch.testing.assert_close(actual_grad, expected_grad) + class _RmsNorm(torch.nn.Module): - def __init__(self, dim: int, eps: float = 1e-6): + def __init__( + self, + dim: int, + eps: float = 1e-6, + reduction_dim: int = -1, + keepdim: bool = True, + tensor_eps: bool = False, + ): super().__init__() self.weight = torch.nn.Parameter(torch.ones(dim)) self.eps = eps + self.reduction_dim = reduction_dim + self.keepdim = keepdim + self.tensor_eps = tensor_eps def forward(self, x): + norm = x * torch.rsqrt( + x.pow(2).mean(self.reduction_dim, keepdim=self.keepdim) + + (torch.tensor([self.eps]) if self.tensor_eps else self.eps) + ) + return norm * self.weight + + class _Gemma4RmsNorm(_RmsNorm): + def __init__( + self, + dim: int, + reduction_dim: int = -1, + negative_eps: bool = False, + tensor_eps: bool = False, + tensor_exponent: bool = False, + eps_first: bool = False, + activation_scale: float | None = None, + ): + super().__init__(dim) + self.reduction_dim = reduction_dim + self.negative_eps = negative_eps + self.tensor_eps = tensor_eps + self.tensor_exponent = tensor_exponent + self.eps_first = eps_first + if activation_scale is not None: + self.register_buffer("layer_scalar", torch.tensor([activation_scale])) + + def forward(self, x, statistic_input=None): + eps = torch.tensor(self.eps) if self.tensor_eps else self.eps + exponent = torch.tensor(-0.5) if self.tensor_exponent else -0.5 + statistic_input = x if statistic_input is None else statistic_input + statistic = statistic_input.pow(2).mean(self.reduction_dim, keepdim=True) + eps = -eps if self.negative_eps else eps + statistic = eps + statistic if self.eps_first else statistic + eps + norm = x * torch.pow(statistic, exponent) + output = norm * self.weight + if hasattr(self, "layer_scalar"): + output.mul_(self.layer_scalar) + return output + + class _RmsNormWithIntermediate(_RmsNorm): + def forward(self, x): + norm = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + return (norm + 1) * self.weight + + class _RmsNormWithParameterView(_RmsNorm): + def forward(self, x): + norm = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + return norm * self.weight.view_as(self.weight) + + class _RmsNormWithRuntimeWeight(torch.nn.Module): + def __init__(self): + super().__init__() + self.eps = 1e-6 + + def forward(self, x, weight): norm = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + return norm * weight + + class _RmsNormWithProductSquare(_RmsNorm): + def forward(self, x, y): + norm = x * torch.rsqrt((x * y).mean(-1, keepdim=True) + self.eps) + return norm * self.weight + + class _RsqrtTensorSquareExponent(_RmsNorm): + def forward(self, x): + norm = x * torch.rsqrt( + torch.pow(x, torch.tensor(4)).mean(-1, keepdim=True) + self.eps + ) return norm * self.weight + class _Fp32RmsNorm(_RmsNorm): + def forward(self, x): + input_dtype = x.dtype + x = x.float() + norm = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + return self.weight * norm.to(input_dtype) + + class _Gemma4Fp32RmsNorm(_Gemma4RmsNorm): + def forward(self, x): + input_dtype = x.dtype + x = x.float() + statistic = x.pow(2).mean(-1, keepdim=True) + self.eps + norm = x * torch.pow(statistic, -0.5) + return norm.to(input_dtype) * self.weight.to(input_dtype) + + class _LlamaFp32RmsNorm(_RmsNorm): + def forward(self, x): + output = x.float() + output = output * torch.rsqrt( + output.pow(2).mean(-1, keepdim=True) + self.eps + ) + return output.type_as(x) * self.weight.type_as(x) + + @staticmethod + @unpack_pass_fixtures + def test_annotation( + backend_type: QnnExecuTorchBackendType, + assertions: Assertions, + pass_pipeline: PassPipeline, + subtests, + ): + for name, model, shape in ( + ( + "shape_preserving_weight_view", + RecomposeRmsNorm._RmsNormWithParameterView(16), + (1, 4, 8, 16), + ), + ("identity_casts", RecomposeRmsNorm._Fp32RmsNorm(8), (2, 4, 8)), + ( + "rank_one_known_tensor_epsilon", + RecomposeRmsNorm._RmsNorm( + 8, reduction_dim=None, keepdim=False, tensor_eps=True + ), + (8,), + ), + ): + with subtests.test(msg=name): + inputs = (torch.rand(shape) + 0.5,) + if model.tensor_eps: + program = torch.export.export(model, inputs) + # Exercise scalar values available through constant metadata. + add = next( + node + for node in program.graph.nodes + if node.target + in {torch.ops.aten.add.Tensor, exir_ops.edge.aten.add.Tensor} + ) + add.args[1].meta["val"].constant = torch.tensor([model.eps]) + _passes.RecomposeRmsNorm( + quantization_capture=True, edge_program=program + ).call(program.graph_module) + gm = program.module() + else: + gm = pass_pipeline.lower_annotation_gm( + module=model, + sample_input=inputs, + target_pass=_passes.RecomposeRmsNorm, + backend_type=backend_type, + ) + assertions.assert_target_count(gm, torch.ops.aten.rms_norm.default, 1) + if name == "shape_preserving_weight_view": + assertions.assert_no_target( + gm, + { + torch.ops.aten.mean.dim, + torch.ops.aten.view.default, + torch.ops.aten.view_as.default, + torch.ops.aten.view_copy.default, + }, + ) + rms_norm = next( + node + for node in gm.graph.nodes + if node.target == torch.ops.aten.rms_norm.default + ) + assert rms_norm.args[2].op == "get_attr" + torch.testing.assert_close(gm(*inputs), model(*inputs)) + @staticmethod @unpack_pass_fixtures def test( @@ -4927,20 +5199,202 @@ def test( backend_type: QnnExecuTorchBackendType, assertions: Assertions, pass_pipeline: PassPipeline, + subtests, ): + for name, options, expected_count in ( + ("saved_buffers", {}, 1), + ("activation_mutation", {"mutate": "activation"}, 1), + ( + "activation_reads_epsilon", + {"mutate": "activation_reads_epsilon"}, + 1, + ), + ( + "fresh_scalar_mutation", + {"mutate": "scalar_activation"}, + 1, + ), + ( + "computed_epsilon_mutation", + {"mutate": "computed_epsilon"}, + 0, + ), + ( + "trainable_exponent", + { + "square": torch.nn.Parameter(torch.tensor(2.0)), + "detach_exponent": True, + }, + 0, + ), + ("broadcast_epsilon", {"eps": torch.ones(1, 1, 1, 1)}, 0), + ( + "aliased_buffer_mutation", + {"mutate": "buffer", "detach_exponent": True}, + 0, + ), + ("aliased_literal_mutation", {"mutate": "literal"}, 0), + ("mutated_epsilon", {"mutate": "epsilon"}, 0), + ( + "promoted_epsilon_dtype", + {"eps": torch.tensor(1e-50, dtype=torch.float64), "keepdim": False}, + 0, + ), + ): + with subtests.test(msg=name): + RecomposeRmsNorm._test_saved(options, expected_count, assertions) + dim = 16 inputs = (torch.randn(1, 4, 8, dim),) - gm = pass_pipeline.lower_edge_ep( - module=RecomposeRmsNorm._RmsNorm(dim), - sample_input=inputs, - backend_type=backend_type, - compile_spec=compile_spec, - target_pass=_passes.RecomposeRmsNorm, - quantizer=quantizer, - ).graph_module - assertions.assert_target_count(gm, exir_ops.edge.aten.rms_norm.default, 1) - assertions.assert_no_target(gm, exir_ops.edge.aten.rsqrt.default) - assertions.assert_no_target(gm, exir_ops.edge.aten.mean.dim) + for name, module in ( + ( + "gemma_activation_mutation", + RecomposeRmsNorm._Gemma4RmsNorm(dim, activation_scale=0.75), + ), + ("rsqrt", RecomposeRmsNorm._RmsNorm(dim)), + ( + "shape_preserving_weight_view", + RecomposeRmsNorm._RmsNormWithParameterView(dim), + ), + ( + "gemma_positive_last_reduction_dim", + RecomposeRmsNorm._Gemma4RmsNorm(dim, reduction_dim=3), + ), + ( + "gemma_tensor_scalars_commuted_eps", + RecomposeRmsNorm._Gemma4RmsNorm( + dim, tensor_exponent=True, tensor_eps=True, eps_first=True + ), + ), + ): + with subtests.test(msg=name): + program = pass_pipeline.lower_edge_ep( + module=module, + sample_input=inputs, + backend_type=backend_type, + compile_spec=compile_spec, + target_pass=_passes.RecomposeRmsNorm, + quantizer=quantizer, + ) + gm = program.graph_module + assertions.assert_target_count( + gm, exir_ops.edge.aten.rms_norm.default, 1 + ) + assertions.assert_no_target(gm, exir_ops.edge.aten.mean.dim) + rms_norm = next( + node + for node in gm.graph.nodes + if node.target == exir_ops.edge.aten.rms_norm.default + ) + assert rms_norm.args[2].meta["val"].shape == (dim,) + assert math.isclose(rms_norm.args[3], module.eps, rel_tol=0.02) + if name == "shape_preserving_weight_view": + assertions.assert_no_target( + gm, exir_ops.edge.aten.view_copy.default + ) + if name == "rsqrt": + assertions.assert_no_target(gm, exir_ops.edge.aten.rsqrt.default) + if name == "gemma_activation_mutation" and quantizer is None: + torch.testing.assert_close( + program.module()(*inputs), module(*inputs) + ) + + wrong_weight_shape = RecomposeRmsNorm._RmsNorm(dim) + wrong_weight_shape.weight = torch.nn.Parameter(torch.ones(1)) + for name, module, module_inputs in ( + ( + "rsqrt_tensor_square_exponent", + RecomposeRmsNorm._RsqrtTensorSquareExponent(dim), + inputs, + ), + ( + "gemma_wrong_reduction_dim", + RecomposeRmsNorm._Gemma4RmsNorm(dim, reduction_dim=0), + inputs, + ), + ( + "gemma_negative_eps", + RecomposeRmsNorm._Gemma4RmsNorm(dim, negative_eps=True), + inputs, + ), + ( + "intermediate_add", + RecomposeRmsNorm._RmsNormWithIntermediate(dim), + inputs, + ), + ( + "wrong_weight_shape_scalar_like", + wrong_weight_shape, + inputs, + ), + ( + "runtime_weight", + RecomposeRmsNorm._RmsNormWithRuntimeWeight(), + (inputs[0], torch.ones(dim)), + ), + ( + "product_square", + RecomposeRmsNorm._RmsNormWithProductSquare(dim), + ( + torch.rand_like(inputs[0]) + 0.5, + torch.rand_like(inputs[0]) + 0.5, + ), + ), + ( + "gemma_different_square_input", + RecomposeRmsNorm._Gemma4RmsNorm(dim), + (torch.randn(1, 4, 8, dim), torch.randn(1, 4, 8, dim)), + ), + ): + with subtests.test(msg=name): + gm = pass_pipeline.lower_edge_ep( + module=module, + sample_input=module_inputs, + backend_type=backend_type, + compile_spec=compile_spec, + target_pass=_passes.RecomposeRmsNorm, + quantizer=quantizer, + ).graph_module + assertions.assert_target_count( + gm, exir_ops.edge.aten.rms_norm.default, 0 + ) + assertions.assert_target_count(gm, exir_ops.edge.aten.mean.dim, 1) + if name == "gemma_negative_eps": + assertions.assert_target_count( + gm, + { + exir_ops.edge.aten.add.Scalar, + exir_ops.edge.aten.add.Tensor, + }, + 1, + ) + + if quantizer is None: + for name, module in ( + ("qwen2_fp32_cast", RecomposeRmsNorm._Fp32RmsNorm(dim).half()), + ( + "gemma4_fp32_pow_neg_half_cast", + RecomposeRmsNorm._Gemma4Fp32RmsNorm(dim).half(), + ), + ("llama_fp32_cast", RecomposeRmsNorm._LlamaFp32RmsNorm(dim).half()), + ): + with subtests.test(msg=name): + gm = pass_pipeline.lower_edge_ep( + module=module, + sample_input=(torch.randn(1, 4, 8, dim, dtype=torch.float16),), + backend_type=backend_type, + compile_spec=compile_spec, + target_pass=_passes.RecomposeRmsNorm, + quantizer=quantizer, + ).graph_module + rms_norm = next( + node + for node in gm.graph.nodes + if node.target == exir_ops.edge.aten.rms_norm.default + ) + assert rms_norm.args[0].meta["val"].dtype == torch.float16 + assert rms_norm.args[2].meta["val"].dtype == torch.float16 + assert rms_norm.meta["val"].dtype == torch.float16 class RemoveRedundancy: @@ -4948,6 +5402,38 @@ class _Clone(torch.nn.Module): def forward(self, x): return x.clone() + 1 + class _RmsNormParameterView(torch.nn.Module): + def __init__(self, dim: int): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(dim)) + + def forward(self, x): + return torch.ops.aten.rms_norm.default( + x, [self.weight.numel()], self.weight.view_as(self.weight), 1e-6 + ) + + class _RmsNormReshapedParameterView(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(2, 8)) + + def forward(self, x): + return torch.ops.aten.rms_norm.default(x, [16], self.weight.view(16), 1e-6) + + class _RmsNormRuntimeWeightView(torch.nn.Module): + def forward(self, x, weight): + return torch.ops.aten.rms_norm.default( + x, [weight.numel()], weight.view_as(weight), 1e-6 + ) + + class _StaticWeightView(torch.nn.Module): + def __init__(self, dim: int): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(dim)) + + def forward(self, x): + return x + self.weight.view_as(self.weight) + @staticmethod @unpack_pass_fixtures def test( @@ -4956,6 +5442,7 @@ def test( backend_type: QnnExecuTorchBackendType, assertions: Assertions, pass_pipeline: PassPipeline, + subtests, ): inputs = (torch.randn(3, 4),) gm = pass_pipeline.lower_edge_ep( @@ -4971,6 +5458,55 @@ def test( ) assertions.assert_target_count(gm, exir_ops.edge.aten.add.Tensor, 1) + if quantizer is None: + for name, module, module_inputs, expect_removed in ( + ( + "rms_norm_parameter_view", + RemoveRedundancy._RmsNormParameterView(16), + (torch.randn(1, 4, 16),), + True, + ), + ( + "rms_norm_reshaped_parameter_view", + RemoveRedundancy._RmsNormReshapedParameterView(), + (torch.randn(1, 4, 16),), + False, + ), + ( + "rms_norm_runtime_weight_view", + RemoveRedundancy._RmsNormRuntimeWeightView(), + (torch.randn(1, 4, 16), torch.ones(16)), + False, + ), + ( + "static_view_outside_rms_norm", + RemoveRedundancy._StaticWeightView(16), + (torch.randn(1, 4, 16),), + False, + ), + ): + with subtests.test(msg=name): + gm = pass_pipeline.lower_edge_ep( + module=module, + sample_input=module_inputs, + backend_type=backend_type, + compile_spec=compile_spec, + target_pass=_passes.RemoveRedundancy, + quantizer=quantizer, + ).graph_module + if expect_removed: + rms_norm = next( + node + for node in gm.graph.nodes + if node.target == exir_ops.edge.aten.rms_norm.default + ) + assert rms_norm.args[1] == [16] + assert rms_norm.args[2].op == "placeholder" + else: + assertions.assert_target_count( + gm, exir_ops.edge.aten.view_copy.default, 1 + ) + class ReplaceArangeArgs: class _Arange(torch.nn.Module): diff --git a/backends/qualcomm/tests/test_passes.py b/backends/qualcomm/tests/test_passes.py index cbd3c4ed5a8..d58d4b1e2a9 100644 --- a/backends/qualcomm/tests/test_passes.py +++ b/backends/qualcomm/tests/test_passes.py @@ -67,6 +67,7 @@ Unfold, ) from executorch.backends.qualcomm.utils.constants import ( + QCOM_DATA, QCOM_DTYPE, QCOM_ENCODING, QCOM_QUANT_ATTRS, @@ -590,6 +591,118 @@ def test_context_loader_op_lowers_with_ir_validation(self): ) self.assertEqual(1, len(context_loader_nodes)) + def test_rms_norm_builder_reads_keyword_weight_and_eps(self): + from executorch.backends.qualcomm.builders.node_visitor_manager import ( + _node_visitor_dict, + ) + from executorch.backends.qualcomm.builders.qnn_constants import OpRmsNorm + + for logical_dtype, storage_dtype, eps, expected_eps in ( + (torch.float32, torch.float32, 0.125, 0.125), + (torch.float32, torch.float32, None, torch.finfo(torch.float32).eps), + (torch.float16, torch.float16, None, torch.finfo(torch.float16).eps), + (torch.float32, torch.uint8, 0.125, 0.125), + (torch.float32, torch.uint8, None, torch.finfo(torch.float32).eps), + ): + with self.subTest( + logical_dtype=logical_dtype, storage_dtype=storage_dtype, eps=eps + ): + graph = torch.fx.Graph() + input_node = graph.placeholder("input") + input_node.meta["val"] = torch.empty(1, 4, 16, dtype=logical_dtype) + rms_input_node = input_node + if storage_dtype != logical_dtype: + quantize_node = graph.call_function( + exir_ops.edge.quantized_decomposed.quantize_per_tensor.default, + (input_node, 1.0, 0, 0, 255, storage_dtype), + ) + quantize_node.meta["val"] = torch.empty( + 1, 4, 16, dtype=storage_dtype + ) + rms_input_node = graph.call_function( + exir_ops.edge.quantized_decomposed.dequantize_per_tensor.default, + (quantize_node, 1.0, 0, 0, 255, storage_dtype), + ) + rms_input_node.meta["val"] = torch.empty( + 1, 4, 16, dtype=logical_dtype + ) + weight_node = graph.placeholder("weight") + node = graph.call_function( + exir_ops.edge.aten.rms_norm.default, + (rms_input_node, [16]), + {"weight": weight_node, "eps": eps}, + ) + node.meta["val"] = torch.empty(1, 4, 16, dtype=logical_dtype) + edge_program = MagicMock() + visitor = _node_visitor_dict["aten.rms_norm.default"]( + {}, edge_program, False + ) + visitor.get_tensor = MagicMock( + side_effect=lambda input_node, *_: input_node.meta["val"] + ) + visitor.define_tensor = MagicMock(return_value=MagicMock()) + + with ( + patch( + "executorch.backends.qualcomm.builders.op_rms_norm." + "get_parameter", + return_value=torch.arange(1, 17, dtype=torch.float32), + ) as get_parameter, + patch( + "executorch.backends.qualcomm.builders.op_rms_norm." + "PyQnnManager.PyQnnOpWrapper", + ) as op_wrapper, + ): + visitor.define_node(node, {}) + + get_parameter.assert_called_once_with(weight_node, edge_program) + epsilon_call = next( + call + for call in op_wrapper.return_value.AddScalarParam.call_args_list + if call.args[0] == OpRmsNorm.param_epsilon + ) + self.assertEqual(expected_eps, epsilon_call.args[2][QCOM_DATA]) + + def test_rms_norm_recomposition_rejects_scaled_eps(self): + from executorch.backends.qualcomm._passes.recompose_rms_norm import ( + RecomposeRmsNorm, + ) + + graph = torch.fx.Graph() + input_node = graph.placeholder("input") + square = graph.call_function(torch.ops.aten.pow.Tensor_Scalar, (input_node, 2)) + mean = graph.call_function(torch.ops.aten.mean.dim, (square, [-1], True)) + add = graph.call_function( + torch.ops.aten.add.Tensor, + (mean, 1e-6), + {"alpha": 2}, + ) + inverse_sqrt = graph.call_function( + torch.ops.aten.pow.Tensor_Scalar, (add, -0.5) + ) + outer_mul = graph.call_function( + torch.ops.aten.mul.Tensor, (input_node, inverse_sqrt) + ) + last_mul = graph.call_function( + torch.ops.aten.mul.Tensor, (outer_mul, input_node) + ) + + self.assertFalse( + RecomposeRmsNorm()._is_valid_rms_norm( + (last_mul, None, outer_mul, inverse_sqrt, add, mean, square) + ) + ) + + def test_rms_norm_recomposition_unwraps_literal_scalar(self): + from executorch.backends.qualcomm._passes.recompose_rms_norm import _get_scalar + + graph = torch.fx.Graph() + scalar = graph.call_function( + torch.ops.aten.lift_fresh_copy.default, (torch.tensor(-0.5),) + ) + + self.assertEqual(_get_scalar(scalar), -0.5) + def test_skip_decomp_table_preserves_conv_backward(self): self.assertIn( torch.ops.aten.convolution_backward.default, @@ -1021,20 +1134,30 @@ def forward(self, x): ms = x.pow(2).mean(self.dim, keepdim=self.keepdim) return x * torch.rsqrt(ms + 1e-6) * self.scale - x = torch.randn(32, 32) # A 0-d scale is not a weight, and a mean over dim 0, over all dims, or - # without keepdim is not a last-dim RMSNorm. - for model in ( - ScaledNorm(-1, torch.tensor(2.0)), - ScaledNorm(0, torch.randn(32)), - ScaledNorm(None, torch.randn(32)), - ScaledNorm(-1, torch.randn(32), keepdim=False), + # without keepdim is not a last-dim RMSNorm. A rank-0 input has no last + # dimension to normalize. + x = torch.randn(32, 32) + for model, sample_input in ( + (ScaledNorm(-1, torch.tensor(2.0)), x), + (ScaledNorm(0, torch.randn(32)), x), + (ScaledNorm(None, torch.randn(32)), x), + (ScaledNorm(-1, torch.randn(32), keepdim=False), x), + (ScaledNorm(-1, torch.randn(1)), torch.randn(())), ): with self.subTest(dim=model.dim, keepdim=model.keepdim): - ep = to_edge(torch.export.export(model, (x,), strict=True)) + ep = to_edge(torch.export.export(model, (sample_input,), strict=True)) ep = ep.exported_program() RecomposeRmsNorm()(ep.graph_module) - torch.testing.assert_close(ep.module()(x), model(x)) + self.assertFalse( + any( + node.target == exir_ops.edge.aten.rms_norm.default + for node in ep.graph.nodes + ) + ) + torch.testing.assert_close( + ep.module()(sample_input), model(sample_input) + ) def test_i64_to_i32_casts_a_top_of_graph_constant_after_placeholders(self): # DecomposeRemainder puts the int64 constant 3 at the top of the graph. diff --git a/backends/qualcomm/tests/test_qnn_delegate.py b/backends/qualcomm/tests/test_qnn_delegate.py index 5a7c36c3191..fc1dec50a93 100644 --- a/backends/qualcomm/tests/test_qnn_delegate.py +++ b/backends/qualcomm/tests/test_qnn_delegate.py @@ -2457,6 +2457,15 @@ def test_qnn_backend_rms_norm(self): with self.subTest(i=i): self.lower_module_and_test_output(module, sample_input) + def test_qnn_backend_fp32_rms_norm(self): + if get_backend_type(self.backend) != QnnExecuTorchBackendType.kHtpBackend: + self.skipTest("validates HTP FP32 RMSNorm accumulation") + module = Fp32RmsNorm(16).half() # noqa: F405 + sample_input = ( + torch.linspace(-4096, 4096, 16, dtype=torch.float16).reshape(1, 1, 1, 16), + ) + self.lower_module_and_test_output(module, sample_input) + def test_qnn_backend_roll(self): modules = [ Roll(shifts=(3, 3), dims=(1, 2)), # noqa: F405