# mypy: allow-untyped-defs
import functools
import logging
from collections.abc import Sequence
from typing import Any, cast, Literal

import torch
from torch._dynamo.utils import counters
from torch._inductor.autoheuristic.autoheuristic import AutoHeuristicSelectAlgorithm
from torch._inductor.autoheuristic.autoheuristic_utils import (
    AHContext,
    context_add_strides,
    context_add_using_tf32,
    mm_operations,
)
from torch._inductor.codegen.cpp_gemm_template import CppGemmTemplate
from torch._inductor.remote_gemm_autotune_cache import gen_best_config
from torch._inductor.virtualized import ops, V
from torch.fx.experimental.proxy_tensor import make_fx
from torch.nn.functional import ScalingType, SwizzleType  # type: ignore[attr-defined]
from torch.torch_version import TorchVersion
from torch.utils._ordered_set import OrderedSet

from .. import config as inductor_config, distributed_autotune, lowering as L
from ..codegen.cutlass.gemm_template import CUTLASS2xGemmTemplate, CUTLASS3xGemmTemplate
from ..codegen.flydsl.flydsl_template import FlyDSLTemplate
from ..codegen.rocm.ck_tile_universal_gemm_template import CKTileGemmTemplate
from ..codegen.rocm.ck_universal_gemm_template import CKGemmTemplate
from ..codegen.subgraph import SubgraphChoiceCaller, SubgraphTemplate
from ..codegen.wrapper import PythonWrapperCodegen
from ..fx_utils import get_fake_args_kwargs
from ..ir import (
    Buffer,
    ChoiceCaller,
    ExternKernel,
    FallbackKernel,
    IRNode,
    is_triton,
    is_unaligned,
    Layout,
)
from ..kernel_inputs import MMKernelInputs
from ..lowering import (
    fallback_handler,
    lowerings,
    make_pointwise,
    make_reduction,
    register_lowering,
    transform_args,
)
from ..select_algorithm import (
    autotune_select_algorithm,
    ExternKernelChoice,
    KernelTemplate,
    NoValidChoicesError,
    realize_inputs,
    TritonTemplate,
)
from ..utils import (
    _IntLike,
    _use_autotune_backend,
    _use_cutlass_for_op,
    ceildiv,
    FOLDED_SCALED_MM_OUTPUT_SCALE,
    GPU_ALIGN_BYTES,
    is_bf16x9_matmul,
    use_aten_gemm_kernels,
    use_ck_gemm_template,
    use_ck_tile_gemm_template,
    use_cpp_gemm_template,
    use_cutlass_template,
    use_decompose_k_choice,
    use_flydsl_gemm_template,
    use_nv_universal_gemm_template,
    use_triton_blackwell_tma_template,
    use_triton_scaling_template,
    use_triton_tdm_template,
    use_triton_template,
    use_triton_tma_template,
)
from .mm_common import (
    _fits_int32_buffer_span,
    _is_static_problem,
    _use_small_mm_pointwise,
    blackwell_persistent_mm_grid,
    load_kernel_template,
    mm_args,
    mm_grid,
    persistent_mm_grid,
    use_native_matmul,
)


try:
    import triton

    triton_version = TorchVersion(triton.__version__)
    has_triton = True
except ImportError:
    triton_version = TorchVersion("0.0.0")
    has_triton = False

log = logging.getLogger(__name__)
aten = torch.ops.aten
prims = torch.ops.prims
MXFPFormat = Literal["mxfp4", "mxfp8"]

# We define each template kernel in a separate file which is the name of the input to load_kernel_template
# (e.g. triton_mm for templates/triton_mm.py.jinja).
# If you are adding a new template, please follow that pattern and add a new file with your implementation in the templates folder.
mm_template = TritonTemplate(
    name="mm",
    grid=mm_grid,
    source=load_kernel_template("triton_mm")
    if (torch.version.hip is None) or triton_version >= "3.3.0"
    # FIXME: To get around rocm failures.
    # The only difference between the two templates is M >= BLOCK_M and N >= BLOCK_N checking.
    # See more details in https://github.com/pytorch/pytorch/pull/146293
    else load_kernel_template("triton_mm_rocm"),
    cache_codegen_enabled_for_template=True,
    prologue_loads_all_inputs=True,
)

persistent_tma_mm_template = TritonTemplate(
    name="mm_persistent_tma",
    grid=persistent_mm_grid,
    source=load_kernel_template("triton_persistent_tma_mm"),
)

# AMD TDM uses Triton's stable tensor descriptor branch from the same maintained
# persistent template source, with ROCm-specific options supplied by its heuristic.
persistent_tdm_mm_template = TritonTemplate(
    name="mm_persistent_tdm",
    grid=persistent_mm_grid,
    source=load_kernel_template("triton_persistent_tma_mm"),
)

# Non-TMA Triton template for persistent MM
# used on AMD
persistent_mm_template = TritonTemplate(
    name="mm_persistent",
    grid=persistent_mm_grid,
    source=load_kernel_template("triton_persistent_mm"),
)


scaled_mm_device_tma_epilogue_scaling_template = TritonTemplate(
    name="scaled_mm_device_tma_epilogue_scaling",
    grid=persistent_mm_grid,
    source=load_kernel_template("triton_epilogue_scaled_mm"),
)


scaled_mm_device_tma_main_loop_scaling_template = TritonTemplate(
    name="scaled_mm_device_tma_main_loop_scaling",
    grid=persistent_mm_grid,
    source=load_kernel_template("triton_main_loop_scaled_mm"),
)

flydsl_mm_template = FlyDSLTemplate(
    name="mm_flydsl",
    source=load_kernel_template("flydsl_mm"),
)

blackwell_ws_persistent_tma_mm_template = TritonTemplate(
    name="blackwell_ws_persistent_tma",
    grid=blackwell_persistent_mm_grid,
    source=load_kernel_template("triton_blackwell_ws_persistent_tma_mm"),
)


# prevent duplication registration of extern functions
@functools.cache
def lazy_register_extern_choice(fn):
    return ExternKernelChoice(fn)


aten_mm = ExternKernelChoice(torch.mm, "at::mm_out", op_overload=aten.mm.out)
aten_mm_dtype = ExternKernelChoice(
    torch.mm,
    "at::mm_dtype_out",
    name="mm_dtype",
    op_overload=aten.mm.dtype_out,
)

aten_addmm = ExternKernelChoice(
    torch.addmm, "at::addmm_out", op_overload=aten.addmm.out
)

aten__int_mm = ExternKernelChoice(
    torch._int_mm, "at::_int_mm_out", op_overload=aten._int_mm.out
)

aten__sparse_semi_structured_mm = ExternKernelChoice(
    torch._sparse_semi_structured_mm,
    "at::_sparse_semi_structured_mm",
    has_out_variant=False,
    op_overload=aten._sparse_semi_structured_mm.default,
)

aten__fp8_mm = ExternKernelChoice(
    torch._scaled_mm, "at::_scaled_mm_out", op_overload=aten._scaled_mm.out
)


def scaled_mm_with_output_scale(
    mat_a,
    mat_b,
    scale_a,
    scale_b,
    output_scale,
    *,
    out_dtype,
    use_fast_accum,
    out=None,
):
    result = torch._scaled_mm(
        mat_a,
        mat_b,
        scale_a=scale_a,
        scale_b=scale_b,
        out_dtype=out_dtype,
        use_fast_accum=use_fast_accum,
        out=out,
    )
    return torch.mul(result, output_scale, out=result)


aten__scaled_mm_with_output_scale = ExternKernelChoice(
    scaled_mm_with_output_scale, None
)


def scaled_mm_v2_choice(
    mat_a,
    mat_b,
    scale_a,
    scale_b,
    bias=None,
    *,
    recipe_a,
    recipe_b,
    out_dtype,
    use_fast_accum,
    kernel=aten._scaled_mm_v2.out,
    **kwargs,
):
    """Restore v2's tensor-list schema from flat autotuning inputs."""
    return kernel(
        mat_a,
        mat_b,
        [scale_a],
        [recipe_a],
        [0],
        [scale_b],
        [recipe_b],
        [0],
        bias,
        out_dtype,
        [],
        use_fast_accum,
        **kwargs,
    )


aten__fp8_mm_v2 = ExternKernelChoice(
    scaled_mm_v2_choice,
    name="_scaled_mm_v2",
    kernel_creator=functools.partial(
        scaled_mm_v2_choice,
        kernel=functools.partial(FallbackKernel.create, aten._scaled_mm_v2.default),
    ),
)


def _is_int8_mat(mat):
    return mat.get_dtype() in (torch.int8, torch.uint8)


def bias_addmm(inp, mat1, mat2, *, out=None, alpha=1, beta=1):
    """
    Giving torch.addmm a 1D tensor calls a different (faster) cublasLt
    kernel under the hood.  There are a few shapes where this is slower,
    but they are rare.
    """
    if (inp.stride(0) == 0 and inp.size(0) != 0) or inp.size(0) == 1:
        return torch.addmm(inp[0], mat1, mat2, out=out, alpha=alpha, beta=beta)
    return torch.addmm(inp, mat1, mat2, out=out, alpha=alpha, beta=beta)


def check_supported_striding(mat_a, mat_b) -> None:
    def is_row_major(stride) -> bool:
        return V.graph.sizevars.statically_known_equals(stride[1], 1)

    def is_col_major(stride) -> bool:
        return V.graph.sizevars.statically_known_equals(stride[0], 1)

    def has_zero_dim(size) -> bool:
        return bool(
            V.graph.sizevars.statically_known_equals(size[0], 0)
            or V.graph.sizevars.statically_known_equals(size[1], 0)
        )

    # Check mat_a (self) stride requirements
    torch._check(
        is_row_major(mat_a.get_stride()) or has_zero_dim(mat_a.get_size()),
        lambda: f"mat_a must be row_major, got stride {mat_a.get_stride()}",
    )

    # Check mat_b stride requirements
    torch._check(
        is_col_major(mat_b.get_stride()) or has_zero_dim(mat_b.get_size()),
        lambda: f"mat_b must be col_major, got stride {mat_b.get_stride()}",
    )


def get_flydsl_mm_template_kwargs(
    layout,
    mat1,
    mat2,
    static_shape,
    is_nonzero,
    *,
    mxfp_format=None,
    has_bias=False,
) -> list[dict[str, Any]]:
    """Return shape-compatible FlyDSL GEMM template configurations."""
    # scaled_mm_v2_constraint forces MXFP to TN before lowering; non-TN layout
    # handling is shared with dense mm, but currently unreachable for MXFP.
    from ..heuristics.template.flydsl import (
        get_gemm_configs,
        is_gemm_config_valid_for_shape,
    )

    if not (static_shape and is_nonzero and use_flydsl_gemm_template(layout)):
        return []

    if len(mat1.get_size()) != 2 or len(mat2.get_size()) != 2:
        return []

    sizevars = V.graph.sizevars
    mat1_stride = mat1.get_stride()
    mat2_stride = mat2.get_stride()
    out_stride = layout.stride

    if sizevars.statically_known_equals(mat1_stride[1], 1):
        a_is_transposed = False
    elif sizevars.statically_known_equals(mat1_stride[0], 1):
        a_is_transposed = True
    else:
        return []

    # FlyDSL consumes aten.mm's logical [K, N] RHS view directly.
    if sizevars.statically_known_equals(mat2_stride[0], 1):
        b_is_transposed = True
    elif sizevars.statically_known_equals(mat2_stride[1], 1):
        b_is_transposed = False
    else:
        return []

    if not sizevars.statically_known_equals(out_stride[1], 1):
        return []

    dtype = mat1.get_dtype()
    if mxfp_format is None:
        if mat2.get_dtype() != dtype or layout.dtype != dtype:
            return []
        if dtype not in (torch.float16, torch.bfloat16):
            return []

    a_leading_stride = mat1_stride[1] if a_is_transposed else mat1_stride[0]
    b_leading_stride = mat2_stride[1] if b_is_transposed else mat2_stride[0]

    # Require vectorized tensor origins and row increments to stay GPU-aligned.
    itemsize = dtype.itemsize
    aligned_byte_expressions = (
        mat1.get_layout().offset * itemsize,
        a_leading_stride * itemsize,
        mat2.get_layout().offset * itemsize,
        b_leading_stride * itemsize,
    )
    if (
        is_unaligned(mat1)
        or is_unaligned(mat2)
        or any(
            not sizevars.statically_known_multiple_of(expr, GPU_ALIGN_BYTES)
            for expr in aligned_byte_expressions
        )
    ):
        return []

    m = mat1.get_size()[0]
    _, n = mat2.get_size()
    k = mat1.get_size()[1]
    m_static = PythonWrapperCodegen.statically_known_int_or_none(m)
    n_static = PythonWrapperCodegen.statically_known_int_or_none(n)
    k_static = PythonWrapperCodegen.statically_known_int_or_none(k)
    if m_static is None or n_static is None or k_static is None:
        return []
    if k_static % 32 != 0:
        return []

    tensor_spans = (
        (
            k_static if a_is_transposed else m_static,
            a_leading_stride,
            m_static if a_is_transposed else k_static,
        ),
        (
            n_static if b_is_transposed else k_static,
            b_leading_stride,
            k_static if b_is_transposed else n_static,
        ),
        (m_static, out_stride[0], n_static),
    )
    if any(
        not _fits_int32_buffer_span(
            rows,
            PythonWrapperCodegen.statically_known_int_or_none(stride),
            cols,
            span_itemsize,
        )
        for (rows, stride, cols), span_itemsize in zip(
            tensor_spans, (itemsize, itemsize, layout.dtype.itemsize)
        )
    ):
        return []

    from .vendored_templates.flydsl.kernels import (
        GEMM_DTYPE_BF16,
        GEMM_DTYPE_FP16,
        GEMM_DTYPE_MXFP4,
        GEMM_DTYPE_MXFP8,
    )

    gemm_dtype_id = GEMM_DTYPE_FP16 if dtype == torch.float16 else GEMM_DTYPE_BF16
    extra = {"IS_MXFP": mxfp_format is not None, "HAS_BIAS": has_bias}
    validity = {}
    if mxfp_format is not None:
        gemm_dtype_id = GEMM_DTYPE_MXFP4 if mxfp_format == "mxfp4" else GEMM_DTYPE_MXFP8
        k_static *= 2 if mxfp_format == "mxfp4" else 1
        out_dtype_id = (
            GEMM_DTYPE_FP16 if layout.dtype == torch.float16 else GEMM_DTYPE_BF16
        )
        extra["OUT_DTYPE_ID"] = out_dtype_id
        validity["out_dtype_id"] = out_dtype_id
        validity["has_bias"] = has_bias
    # Filter shape-incompatible configs before autotuning.
    return [
        {
            **gemm_config,
            **extra,
            "GEMM_DTYPE_ID": gemm_dtype_id,
            "GEMM_M": m_static,
            "GEMM_N": n_static,
            "GEMM_K": k_static,
            "A_IS_TRANSPOSED": a_is_transposed,
            "B_IS_TRANSPOSED": b_is_transposed,
        }
        for gemm_config in get_gemm_configs(m_static, n_static, k_static, mxfp_format)
        if is_gemm_config_valid_for_shape(
            m_static,
            n_static,
            k_static,
            gemm_dtype_id,
            gemm_config,
            a_is_transposed=a_is_transposed,
            b_is_transposed=b_is_transposed,
            **validity,
        )
    ]


aten_bias_addmm = ExternKernelChoice(bias_addmm, None)


def _check_addmm_input_metadata(inp, mat1, mat2) -> None:
    torch._check(
        inp.get_dtype() == mat1.get_dtype() and inp.get_dtype() == mat2.get_dtype(),
        lambda: "input dtypes must be the same",
    )
    torch._check(
        inp.get_device() == mat1.get_device() and inp.get_device() == mat2.get_device(),
        lambda: "all inputs must be on the same device",
    )


def decomposeK(a, b, k_splits):
    m = a.shape[0]
    n = b.shape[1]
    k = a.shape[1]

    k_parts = k // k_splits
    B = k_splits
    a_reshaped = torch.permute(a.reshape(m, B, k_parts), (1, 0, 2))
    b_reshaped = b.reshape(B, k_parts, n)
    result = torch.bmm(a_reshaped, b_reshaped, out_dtype=torch.float32)
    reduced_buf = torch.sum(result, 0)
    return reduced_buf.to(a.dtype)


class DecomposeKSugraphTemplate(SubgraphTemplate):
    def __init__(self):
        super().__init__(
            name="decompose_k",
        )

    def generate(  # type: ignore[override]
        self,
        input_nodes: list[Buffer],
        layout: Layout,
        k_split: int,
    ) -> SubgraphChoiceCaller:
        from torch._dispatch.python import enable_python_dispatcher

        from ..decomposition import select_decomp_table

        name = f"decompose_k_mm_{k_split}_split"
        description = f"{k_split=}"

        with enable_python_dispatcher():
            decompositions = select_decomp_table()
            fn = make_fx(
                functools.partial(decomposeK, k_splits=k_split),
                decompositions,
            )

            return super().generate(
                name=name,
                input_nodes=input_nodes,
                layout=layout,
                make_fx_graph=fn,
                description=description,
            )


decompose_k_subgraph_template = DecomposeKSugraphTemplate()


class ContiguousTemplate(SubgraphTemplate):
    def __init__(self, name: str, description: str, fn: Any):
        self.name = name
        self.description = description
        self.fn = fn
        super().__init__(
            name=name,
        )

    def generate(  # type: ignore[override]
        self,
        input_nodes: list[Buffer],
        layout: Layout,
    ) -> SubgraphChoiceCaller:
        from torch._dispatch.python import enable_python_dispatcher

        from ..decomposition import select_decomp_table

        with enable_python_dispatcher():
            decompositions = select_decomp_table()
            fn = make_fx(
                self.fn,
                decompositions,
            )

            return super().generate(
                name=self.name,
                input_nodes=input_nodes,
                layout=layout,
                make_fx_graph=fn,
                description=self.description,
            )


def contiguous_mm(a, b):
    return torch.mm(a, b.contiguous())


def contiguous_addmm(inp, a, b):
    return torch.addmm(inp, a, b.contiguous())


mm_contiguous_subgraph_template = ContiguousTemplate(
    "contiguous_mm", "contiguous mm", contiguous_mm
)
addmm_contiguous_subgraph_template = ContiguousTemplate(
    "contiguous_addmm", "contiguous addmm", contiguous_addmm
)


@register_lowering(aten.mm, type_promotion_kind=None)
def tuned_mm(mat1, mat2, out_dtype=None, *, layout=None):
    """
    Lowering for autotuning aten.mm with different backends (Aten, Triton, CUTLASS, etc.)
    """
    use_bf16x9 = is_bf16x9_matmul(mat1.get_device().type, mat1.get_dtype())
    if out_dtype is not None:
        input_dtype = mat1.get_dtype()
        torch._check(
            mat2.get_dtype() == input_dtype,
            lambda: "input dtypes must be the same",
        )
        torch._check(
            mat1.get_device().type in ("cuda", "xpu"),
            lambda: "out_dtype is only supported for CUDA or XPU",
        )
        torch._check(
            out_dtype == input_dtype
            or (
                out_dtype == torch.float32
                and input_dtype in (torch.float16, torch.bfloat16)
            ),
            lambda: (
                "out_dtype must be the same as input dtype or fp32 for fp16/bf16 inputs"
            ),
        )

    # Lower matmul-related operations (e.g., torch.matmul / torch.bmm / torch.addmm)
    # into native matmul IR using `ops.dot`. When we see a matmul pattern
    # (C[y, x] = A[y, r] * B[r, x]), the core idea is to emulate a broadcasted
    # multiply followed by a sum.
    #
    # For example, given `C = torch.matmul(A, B)`, this can be rewritten as:
    #
    #     Prod = A.unsqueeze(-1) * B.unsqueeze(0)
    #     C = Prod.sum(dim=1)
    #
    # Instead of explicitly using `ops.mul` and `ops.reduction("sum")`, we lower
    # these into `ops.dot` (pointwise) and `ops.reduction("dot")`. These IR nodes
    # are semantically equivalent to the `ops.mul` + `ops.reduction("sum")`
    # combination, but are lowered to `tl.dot` during the code generation phase.
    if use_native_matmul(mat1, mat2):
        mat1 = lowerings[aten.unsqueeze](mat1, -1)
        mat2 = lowerings[aten.unsqueeze](mat2, 0)
        args, kwargs = transform_args(
            args=[mat1, mat2],
            kwargs={},
            broadcast=True,
            type_promotion_kind=None,
            convert_input_to_bool=False,
        )  # Handles broadcasting the arguments

        if inductor_config.triton.codegen_upcast_to_fp32 and mat1.dtype in [
            torch.float16,
            torch.bfloat16,
        ]:

            def _to_dtype(x):
                return ops.to_dtype(x, mat1.dtype, use_compute_types=False)

            args = [make_pointwise(_to_dtype)(x) for x in args]
        mul_pointwise = make_pointwise(ops.dot)(*args)
        dot_reduction = make_reduction("dot")(mul_pointwise, 1)

        return dot_reduction

    # TODO(coconutruben): integrate into MMKernelInputs when all callsites use that
    m, n, k, layout, mat1, mat2 = mm_args(
        mat1, mat2, layout=layout, out_dtype=out_dtype
    )

    if (
        not use_bf16x9
        and out_dtype is None
        and _use_small_mm_pointwise(m, k, n, layout.device.type)
    ):
        counters["inductor"]["decompose_mm_pointwise"] += 1
        mat1 = L.unsqueeze(mat1, -1)
        mat2 = L.unsqueeze(mat2, 0)
        return L.sum_(L.mul(mat1, mat2), axis=1)

    static_shape, is_nonzero = _is_static_problem(layout)
    name = "mm"

    # Create MMKernelInputs for standard MM at the top
    kernel_inputs = MMKernelInputs([mat1, mat2], out_dtype=out_dtype)

    # below is for getting an overview logging info of inductor mms
    counters["aten_mm_info"][f"aten.mm_{m}_{n}_{k}"] += 1
    log.info(
        "Tuned aten.mm: m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, output_layout=%s",
        m,
        n,
        k,
        mat1.get_dtype(),
        mat2.get_dtype(),
        layout,
    )

    choices: list[ChoiceCaller] = []
    static_shape, is_nonzero = _is_static_problem(layout)

    aten_handler: ExternKernelChoice = aten_mm
    aten_extra_kwargs: dict[str, Any] = {}
    if out_dtype is not None:
        aten_handler = aten_mm_dtype
        aten_extra_kwargs = {"out_dtype": out_dtype}

    if use_bf16x9:
        # See Note [BF16x9 precision] in torch/_inductor/utils.py.
        choices.extend(
            V.choices.get_template_configs(
                kernel_inputs,
                [aten_handler],
                name,
                kwarg_overrides={aten_handler.uid: aten_extra_kwargs},
            )
        )
        node, _ = autotune_select_algorithm(
            name, choices, kernel_inputs.nodes(), layout
        )
        return node

    templates_to_use: list[ExternKernelChoice | KernelTemplate] = []
    kwarg_overrides: dict[str, dict[str, Any]] = {}
    if use_aten_gemm_kernels():
        templates_to_use.append(aten_handler)
        if aten_extra_kwargs:
            kwarg_overrides[aten_handler.uid] = aten_extra_kwargs

    if (
        out_dtype is None
        and is_nonzero
        and use_triton_template(layout, check_max_autotune=True)
    ):
        if use_decompose_k_choice(m, n, k):
            templates_to_use.append(decompose_k_subgraph_template)
        # Triton Templates typically perform very poorly for large K.
        # Its highly unlikely that if we want to use decompose_k, then
        # Triton will ever win.
        #
        # To be conservative we increase this threshold for N/M by 2.
        is_exhaustive = inductor_config.max_autotune_gemm_search_space == "EXHAUSTIVE"
        if is_exhaustive or not use_decompose_k_choice(m, n, k, threshold_multiple=2):
            templates_to_use.append(mm_template)

            if use_triton_blackwell_tma_template(
                mat1, mat2, output_layout=layout, add_guards=True
            ):
                templates_to_use.append(blackwell_ws_persistent_tma_mm_template)
            else:
                # TDM is an additional ROCm candidate, not a replacement for the
                # ordinary persistent template. Descriptor block filtering can
                # empty the TDM config pool, and displacing the ordinary
                # candidate would then leave autotuning worse off than with TDM
                # switched off entirely.
                if use_triton_tdm_template(mat1, mat2):
                    templates_to_use.append(persistent_tdm_mm_template)
                if use_triton_tma_template(
                    mat1, mat2, output_layout=layout, add_guards=True
                ):
                    if torch.version.hip is None:
                        templates_to_use.append(persistent_tma_mm_template)
                    else:
                        templates_to_use.append(persistent_mm_template)

        templates_to_use.append(mm_contiguous_subgraph_template)

    choices.extend(
        V.choices.get_template_configs(
            kernel_inputs,
            templates_to_use,
            "mm",
            kwarg_overrides=kwarg_overrides,
        )
    )

    if (
        out_dtype is None
        and is_nonzero
        and use_cutlass_template(layout, m, n, k)
        and _use_cutlass_for_op("mm")
    ):
        CUTLASS3xGemmTemplate.add_cutlass_gemm_choices(
            choices, layout, kernel_inputs.nodes()
        )

    if out_dtype is None and is_nonzero and use_ck_gemm_template(layout, m, n, k):
        CKGemmTemplate.add_ck_gemm_choices(choices, layout, kernel_inputs.nodes())
    if out_dtype is None and is_nonzero and use_ck_tile_gemm_template(layout, m, n, k):
        CKTileGemmTemplate.add_choices(choices, layout, kernel_inputs.nodes())

    if out_dtype is None:
        flydsl_configs = get_flydsl_mm_template_kwargs(
            layout, mat1, mat2, static_shape, is_nonzero
        )
        if flydsl_configs:
            flydsl_input_nodes = list(kernel_inputs.nodes())
            for flydsl_kwargs in flydsl_configs:
                flydsl_mm_template.maybe_append_choice(
                    choices,
                    input_nodes=flydsl_input_nodes,
                    layout=layout,
                    **flydsl_kwargs,
                )

    if (
        out_dtype is None
        and is_nonzero
        and use_nv_universal_gemm_template(layout, m, n, k, mat1, mat2)
    ):
        from ..codegen.nv_universal_gemm import add_nv_universal_gemm_choices

        add_nv_universal_gemm_choices(choices, layout, kernel_inputs)

    if out_dtype is None and use_cpp_gemm_template(layout, mat1, mat2):
        CppGemmTemplate.add_choices(
            choices,
            layout,
            kernel_inputs.nodes(),
        )

    input_nodes = [mat1, mat2]
    if (
        out_dtype is None
        and is_nonzero
        and use_triton_template(layout)
        and torch._inductor.config.run_autoheuristic(name)
        and is_triton(mat1)
    ):
        always_included = []
        if use_aten_gemm_kernels():
            always_included.append("extern_mm")
        num_choices_before_extra_configs = len(choices)
        choices.extend(
            V.choices.get_template_configs(
                # TODO(coconutruben): remove once we deprecate ah
                # mm-extra is a hack to keep the ah functionality alive
                # while we transition to the unified kwargs retrieval
                kernel_inputs,
                [mm_template],
                "mm-ah",
            )
        )

        # using AutoHeuristic for ranking
        ah_choices = mm_autoheuristic(
            mat1,
            mat2,
            m,
            n,
            k,
            choices,
            name,
            input_nodes,
            mm_operations(),
            None,
            top_k=10,
            always_included=always_included,
        )
        if not torch._inductor.config.collect_autoheuristic(name):
            # if we are collecting data, we do not want to modify choices
            if ah_choices is not None and len(ah_choices) > 0:
                # the order in which autoheuristic returns choices is not the same as
                # as the order of choices, which affects things like epilogue fusion.
                # once epilogue fusion benchmarks choices in sorted order, I think we can
                # just use the order returned by autoheuristic
                choices = [choice for choice in choices if choice in ah_choices]
            else:
                choices = choices[:num_choices_before_extra_configs]

    if out_dtype is None:
        for k in inductor_config.external_matmul:
            choices.append(
                lazy_register_extern_choice(k).bind(kernel_inputs.nodes(), layout)
            )

    best_config_future = None
    if out_dtype is None and torch._inductor.config.remote_gemm_autotune_cache:
        # Purposely not awaiting the future here - this kicks off the best config lookup at lowering time
        # The future will be awaited at scheduling time in select_algorithm.py
        best_config_future = gen_best_config(mat1, mat2)

    if box := distributed_autotune.maybe_autotune_remote(
        name, choices, kernel_inputs.nodes(), layout
    ):
        return box

    node, _ = autotune_select_algorithm(
        name,
        choices,
        kernel_inputs.nodes(),
        layout,
        best_config_future=best_config_future,
    )
    return node


@register_lowering(aten._int_mm, type_promotion_kind=None)
def tuned_int_mm(mat1, mat2, *, layout=None):
    # TODO(coconutruben): integrate into MMKernelInputs when all callsites use that
    m, n, k, layout, mat1, mat2 = mm_args(
        mat1, mat2, layout=layout, out_dtype=torch.int32
    )
    name = "int_mm"
    # below is for getting an overview logging info of inductor mms
    counters["aten_mm_info"][f"aten._int_mm_{m}_{n}_{k}"] += 1
    log.info(
        "Tuned aten._int_mm: m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, output_layout=%s",
        m,
        n,
        k,
        mat1.get_dtype(),
        mat2.get_dtype(),
        layout,
    )

    static_shape, is_nonzero = _is_static_problem(layout)
    use_cutlass = static_shape and is_nonzero and use_cutlass_template(layout, m, n, k)
    choices: list[ChoiceCaller] = []

    # Create MMKernelInputs for Int MM
    kernel_inputs = MMKernelInputs([mat1, mat2], out_dtype=torch.int32)

    # Collect all templates for unified call
    templates_to_use: list[ExternKernelChoice | KernelTemplate] = []
    if use_aten_gemm_kernels():
        templates_to_use.append(aten__int_mm)

    if is_nonzero and use_triton_template(
        layout, enable_int32=True, check_max_autotune=False
    ):
        templates_to_use.append(mm_template)

    # Single unified call for all templates
    choices.extend(
        V.choices.get_template_configs(kernel_inputs, templates_to_use, name)
    )

    if use_cutlass and _use_cutlass_for_op(name):
        CUTLASS3xGemmTemplate.add_cutlass_gemm_choices(
            choices, layout, kernel_inputs.nodes(), fuseable=True, non_fuseable=True
        )

    node, _ = autotune_select_algorithm(name, choices, kernel_inputs.nodes(), layout)
    return node


@register_lowering(aten.addmm, type_promotion_kind=None)
def tuned_addmm(inp, mat1, mat2, *, alpha=1, beta=1, layout=None):
    """
    Lowering for autotuning aten.addmm with different backends (Aten, Triton, CUTLASS, etc.)
    """
    use_bf16x9 = is_bf16x9_matmul(mat1.get_device().type, mat1.get_dtype())
    if not use_bf16x9 and beta == 0 and mat1.get_device().type == "cuda":
        _check_addmm_input_metadata(inp, mat1, mat2)
        if alpha == 0:
            _, _, _, layout, mat1, mat2 = mm_args(mat1, mat2, layout=layout)
            return lowerings[aten.full](
                layout.size,
                0,
                dtype=layout.dtype,
                device=layout.device,
            )
        if layout is not None:
            result = lowerings[aten.mm](mat1, mat2, layout=layout)
        else:
            result = lowerings[aten.mm](mat1, mat2)
        if alpha != 1:
            result = lowerings[aten.mul](alpha, result)
        return result

    if use_native_matmul(mat1, mat2):
        if beta == 0:
            _check_addmm_input_metadata(inp, mat1, mat2)
            arg1 = 0
        else:
            arg1 = lowerings[aten.mul](beta, inp)

        if alpha == 0:
            arg2 = 0
        else:
            arg2 = lowerings[aten.mul](alpha, lowerings[aten.mm](mat1, mat2))

        return lowerings[aten.add](arg1, arg2)

    # TODO(coconutruben): integrate into MMKernelInputs when all callsites use that
    m, n, k, layout, mat1, mat2, inp_expanded = mm_args(mat1, mat2, inp, layout=layout)
    inp = realize_inputs(inp)
    static_shape, is_nonzero = _is_static_problem(layout)
    name = "addmm"

    # Create MMKernelInputs for AddMM at the top
    kernel_inputs = MMKernelInputs(
        [inp_expanded, mat1, mat2], scalars=dict(alpha=alpha, beta=beta)
    )
    kernel_inputs_aten = MMKernelInputs(
        [inp, mat1, mat2], scalars=dict(alpha=alpha, beta=beta)
    )

    choices: list[ChoiceCaller] = []

    # below is for getting an overview logging info of inductor mms
    counters["aten_mm_info"][f"aten.addmm_{m}_{n}_{k}"] += 1
    log.info(
        "Tuned aten.addmm: m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, output_layout=%s",
        m,
        n,
        k,
        mat1.get_dtype(),
        mat2.get_dtype(),
        layout,
    )
    if (
        use_bf16x9
        or (not is_nonzero)
        or (not (inductor_config.max_autotune or inductor_config.max_autotune_gemm))
    ):
        choices.extend(
            V.choices.get_template_configs(
                kernel_inputs_aten,
                [aten_addmm],
                name,
            )
        )
        node, _ = autotune_select_algorithm(
            name, choices, kernel_inputs.nodes(), layout
        )
        return node

    templates_to_use: list[ExternKernelChoice | KernelTemplate] = []

    if use_aten_gemm_kernels():
        aten_templates: list[ExternKernelChoice | KernelTemplate] = [aten_addmm]
        if (
            len(inp.get_size()) == 2
            and inp.get_stride()[0] == 0
            and inductor_config.triton.autotune_cublasLt
            and not V.graph.cpp_wrapper  # bias_addmm only has a Python implementation
        ):
            aten_templates.append(aten_bias_addmm)

        # On ROCm, ATen choices use original bias input; non-ROCm keeps unified inputs.
        choices.extend(
            V.choices.get_template_configs(kernel_inputs_aten, aten_templates, name)
        )

    if is_nonzero and use_triton_template(layout, check_max_autotune=False):
        templates_to_use.append(mm_template)

        if use_triton_blackwell_tma_template(
            mat1, mat2, output_layout=layout, add_guards=True
        ):
            templates_to_use.append(blackwell_ws_persistent_tma_mm_template)
        else:
            # See tuned_mm: TDM is additive, it does not displace the ordinary
            # persistent candidate.
            if use_triton_tdm_template(mat1, mat2):
                templates_to_use.append(persistent_tdm_mm_template)
            if use_triton_tma_template(
                mat1, mat2, output_layout=layout, add_guards=True
            ):
                if torch.version.hip is None:
                    templates_to_use.append(persistent_tma_mm_template)
                else:
                    templates_to_use.append(persistent_mm_template)

        # Manually call get_template_configs as use 1-D bias if possible
        choices.extend(
            V.choices.get_template_configs(
                kernel_inputs_aten, [addmm_contiguous_subgraph_template], name
            )
        )
    # Single unified call for all templates
    choices.extend(
        V.choices.get_template_configs(kernel_inputs, templates_to_use, name)
    )

    if (
        is_nonzero
        and use_cutlass_template(layout, m, n, k)
        and _use_cutlass_for_op(name)
    ):
        CUTLASS3xGemmTemplate.add_cutlass_gemm_choices(
            choices,
            layout,
            # reorder here because CUTLASS expects (x, w, bias) but torch
            # is bias, x, w
            kernel_inputs.nodes(reorder=[1, 2, 0]),
            alpha=alpha,
            beta=beta,
            input_reorder=[2, 0, 1],
        )

    if is_nonzero and use_nv_universal_gemm_template(layout, m, n, k, mat1, mat2):
        from ..codegen.nv_universal_gemm import add_nv_universal_addmm_choices

        # inp is the original (un-expanded) bias, so a 1D row vector routes to
        # the row-broadcast epilogue impl.
        add_nv_universal_addmm_choices(
            choices, layout, kernel_inputs, inp, alpha=alpha, beta=beta
        )

    if is_nonzero and use_ck_gemm_template(layout, m, n, k):
        CKGemmTemplate.add_ck_gemm_choices(
            choices,
            layout,
            # reorder here because CK expects (x, w, bias) but torch
            # is bias, x, w
            kernel_inputs.nodes(reorder=[1, 2, 0]),
            alpha=alpha,
            beta=beta,
            input_reorder=[2, 0, 1],
        )

    if use_cpp_gemm_template(layout, mat1, mat2):
        CppGemmTemplate.add_choices(
            choices,
            layout,
            kernel_inputs.nodes(),
            alpha=alpha,
            beta=beta,
            has_bias=True,
        )

    node, _ = autotune_select_algorithm(name, choices, kernel_inputs.nodes(), layout)
    return node


@register_lowering(aten._sparse_semi_structured_mm, type_promotion_kind=None)
def tuned_sparse_semi_structured_mm(
    mat1, mat1_meta, mat2, *, out_dtype=None, layout=None
):
    from torch._inductor.select_algorithm import realize_inputs

    # TODO(coconturuben): support V.choices.get_mm_configs for sparse_semi_structured_mm
    mat1, mat1_meta, mat2 = realize_inputs(mat1, mat1_meta, mat2)
    m1, k1 = mat1.get_size()
    m2, _ = mat1_meta.get_size()
    k2, n = mat2.get_size()
    m = V.graph.sizevars.check_equals_and_simplify(m1, m2)
    k = V.graph.sizevars.check_equals_and_simplify(2 * k1, k2)
    if layout is None:
        from torch._inductor.ir import FixedLayout

        layout = FixedLayout(
            mat2.get_device(),
            out_dtype if out_dtype else mat2.get_dtype(),
            [m, n],
            [n, 1],
        )
    else:
        if out_dtype is not None:
            raise AssertionError("out_dtype is ignored if layout is specified.")

    choices = (
        [
            aten__sparse_semi_structured_mm.bind(
                (mat1, mat1_meta, mat2), layout, out_dtype=out_dtype
            )
        ]
        if use_aten_gemm_kernels()
        else []
    )

    if (
        m * n != 0
        and use_cutlass_template(layout, m, n, k)
        and _use_cutlass_for_op("sparse_semi_structured_mm")
    ):
        CUTLASS2xGemmTemplate.add_cutlass_gemm_choices(
            choices, layout, [mat1, mat2, mat1_meta], fuseable=True, non_fuseable=True
        )

    node, _ = autotune_select_algorithm(
        "sparse_semi_structured_mm", choices, (mat1, mat1_meta, mat2), layout
    )
    return node


scaling_pairs = [
    (ScalingType.TensorWise, ScalingType.TensorWise),
    (ScalingType.RowWise, ScalingType.RowWise),
    (ScalingType.BlockWise1x128, ScalingType.BlockWise128x128),
    (ScalingType.BlockWise1x128, ScalingType.BlockWise1x128),
    (ScalingType.BlockWise128x128, ScalingType.BlockWise1x128),
]


epilogue_scaling_types = [ScalingType.TensorWise, ScalingType.RowWise]
main_loop_scaling_types = [ScalingType.BlockWise1x128, ScalingType.BlockWise128x128]


def _is_tensorwise_scaling(sz: Sequence[_IntLike]) -> bool:
    return (len(sz) == 0) or all(
        V.graph.sizevars.statically_known_equals(d, 1) for d in sz
    )


def _is_rowwise_scaling(sz: Sequence[_IntLike], transpose: bool) -> bool:
    idx = 0 if transpose else -1
    return V.graph.sizevars.statically_known_equals(sz[idx], 1)


def _is_blockwise1xTILESIZE_scaling(
    sz: Sequence[_IntLike],
    tensor_sz: Sequence[_IntLike],
    tile_size: int,
    transpose: bool,
) -> bool:
    # scale_a is [M, ceil(K/tile)]. scale_b is [N, ceil(K/tile)] as the template
    # reads it, or [ceil(K/tile), N] as v1 torch._scaled_mm passes it; this runs
    # on both the raw and the normalized input, so accept either.
    sizevars = V.graph.sizevars
    if not transpose:
        return sizevars.statically_known_equals(
            sz[0], tensor_sz[0]
        ) and sizevars.statically_known_equals(sz[1], ceildiv(tensor_sz[1], tile_size))
    out_dim, k_blocks = tensor_sz[1], ceildiv(tensor_sz[0], tile_size)
    # Template layout [N, ceil(K/tile)] (v2 raw, and v1 after normalization).
    normalized = sizevars.statically_known_equals(
        sz[0], out_dim
    ) and sizevars.statically_known_equals(sz[1], k_blocks)
    # v1 raw layout [ceil(K/tile), N].
    v1_raw = sizevars.statically_known_equals(
        sz[0], k_blocks
    ) and sizevars.statically_known_equals(sz[1], out_dim)
    return normalized or v1_raw


def _blockwise128x128_shape_match(
    sz: Sequence[_IntLike],
    tensor_sz: Sequence[_IntLike],
    transpose: bool,
) -> tuple[bool, bool, _IntLike]:
    # Triton reads [out_blocks, k_blocks]; cuBLAS supplies the transpose of that.
    # _scaled_mm_v2 pads K to a multiple of 4, torch._scaled_mm does not, so both
    # widths are a valid cuBLAS shape. Returns whether sz matches each layout, and
    # the K extent the cuBLAS form implies -- which is also its row stride.
    if transpose:
        out_blocks = ceildiv(tensor_sz[1], 128)
        k_blocks = ceildiv(tensor_sz[0], 128)
    else:
        out_blocks = ceildiv(tensor_sz[0], 128)
        k_blocks = ceildiv(tensor_sz[1], 128)
    k_blocks_padded = ceildiv(k_blocks, 4) * 4
    sizevars = V.graph.sizevars
    triton_ok = sizevars.statically_known_equals(
        sz[0], out_blocks
    ) and sizevars.statically_known_equals(sz[1], k_blocks)
    cublas_out_ok = sizevars.statically_known_equals(sz[1], out_blocks)
    cublas_padded = cublas_out_ok and sizevars.statically_known_equals(
        sz[0], k_blocks_padded
    )
    cublas_unpadded = cublas_out_ok and sizevars.statically_known_equals(
        sz[0], k_blocks
    )
    cublas_k = k_blocks_padded if cublas_padded else k_blocks
    return triton_ok, cublas_padded or cublas_unpadded, cublas_k


def _is_blockwise128x128_scaling(
    sz: Sequence[_IntLike],
    tensor_sz: Sequence[_IntLike],
    transpose: bool = False,
) -> bool:
    triton_ok, cublas_ok, _ = _blockwise128x128_shape_match(sz, tensor_sz, transpose)
    return triton_ok or cublas_ok


def _uses_cublas_blockwise128x128_layout(
    scale: Any,
    mat: Any,
    scale_option: ScalingType,
    transpose: bool = False,
) -> bool:
    if scale_option != ScalingType.BlockWise128x128:
        return False
    triton_layout, cublas_layout, cublas_k = _blockwise128x128_shape_match(
        scale.get_size(), mat.get_size(), transpose
    )
    if not cublas_layout:
        return False
    if not triton_layout:
        return True

    # Both shapes match, so only the strides tell them apart.
    sizevars = V.graph.sizevars
    scale_stride = scale.maybe_get_stride()
    if scale_stride is None:
        return False
    return sizevars.statically_known_equals(
        scale_stride[0], 1
    ) and sizevars.statically_known_equals(scale_stride[1], cublas_k)


def _scale_is_transposed(
    scale: Any,
    mat: Any,
    scale_option: ScalingType,
    transpose: bool = False,
    v1_scale_layout: bool = False,
) -> bool:
    # The template reads a blockwise scale as [out, k_blocks]. cuBLAS supplies the
    # transpose of that (128x128 is also K-padded, which the k_blocks mask clips),
    # and so does torch._scaled_mm for a 1x128 scale_b. The caller passes the ABI
    # because at N == ceil(K/128) the scale is square and the two have one shape.
    if transpose and v1_scale_layout and scale_option == ScalingType.BlockWise1x128:
        return True
    return _uses_cublas_blockwise128x128_layout(scale, mat, scale_option, transpose)


def is_desired_scaling(
    t: IRNode,
    scale_size: Sequence[_IntLike],
    scaling_type: ScalingType,
    transpose: bool = False,
) -> bool:
    match scaling_type:
        case ScalingType.TensorWise:
            return _is_tensorwise_scaling(scale_size)
        case ScalingType.RowWise:
            return _is_rowwise_scaling(scale_size, transpose)
        case ScalingType.BlockWise1x128:
            return _is_blockwise1xTILESIZE_scaling(
                scale_size, t.get_size(), 128, transpose
            )
        case ScalingType.BlockWise128x128:
            return _is_blockwise128x128_scaling(scale_size, t.get_size(), transpose)
        case _:
            raise AssertionError(f"Unsupported scaling type {scaling_type}")


def get_tile_size(scale_option: ScalingType) -> int:
    match scale_option:
        case ScalingType.BlockWise128x128:
            return 128
        case ScalingType.BlockWise1x128:
            return 128
        case _:
            raise AssertionError(
                f"Unsupported scaling type {scale_option} in get_tile_size"
            )


def get_scaling_options(
    mat_a: IRNode,
    mat_b: IRNode,
    scale_a_size: Sequence[_IntLike],
    scale_b_size: Sequence[_IntLike],
) -> tuple[ScalingType, ScalingType]:
    for scale_option_a, scale_option_b in scaling_pairs:
        if is_desired_scaling(
            mat_a, scale_a_size, scale_option_a
        ) and is_desired_scaling(mat_b, scale_b_size, scale_option_b, transpose=True):
            return scale_option_a, scale_option_b

    raise AssertionError(
        f"Inductor Triton does not support scale_a.shape = {scale_a_size}, scale_b.shape = {scale_b_size}"
    )  # verify that shapes are supported by at least one existing pairing


def _flydsl_mxfp_bias_supported(
    bias: Any, mat_b: Any, out_dtype: torch.dtype | None
) -> bool:
    if bias is None:
        return True
    if not isinstance(bias, IRNode):
        return False
    size = bias.get_size()
    if len(size) != 1:
        return False
    if bias.get_device() != mat_b.get_device():
        return False
    # Bias dtype must match the output. The JIT name only records bias presence,
    # so a different bias dtype would reuse the same cached symbol.
    if bias.get_dtype() != out_dtype:
        return False
    return V.graph.sizevars.statically_known_equals(size[0], mat_b.get_size()[-1])


def _get_rocm_mxfp_v2_format(
    mat_a: Any,
    mat_b: Any,
    scale_a: list[Any],
    recipe_a: list[int],
    swizzle_a: list[int],
    scale_b: list[Any],
    recipe_b: list[int],
    swizzle_b: list[int],
    bias: Any,
    out_dtype: torch.dtype | None,
    contraction_dim: list[int] | None,
    use_fast_accum: bool,
) -> MXFPFormat | None:
    common_contract = (
        torch.version.hip is not None
        and len(scale_a) == 1
        and len(scale_b) == 1
        and recipe_a == [ScalingType.BlockWise1x32.value]
        and recipe_b == [ScalingType.BlockWise1x32.value]
        and swizzle_a == [SwizzleType.NO_SWIZZLE.value]
        and swizzle_b == [SwizzleType.NO_SWIZZLE.value]
        and scale_a[0].get_dtype() == torch.float8_e8m0fnu
        and scale_b[0].get_dtype() == torch.float8_e8m0fnu
        and out_dtype in (torch.bfloat16, torch.float16)
        and _flydsl_mxfp_bias_supported(bias, mat_b, out_dtype)
        and not contraction_dim
        and use_fast_accum is False
    )
    if not common_contract:
        return None
    if mat_a.get_dtype() != mat_b.get_dtype():
        return None
    formats: dict[torch.dtype, MXFPFormat] = {
        torch.float8_e4m3fn: "mxfp8",
        torch.float4_e2m1fn_x2: "mxfp4",
    }
    return formats.get(mat_a.get_dtype())


def get_flydsl_mxfp_template_kwargs(
    mxfp_format: MXFPFormat,
    layout: Layout,
    mat_a: Any,
    mat_b: Any,
    scale_a: Any,
    scale_b: Any,
    bias: Any = None,
) -> list[dict[str, Any]]:
    """Return shape-compatible configs for one gfx950 MXFP operand format."""

    if not use_flydsl_gemm_template(layout):
        return []

    nodes = (mat_a, mat_b, scale_a, scale_b)
    if any(
        node.get_device() != layout.device or len(node.get_size()) != 2
        for node in nodes
    ):
        return []
    if is_unaligned(scale_a) or is_unaligned(scale_b):
        return []

    if mxfp_format not in ("mxfp4", "mxfp8"):
        raise AssertionError(f"unsupported MXFP format: {mxfp_format}")
    elements_per_byte = 2 if mxfp_format == "mxfp4" else 1
    expected_dtype = (
        torch.float4_e2m1fn_x2 if mxfp_format == "mxfp4" else torch.float8_e4m3fn
    )

    static_ints = PythonWrapperCodegen.statically_known_list_of_ints_or_none
    a_shape, b_shape, out_shape, out_stride = map(
        static_ints, (mat_a.get_size(), mat_b.get_size(), layout.size, layout.stride)
    )
    if any(value is None for value in (a_shape, b_shape, out_shape, out_stride)):
        return []
    m, k_storage = a_shape
    b_k_storage, n = b_shape
    k = k_storage * elements_per_byte
    if (
        min(m, n, k_storage) <= 0
        or b_k_storage != k_storage
        or k % 128 != 0
        or k > 2**31 - 1
        or out_shape != [m, n]
        or out_stride != [n, 1]
        or (mat_a.get_dtype(), mat_b.get_dtype()) != (expected_dtype, expected_dtype)
        or layout.dtype not in (torch.bfloat16, torch.float16)
    ):
        return []
    for scale, rows in ((scale_a, m), (scale_b, n)):
        if (
            static_ints(scale.get_size()) != [rows, k // 32]
            or static_ints(scale.get_stride()) != [k // 32, 1]
            or scale.get_dtype() != torch.float8_e8m0fnu
            or not _fits_int32_buffer_span(
                rows, k // 32, k // 32, scale.get_dtype().itemsize
            )
        ):
            return []
    static_int = PythonWrapperCodegen.statically_known_int_or_none
    if static_int(layout.offset) != 0 or any(
        static_int(node.get_layout().offset) != 0 for node in nodes
    ):
        return []

    return get_flydsl_mm_template_kwargs(
        layout,
        mat_a,
        mat_b,
        True,
        True,
        mxfp_format=mxfp_format,
        has_bias=bias is not None,
    )


def scaled_mm_v2_constraint(
    fx_node: torch.fx.Node, *args: Any, **kwargs: Any
) -> tuple[tuple[Any, ...], dict[str, Any]]:
    """Construct kernel-compatible layouts instead of preserving eager strides."""
    if not isinstance(fx_node.target, torch._ops.OpOverload):
        raise AssertionError("scaled_mm_v2_constraint expects an OpOverload")
    names = [arg.name for arg in fx_node.target._schema.arguments]
    operands = dict(zip(names, args))
    operands.update(kwargs)

    # The optimized lowering requires row-major A and column-major B. Keep
    # compatible leading dimensions rather than making both matrices dense.
    for name, inner_dim in (("self", 1), ("mat2", 0)):
        matrix = operands[name]
        strides = matrix.maybe_get_stride()
        if strides is None or not V.graph.sizevars.statically_known_equals(
            strides[inner_dim], 1
        ):
            m, n = matrix.get_size()
            strides = (n, 1) if inner_dim == 1 else (1, m)
            operands[name] = ExternKernel.require_exact_strides(matrix, strides)

    device = operands["self"].get_device_or_error().type
    for side in ("a", "b"):
        scales = operands[f"scale_{side}"]
        recipes = operands[f"recipe_{side}"]
        constrained_scales = []
        for index, (scale, recipe) in enumerate(zip(scales, recipes, strict=True)):
            recipe = ScalingType(recipe)
            if (
                device == "cuda"
                and not torch.version.hip
                and recipe in (ScalingType.BlockWise1x128, ScalingType.BlockWise128x128)
            ):
                matrix = operands["self" if side == "a" else "mat2"]
                if recipe == ScalingType.BlockWise128x128 and all(
                    _blockwise128x128_shape_match(
                        scale.get_size(), matrix.get_size(), transpose=side == "b"
                    )[:2]
                ):
                    # Ambiguous scale shapes encode their orientation in strides.
                    # Use FX metadata even when a producer's IR layout is flexible.
                    _, fake_args, fake_kwargs = get_fake_args_kwargs(fx_node)
                    fake_operands = dict(zip(names, fake_args))
                    fake_operands.update(fake_kwargs)
                    fake_scale = fake_operands[f"scale_{side}"][index]
                    if not isinstance(fake_scale, torch.Tensor):
                        raise AssertionError("expected scale tensor metadata")
                    scale = L.constrain_to_fake_tensor(scale, fake_scale)
                else:
                    scale = ExternKernel.require_exact_strides(
                        scale, (1, scale.get_size()[0])
                    )
            elif recipe != ScalingType.TensorWise:
                # Rowwise and packed/swizzled MX/NV scales are dense. XPU
                # consumes row-major DeepSeek scales via oneDNN as well.
                scale = ExternKernel.require_contiguous(scale)
            constrained_scales.append(scale)
        operands[f"scale_{side}"] = constrained_scales

    if operands.get("bias") is not None:
        operands["bias"] = ExternKernel.require_contiguous(operands["bias"])

    # Keep the caller's argument structure for mutation propagation (including out).
    return tuple(operands[name] for name in names[: len(args)]), {
        name: operands[name] for name in kwargs
    }


L.add_layout_constraint(aten._scaled_mm_v2, scaled_mm_v2_constraint)


# Inductor has no template or extern choice that understands swizzled scale
# layouts for _scaled_mm_v2 yet; defer those to the eager op. add_to_fallback_set
# is False because this handler is invoked manually from the lowering below, not
# registered as the op's global fallback.
scaled_mm_v2_fallback = fallback_handler(
    aten._scaled_mm_v2.default, add_to_fallback_set=False
)


@register_lowering(aten._scaled_mm_v2.default, type_promotion_kind=None)
def tuned_scaled_mm_v2(
    mat_a,
    mat_b,
    scale_a: list[Any],
    recipe_a: list[int],
    swizzle_a: list[int],
    scale_b: list[Any],
    recipe_b: list[int],
    swizzle_b: list[int],
    bias=None,
    out_dtype=None,
    contraction_dim=None,
    use_fast_accum=False,
    layout=None,
):
    """
    Performs an optimized matrix multiplication where scaling factors are
    applied to the inputs per the supplied recipes, and optionally swizzled.

    This is the _scaled_mm_v2 API, which takes scale recipes (ScalingType) and
    swizzle patterns alongside the scale tensors, and supports multi-level
    scaling via lists.
    """

    mxfp_format = _get_rocm_mxfp_v2_format(
        mat_a,
        mat_b,
        scale_a,
        recipe_a,
        swizzle_a,
        scale_b,
        recipe_b,
        swizzle_b,
        bias,
        out_dtype,
        contraction_dim,
        use_fast_accum,
    )
    if mxfp_format is not None:
        m, n, k, mxfp_layout, mxfp_a, mxfp_b = mm_args(
            mat_a, mat_b, layout=layout, out_dtype=out_dtype
        )
        mxfp_scale_a, mxfp_scale_b = realize_inputs(scale_a[0], scale_b[0])
        mxfp_nodes = [mxfp_a, mxfp_b, mxfp_scale_a, mxfp_scale_b]
        mxfp_bias = realize_inputs(bias) if bias is not None else None
        if mxfp_bias is not None:
            mxfp_nodes.append(mxfp_bias)
        mxfp_choices: list[ChoiceCaller] = []
        for mxfp_kwargs in get_flydsl_mxfp_template_kwargs(
            mxfp_format,
            mxfp_layout,
            mxfp_a,
            mxfp_b,
            mxfp_scale_a,
            mxfp_scale_b,
            mxfp_bias,
        ):
            flydsl_mm_template.maybe_append_choice(
                mxfp_choices,
                input_nodes=mxfp_nodes,
                layout=mxfp_layout,
                **mxfp_kwargs,
            )
        if mxfp_choices:
            # ATen rejects non-multiples of 32 with ValueError, which autotune
            # does not treat as a skipped choice. k is the packed storage dim.
            aten_mxfp_ok = all(
                V.graph.sizevars.statically_known_multiple_of(dim, 32)
                for dim in (m, n, k)
            )
            if use_aten_gemm_kernels() and aten_mxfp_ok:
                mxfp_choices.insert(
                    0,
                    aten__fp8_mm_v2.bind(
                        mxfp_nodes,
                        mxfp_layout,
                        recipe_a=recipe_a[0],
                        recipe_b=recipe_b[0],
                        out_dtype=out_dtype,
                        use_fast_accum=use_fast_accum,
                    ),
                )
            logical_k = k * (2 if mxfp_format == "mxfp4" else 1)
            counters["aten_mm_info"][
                f"aten._scaled_mm_v2.default_{m}_{n}_{logical_k}"
            ] += 1
            log.info(
                "Tuned FlyDSL MXFP scaled_mm: m=%s, n=%s, k=%s",
                m,
                n,
                logical_k,
            )
            node, _ = autotune_select_algorithm(
                "scaled_mm",
                mxfp_choices,
                mxfp_nodes,
                mxfp_layout,
            )
            return node

    # Inductor only has Triton/extern lowerings for single-level, fp32-scaled,
    # non-swizzled _scaled_mm_v2 with the "supported" recipes (TensorWise,
    # RowWise, and DeepSeek BlockWise1x128/128x128). Everything else has no
    # template or extern choice here, so defer to the eager _scaled_mm_v2 op:
    #   - swizzled scale layouts (e.g. CUDA/Blackwell MXFP8/NVFP4)
    #   - the blockwise MX/NVFP4 recipes BlockWise1x32/1x16 (also how XPU
    #     expresses MX/NVFP4, with NO_SWIZZLE)
    #   - multi-level scales (two-level NVFP4)
    #   - any non-fp32 block scale
    # Call the eager v2 op directly to preserve its recipes and scale conventions.
    def fallback():
        # contraction_dim is a non-optional int[] in the schema (default []);
        # this lowering defaults it to None, so coerce before the eager call.
        fallback_contraction_dim = [] if contraction_dim is None else contraction_dim
        return scaled_mm_v2_fallback(
            mat_a,
            mat_b,
            scale_a,
            recipe_a,
            swizzle_a,
            scale_b,
            recipe_b,
            swizzle_b,
            bias,
            out_dtype,
            fallback_contraction_dim,
            use_fast_accum,
        )

    def check_supported_recipe(recipe: list[int]) -> bool:
        disallowed = OrderedSet([ScalingType.BlockWise1x16, ScalingType.BlockWise1x32])
        return all(ScalingType(r) not in disallowed for r in recipe)

    is_single_level_scale = len(scale_a) == 1 and len(scale_b) == 1
    supported_recipe = check_supported_recipe(recipe_a) and check_supported_recipe(
        recipe_b
    )
    if (
        any(s != 0 for s in swizzle_a)
        or any(s != 0 for s in swizzle_b)
        or not supported_recipe
        or not is_single_level_scale
        or scale_a[0].dtype != torch.float32
    ):
        return fallback()

    if mat_a.get_device().type == "mps":
        return fallback()

    def _is_dynamic(sz) -> bool:
        return PythonWrapperCodegen.statically_known_int_or_none(sz) is None

    if ScalingType(recipe_a[0]) == ScalingType.BlockWise128x128 and (
        _is_dynamic(mat_a.get_size()[0]) or _is_dynamic(mat_a.get_size()[1])
    ):
        return fallback()
    if ScalingType(recipe_b[0]) == ScalingType.BlockWise128x128 and (
        _is_dynamic(mat_b.get_size()[1]) or _is_dynamic(mat_b.get_size()[0])
    ):
        return fallback()

    # TODO(coconutruben): integrate into MMKernelInputs when all callsites use that
    m, n, k, layout, mat_a, mat_b = mm_args(
        mat_a, mat_b, layout=layout, out_dtype=out_dtype
    )
    # below is for getting an overview logging info of inductor mms
    counters["aten_mm_info"][f"aten._scaled_mm_v2.default_{m}_{n}_{k}"] += 1
    log.info(
        "Tuned aten._scaled_mm_v2.default: m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, output_layout=%s",
        m,
        n,
        k,
        mat_a.get_dtype(),
        mat_b.get_dtype(),
        layout,
    )
    name = "scaled_mm"
    check_supported_striding(mat_a, mat_b)

    scale_a_real, scale_b_real = realize_inputs(scale_a[0], scale_b[0])

    # Note: No NVFP4 support at this point - can ignore swizzling, and take only the
    #       first scale types passed.
    scale_option_a, scale_option_b = (
        ScalingType(recipe_a[0]),
        ScalingType(recipe_b[0]),
    )

    bias_real = realize_inputs(bias) if bias else None

    input_nodes: list[Any]

    if not bias:
        input_nodes = [mat_a, mat_b, scale_a_real, scale_b_real]
    else:
        input_nodes = [mat_a, mat_b, scale_a_real, scale_b_real, bias_real]

    # Create MMKernelInputs for Scaled MM (matrices are at indices 0, 1)
    kernel_inputs = MMKernelInputs(
        input_nodes, mat1_idx=0, mat2_idx=1, out_dtype=out_dtype
    )

    choices: list[ChoiceCaller] = []

    # Collect all templates for unified call
    templates_to_use: list[ExternKernelChoice | KernelTemplate] = []
    kwarg_overrides = {}

    if use_aten_gemm_kernels():
        # DeepSeek v2 scale axes/padding differ from the legacy extern schema.
        # Keep the existing v1 choice for compatible tensorwise/rowwise paths.
        extern_kwargs = dict(out_dtype=out_dtype, use_fast_accum=use_fast_accum)
        if (
            scale_option_a in main_loop_scaling_types
            or scale_option_b in main_loop_scaling_types
        ):
            choice = aten__fp8_mm_v2
            extern_kwargs.update(recipe_a=recipe_a[0], recipe_b=recipe_b[0])
        else:
            choice = aten__fp8_mm
        templates_to_use.append(choice)
        kwarg_overrides[choice.uid] = extern_kwargs

    _, is_nonzero = _is_static_problem(layout)

    if (
        # We don't have triton lowerings for the MX variants yet
        is_single_level_scale
        and supported_recipe
        and scale_a[0].dtype == torch.float32
        and is_nonzero
        and use_triton_template(layout, enable_float8=True, check_max_autotune=False)
    ):
        overriders = dict(USE_FAST_ACCUM=use_fast_accum)

        # TODO (paulzhan): There is no template that exists for bias and TMA
        # Don't run tma template currently if bias exist
        if (
            use_triton_tma_template(mat_a, mat_b, output_layout=layout, add_guards=True)
            and not bias
        ):
            overriders["SCALE_RECIPE_A"] = scale_option_a.value
            overriders["SCALE_RECIPE_B"] = scale_option_b.value

            if use_triton_scaling_template(
                scale_option_a, scale_option_b, epilogue_scaling_types
            ):
                templates_to_use.append(scaled_mm_device_tma_epilogue_scaling_template)
                kwarg_overrides[scaled_mm_device_tma_epilogue_scaling_template.uid] = (
                    dict(overriders)
                )
            elif use_triton_scaling_template(
                scale_option_a, scale_option_b, main_loop_scaling_types
            ):
                # Shared inference accepts v1 RHS shapes; v2 loads require v2 extents.
                for scale, mat, outer, recipe, transpose in (
                    (scale_a_real, mat_a, m, scale_option_a, False),
                    (scale_b_real, mat_b, n, scale_option_b, True),
                ):
                    scale_size = scale.get_size()
                    if len(scale_size) != 2:
                        raise RuntimeError("DeepSeek scales must be two-dimensional")
                    if recipe == ScalingType.BlockWise1x128:
                        for actual, expected in zip(
                            scale_size, (outer, ceildiv(k, 128))
                        ):
                            V.graph.sizevars.check_equals(actual, expected)
                    elif not is_desired_scaling(mat, scale_size, recipe, transpose):
                        raise RuntimeError("DeepSeek scale shape does not match recipe")
                overriders["TILE_SIZE_A"] = get_tile_size(scale_option_a)
                overriders["TILE_SIZE_B"] = get_tile_size(scale_option_b)
                overriders["SCALE_A_TRANSPOSED"] = _scale_is_transposed(
                    scale_a_real, mat_a, scale_option_a
                )
                overriders["SCALE_B_TRANSPOSED"] = _scale_is_transposed(
                    scale_b_real,
                    mat_b,
                    scale_option_b,
                    transpose=True,
                    v1_scale_layout=False,
                )

                templates_to_use.append(scaled_mm_device_tma_main_loop_scaling_template)
                kwarg_overrides[scaled_mm_device_tma_main_loop_scaling_template.uid] = (
                    dict(overriders)
                )
            else:
                raise AssertionError(
                    "Inductor Triton does not support scaling options that are present "
                    + "in both epilogue scaling and main loop scaling"
                )

        if (
            use_triton_blackwell_tma_template(
                mat_a, mat_b, output_layout=layout, add_guards=True
            )
            and not bias
        ):
            templates_to_use.append(blackwell_ws_persistent_tma_mm_template)
            kwarg_overrides[blackwell_ws_persistent_tma_mm_template.uid] = dict(
                overriders
            )

        if use_triton_scaling_template(
            scale_option_a, scale_option_b, epilogue_scaling_types
        ):
            templates_to_use.append(mm_template)
            kwarg_overrides[mm_template.uid] = dict(overriders)

    # Single unified call for all templates
    choices.extend(
        V.choices.get_template_configs(
            kernel_inputs,
            templates_to_use,
            name,
            kwarg_overrides=kwarg_overrides,
        )
    )

    # NVGEMM get_kernels() will return empty if the scaling mode/dtype is unsupported
    if is_nonzero and use_nv_universal_gemm_template(layout, m, n, k, mat_a, mat_b):
        from ..codegen.nv_universal_gemm import add_nv_universal_scaled_gemm_choices

        add_nv_universal_scaled_gemm_choices(
            choices,
            layout,
            input_nodes,
            kernel_inputs=kernel_inputs,
        )

    if (
        is_nonzero
        and use_cutlass_template(layout, m, n, k)
        and _use_cutlass_for_op(name)
    ):
        CUTLASS3xGemmTemplate.add_cutlass_gemm_choices(
            choices,
            layout,
            kernel_inputs.nodes(),  # type: ignore[arg-type]
            use_fast_accum=use_fast_accum,  # type: ignore[arg-type]
        )

    if is_nonzero and use_ck_gemm_template(layout, m, n, k):
        CKGemmTemplate.add_ck_gemm_choices(choices, layout, kernel_inputs.nodes())

    node, _ = autotune_select_algorithm(name, choices, kernel_inputs.nodes(), layout)
    return node


@register_lowering(aten._scaled_mm.default, type_promotion_kind=None)  # type: ignore[misc]
def tuned_scaled_mm(
    mat_a,
    mat_b,
    scale_a,
    scale_b,
    bias=None,
    scale_result=None,
    out_dtype=None,
    use_fast_accum=False,
    layout=None,
):
    """
    Performs an optimized matrix multiplication where scaling factors are applied
    to the inputs and/or output.

    Args:
        mat1 (Tensor): First input matrix
        mat2 (Tensor): Second input matrix
        scale1 (Tensor): Scale factor applied to mat1 (supports broadcasting)
        scale2 (Tensor): Scale factor applied to mat2 (supports broadcasting)
        bias (Tensor, optional): Optional bias tensor to add to the result
        layout: Layout hint for optimization

    Returns:
        Tensor: The result of the scaled matrix multiplication
    """
    # TODO(coconutruben): integrate into MMKernelInputs when all callsites use that
    m, n, k, layout, mat_a, mat_b = mm_args(
        mat_a, mat_b, layout=layout, out_dtype=out_dtype
    )
    # below is for getting an overview logging info of inductor mms
    counters["aten_mm_info"][f"aten._scaled_mm.default_{m}_{n}_{k}"] += 1
    log.info(
        "Tuned aten._scaled_mm.default: m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, output_layout=%s",
        m,
        n,
        k,
        mat_a.get_dtype(),
        mat_b.get_dtype(),
        layout,
    )
    name = "scaled_mm"
    check_supported_striding(mat_a, mat_b)

    scale_a_real, scale_b_real = realize_inputs(scale_a, scale_b)
    folded_output_scale = bool(
        V.graph.current_node.meta.get(FOLDED_SCALED_MM_OUTPUT_SCALE, False)
    )
    folded_output_scale_real = (
        realize_inputs(scale_result)
        if folded_output_scale and scale_result is not None
        else None
    )

    def apply_folded_output_scale(node):
        if folded_output_scale_real is None:
            return node
        # The matched scale is a 0-D FP32 tensor, so eager treats it as a
        # wrapped scalar and keeps the matrix result dtype during promotion.
        scale = L.to_dtype(folded_output_scale_real, layout.dtype)
        scale = L.expand(scale, node.get_size())
        return lowerings[aten.mul.Tensor](node, scale)

    bias_real = realize_inputs(bias) if bias else None

    input_nodes: list[Any]

    if not bias:
        input_nodes = [mat_a, mat_b, scale_a_real, scale_b_real]
    else:
        input_nodes = [mat_a, mat_b, scale_a_real, scale_b_real, bias_real]

    # Create MMKernelInputs for Scaled MM (matrices are at indices 0, 1)
    kernel_inputs = MMKernelInputs(
        input_nodes, mat1_idx=0, mat2_idx=1, out_dtype=out_dtype
    )

    choices: list[ChoiceCaller] = []

    # Collect all templates for unified call
    templates_to_use: list[ExternKernelChoice | KernelTemplate] = []
    kwarg_overrides = {}

    if use_aten_gemm_kernels():
        templates_to_use.append(aten__fp8_mm)
        kwarg_overrides[aten__fp8_mm.uid] = dict(
            out_dtype=out_dtype, use_fast_accum=use_fast_accum
        )

    _, is_nonzero = _is_static_problem(layout)

    # The vendored Blackwell block-scaled NVGEMM kernel has a native FP32
    # output-scale argument. Prefer that semantic path only for the private
    # graph-level ``_scaled_mm(...) * scalar`` rewrite. A public ``scale_result``
    # argument has different ATen semantics and must not be treated as alpha.
    if (
        folded_output_scale_real is not None
        and is_nonzero
        and _use_autotune_backend("NVGEMM")
        and use_nv_universal_gemm_template(layout, m, n, k, mat_a, mat_b)
    ):
        from ..codegen.nv_universal_gemm import (
            add_nv_universal_scaled_gemm_choices,
            NVUniversalGemmCaller,
        )

        scaled_choices: list[ChoiceCaller] = []
        add_nv_universal_scaled_gemm_choices(
            scaled_choices,
            layout,
            input_nodes,
            kernel_inputs=kernel_inputs,
            output_scale_node=folded_output_scale_real,
        )
        scaled_input_nodes = [*input_nodes, folded_output_scale_real]
        if scaled_choices and use_aten_gemm_kernels():
            native_request = cast(NVUniversalGemmCaller, scaled_choices[0]).bmreq
            aten__scaled_mm_with_output_scale.maybe_append_choice(
                scaled_choices,
                input_nodes=scaled_input_nodes,
                layout=layout,
                out_dtype=out_dtype,
                use_fast_accum=use_fast_accum,
                benchmark_request_kwargs={
                    "cudagraph_unroll": native_request.cudagraph_unroll,
                    "cudagraph_cold_cache_input_indices": (
                        native_request.cudagraph_cold_cache_input_indices
                    ),
                },
            )
        if scaled_choices:
            try:
                node, _ = autotune_select_algorithm(
                    name, scaled_choices, scaled_input_nodes, layout
                )
                return node
            except NoValidChoicesError:
                pass

    if (
        # We don't have triton lowerings for the MX variants yet
        scale_a.dtype == torch.float32
        and is_nonzero
        and use_triton_template(layout, enable_float8=True, check_max_autotune=False)
    ):
        overriders = dict(USE_FAST_ACCUM=use_fast_accum)

        scale_a_size, scale_b_size = scale_a_real.shape, scale_b_real.shape

        scale_option_a, scale_option_b = get_scaling_options(
            mat_a, mat_b, scale_a_size, scale_b_size
        )

        # TODO (paulzhan): There is no template that exists for bias and TMA
        # Don't run tma template currently if bias exist
        if (
            use_triton_tma_template(mat_a, mat_b, output_layout=layout, add_guards=True)
            and not bias
        ):
            overriders["SCALE_RECIPE_A"] = scale_option_a.value
            overriders["SCALE_RECIPE_B"] = scale_option_b.value

            if use_triton_scaling_template(
                scale_option_a, scale_option_b, epilogue_scaling_types
            ):
                templates_to_use.append(scaled_mm_device_tma_epilogue_scaling_template)
                kwarg_overrides[scaled_mm_device_tma_epilogue_scaling_template.uid] = (
                    dict(overriders)
                )
            elif use_triton_scaling_template(
                scale_option_a, scale_option_b, main_loop_scaling_types
            ):
                overriders["TILE_SIZE_A"] = get_tile_size(scale_option_a)
                overriders["TILE_SIZE_B"] = get_tile_size(scale_option_b)
                overriders["SCALE_A_TRANSPOSED"] = _scale_is_transposed(
                    scale_a_real, mat_a, scale_option_a
                )
                overriders["SCALE_B_TRANSPOSED"] = _scale_is_transposed(
                    scale_b_real,
                    mat_b,
                    scale_option_b,
                    transpose=True,
                    v1_scale_layout=True,
                )

                templates_to_use.append(scaled_mm_device_tma_main_loop_scaling_template)
                kwarg_overrides[scaled_mm_device_tma_main_loop_scaling_template.uid] = (
                    dict(overriders)
                )
            else:
                raise AssertionError(
                    "Inductor Triton does not support scaling options that are present "
                    + "in both epilogue scaling and main loop scaling"
                )

        if (
            use_triton_blackwell_tma_template(
                mat_a, mat_b, output_layout=layout, add_guards=True
            )
            and not bias
        ):
            templates_to_use.append(blackwell_ws_persistent_tma_mm_template)
            kwarg_overrides[blackwell_ws_persistent_tma_mm_template.uid] = dict(
                overriders
            )

        if use_triton_scaling_template(
            scale_option_a, scale_option_b, epilogue_scaling_types
        ):
            templates_to_use.append(mm_template)
            kwarg_overrides[mm_template.uid] = dict(overriders)

    # Single unified call for all templates
    choices.extend(
        V.choices.get_template_configs(
            kernel_inputs,
            templates_to_use,
            name,
            kwarg_overrides=kwarg_overrides,
        )
    )

    # NVGEMM get_kernels() will return empty if the scaling mode/dtype is unsupported
    if is_nonzero and use_nv_universal_gemm_template(layout, m, n, k, mat_a, mat_b):
        from ..codegen.nv_universal_gemm import add_nv_universal_scaled_gemm_choices

        add_nv_universal_scaled_gemm_choices(
            choices,
            layout,
            input_nodes,
            kernel_inputs=kernel_inputs,
        )

    # Early return for MX variants
    if scale_a.dtype != torch.float32:
        node, _ = autotune_select_algorithm(name, choices, input_nodes, layout)
        return apply_folded_output_scale(node)

    if (
        is_nonzero
        and use_cutlass_template(layout, m, n, k)
        and _use_cutlass_for_op(name)
    ):
        CUTLASS3xGemmTemplate.add_cutlass_gemm_choices(
            choices,
            layout,
            kernel_inputs.nodes(),  # type: ignore[arg-type]
            use_fast_accum=use_fast_accum,  # type: ignore[arg-type]
        )

    if is_nonzero and use_ck_gemm_template(layout, m, n, k):
        CKGemmTemplate.add_ck_gemm_choices(choices, layout, kernel_inputs.nodes())

    node, _ = autotune_select_algorithm(name, choices, kernel_inputs.nodes(), layout)
    return apply_folded_output_scale(node)


@functools.cache
def _is_sm7x_or_older_gpu(index: int | None) -> bool:
    props = torch.cuda.get_device_properties(index or 0)
    return props.major <= 7


def dims_are_int(dims):
    return all(isinstance(dim, int) for dim in dims)


def mm_autoheuristic(
    mat1,
    mat2,
    m,
    n,
    k,
    choices,
    name,
    input_nodes,
    ops,
    precondition,
    top_k: int | None = None,
    always_included=None,
):
    m, n, k = get_size_hints(mat1, mat2, m, n, k)
    if not dims_are_int([m, n, k]):
        return None
    mat1_stride, mat2_stride = get_size_hints_strides(mat1, mat2)

    def get_context(m, k, n, mat1, mat2, mat1_stride, mat2_stride):
        context = AHContext()
        context.add_feature("m", m)
        context.add_feature("k", k)
        context.add_feature("n", n)
        context.add_feature("mat1_dtype", mat1.layout.dtype, is_categorical=True)
        context.add_feature("mat2_dtype", mat2.layout.dtype, is_categorical=True)
        context_add_strides(context, "mat1", mat1_stride)
        context_add_strides(context, "mat2", mat2_stride)
        context.add_feature(
            "mat1_iscontig", mat1.layout.is_contiguous(), is_categorical=True
        )
        context.add_feature(
            "mat2_iscontig", mat2.layout.is_contiguous(), is_categorical=True
        )
        if name == "mm":
            context_add_using_tf32(context, mat1.layout.dtype)
        return context

    def fallback():
        return None

    context = get_context(m, k, n, mat1, mat2, mat1_stride, mat2_stride)
    autoheuristic = AutoHeuristicSelectAlgorithm(
        fallback=fallback,
        choices=choices,
        input_nodes=input_nodes,
        context=context,
        name=name,
        augment_context=ops,
        precondition=precondition,
    )

    if top_k is not None:
        # TODO: is there a cleaner way to ensure aten.mm is always included?
        return autoheuristic.get_top_k_choices_caller(
            top_k, always_included=always_included
        )

    return autoheuristic.get_choice_caller()


def get_size_hints(mat1, mat2, m, n, k):
    if not isinstance(m, int) or not isinstance(k, int):
        (m, k) = V.graph.sizevars.optimization_hints(mat1.get_size())

    if not isinstance(n, int) or not isinstance(k, int):
        (k, n) = V.graph.sizevars.optimization_hints(mat2.get_size())
    return m, n, k


def get_size_hints_strides(mat1, mat2):
    mat1_stride = mat1.layout.stride
    mat2_stride = mat2.layout.stride
    strides = [mat1_stride, mat2_stride]
    strides_hints = []
    for stride in strides:
        if not isinstance(stride, int):
            stride = V.graph.sizevars.optimization_hints(stride)
        strides_hints.append(stride)
    return strides_hints[0], strides_hints[1]
