# Copyright (c) 2025-2026, QuACK team.

from typing import Any, Optional, Type, Tuple, Callable, Sequence
from functools import partial

import cutlass
import cutlass.cute as cute
import cutlass.utils.blackwell_helpers as sm100_utils

from cutlass import Int32, Int16, Boolean, const_expr
from cutlass.base_dsl.arch import Arch
from cutlass.cute.nvgpu import cpasync, tcgen05, warp
from cutlass.cute.nvgpu.tcgen05.mma import CtaGroup  # noqa
from cutlass.cutlass_dsl import dsl_user_op
from cutlass.utils import LayoutEnum, block_copy
import cutlass.pipeline
from cutlass._mlir import ir
from cutlass._mlir.dialects import cute_nvgpu as _cute_nvgpu_ir

from torch._vendor.quack import layout_utils
from torch._vendor.quack.utils import make_vector


Sm100MmaPeerBitMask = 0xFEFFFFFF
_TCGEN05_TMEM_OPS = (
    tcgen05.Ld16x128bOp,
    tcgen05.Ld16x256bOp,
    tcgen05.Ld16x32bx2Op,
    tcgen05.Ld16x64bOp,
    tcgen05.Ld32x32bOp,
    tcgen05.LdRed16x32bx2Op,
    tcgen05.LdRed32x32bOp,
    tcgen05.St16x128bOp,
    tcgen05.St16x256bOp,
    tcgen05.St16x32bx2Op,
    tcgen05.St16x64bOp,
    tcgen05.St32x32bOp,
)
_TCGEN05_TMEM_STORE_OPS = (
    tcgen05.St16x128bOp,
    tcgen05.St16x256bOp,
    tcgen05.St16x32bx2Op,
    tcgen05.St16x64bOp,
    tcgen05.St32x32bOp,
)


def tmem_store_atom_from_load_atom(
    copy_atom_t2r: Any,
    src_dtype: Type[cutlass.Numeric],
    dst_dtype: Type[cutlass.Numeric],
) -> cute.CopyAtom:
    """Return the matching tcgen05 R2T store atom for a selected T2R load atom.

    `src_dtype` is the register fragment dtype loaded by T2R; `dst_dtype` is
    the TMEM element dtype to store. Ratio 1 uses CUTLASS's operation-family
    mapping directly. Ratio 2 is intentionally narrow: we allow the current
    Ld32x32b path by halving repeat, and the widest 16dp path by halving the
    vector width. Narrower 16dp cross-family mappings are not mirrored by
    CUTLASS's same-family helper, so they assert until validated.

    C++ CuTe's operation-family mapping is `cute::TMEM::tmem_load_to_store`:
    https://github.com/NVIDIA/cutlass/blob/main/include/cute/atom/copy_traits_sm100.hpp#L3274
    """
    load_op = copy_atom_t2r.op if const_expr(hasattr(copy_atom_t2r, "op")) else copy_atom_t2r
    if const_expr(hasattr(load_op, "op")):
        load_op = load_op.op
    assert src_dtype.width >= dst_dtype.width, "TMEM R2T helper only supports narrowing stores"
    assert src_dtype.width % dst_dtype.width == 0, "TMEM source/destination widths must divide"
    ratio = src_dtype.width // dst_dtype.width
    assert ratio in (1, 2), "TMEM R2T helper only supports src/dst width ratio 1 or 2"
    repeat = load_op.repeat
    unpack = tcgen05.Unpack.NONE
    if const_expr(getattr(load_op, "pack", None) == tcgen05.Pack.PACK_16b_IN_32b):
        unpack = tcgen05.Unpack.UNPACK_32b_IN_16b
    if const_expr(isinstance(load_op, tcgen05.Ld16x64bOp)):
        assert ratio == 1, "No validated ratio-2 store mapping for Ld16x64bOp"
        store_op = tcgen05.St16x64bOp(repeat, unpack)
    elif const_expr(isinstance(load_op, tcgen05.Ld16x128bOp)):
        assert ratio == 1, "No validated ratio-2 store mapping for Ld16x128bOp"
        store_op = tcgen05.St16x128bOp(repeat, unpack)
    elif const_expr(isinstance(load_op, tcgen05.Ld16x256bOp)):
        store_op = (
            tcgen05.St16x256bOp(repeat, unpack)
            if const_expr(ratio == 1)
            else tcgen05.St16x128bOp(repeat, unpack)
        )
    elif const_expr(isinstance(load_op, tcgen05.Ld16x32bx2Op)):
        assert ratio == 1, "No validated ratio-2 store mapping for Ld16x32bx2Op"
        store_op = tcgen05.St16x32bx2Op(repeat, unpack)
    elif const_expr(isinstance(load_op, tcgen05.Ld32x32bOp)):
        if const_expr(ratio == 2):
            assert repeat.value % 2 == 0, "Ld32x32b ratio-2 store needs even repeat"
            repeat = tcgen05.Repetition(repeat.value // 2)
        store_op = tcgen05.St32x32bOp(repeat, unpack)
    else:
        raise TypeError(f"Unsupported TMEM load op for store conversion: {type(load_op)}")
    return cute.make_copy_atom(store_op, dst_dtype)


def _tmem_copy_reg_tv_layout(tiled_copy: cute.TiledCopy):
    """Return the register-side TV layout for a tcgen05 tmem copy."""
    op = tiled_copy.op
    if const_expr(hasattr(op, "op")):
        op = op.op
    if const_expr(isinstance(op, _TCGEN05_TMEM_OPS)):
        # TMEM stores read from registers; all other TMEM copy ops here write
        # registers, including LdRed* reductions that upstream is_tmem_load
        # intentionally does not classify as plain loads.
        return (
            tiled_copy.layout_src_tv_tiled
            if const_expr(isinstance(op, _TCGEN05_TMEM_STORE_OPS))
            else tiled_copy.layout_dst_tv_tiled
        )
    raise TypeError(f"Cannot infer tmem copy direction from tiled_copy.op={op}")


@dsl_user_op
def cvt_copy(
    tiled_copy: cute.TiledCopy,
    src: cute.Tensor,
    dst: cute.Tensor,
    *,
    pred: Optional[cute.Tensor] = None,
    retile: bool = False,
    loc=None,
    ip=None,
    **kwargs,
) -> None:
    assert isinstance(src.iterator, cute.Pointer) and src.memspace == cute.AddressSpace.rmem
    if const_expr(src.element_type != dst.element_type):
        src = src.to(dst.element_type, loc=loc, ip=ip)
    if const_expr(retile):
        src = tiled_copy.retile(src)
    cute.copy(tiled_copy, src, dst, pred=pred, loc=loc, ip=ip, **kwargs)


def _pair_leaf_slices(layout):
    """Slice coords for the two slots of the contiguous (shape 2, stride 1) leaf.

    Also asserts the pair is the fastest nontrivial mode (all earlier modes
    size 1), so the flat colex value order is (slot, rest...).

    Why not flat_divide by 2 like the acc_pair idiom in epilogue/visit.py:
    that works there because compact rmem fragments have the stride-1 pair as
    mode 0 by construction. Here the split also applies to the smem-side
    partition, whose flattened layout carries degenerate leading modes from
    the copy-atom nesting (e.g. (1,2,2,2,1,2):(0,1,256,8,0,16)) — dividing
    mode 0 would fail on the 1:0, and coalescing first can restructure the
    compact src differently from the strided dst, breaking the mode-for-mode
    slot correspondence the caller's select relies on. This helper is the
    same divide with its preconditions made explicit and checked."""
    pair_mode = None
    for i in range(cute.rank(layout)):
        if const_expr(layout[i].shape == 2 and layout[i].stride == 1):
            pair_mode = i
            break
    assert pair_mode is not None, "no contiguous pair leaf; use a plain copy instead"
    assert all(layout[i].shape == 1 for i in range(pair_mode)), (
        "pair leaf must be the fastest nontrivial mode"
    )
    coord0 = tuple(0 if i == pair_mode else None for i in range(cute.rank(layout)))
    coord1 = tuple(1 if i == pair_mode else None for i in range(cute.rank(layout)))
    return coord0, coord1


@dsl_user_op
def cvt_copy_pair_xor_sts32(
    src: cute.Tensor,
    dst: cute.Tensor,
    tidx: Int32,
    *,
    loc=None,
    ip=None,
) -> None:
    """Store 32-bit acc fragments as two conflict-free full-warp STS.32.

    The acc layout gives lane L the contiguous column pair (2q, 2q+1),
    q = L%4, of row L>>2. Under every TMA-legal swizzle the pair's bank is
    4*((q>>1)^r) + 2*(q&1) + slot: the vectorized STS.64 spends the slot bit
    on vector width, leaving 16 reachable banks per half-warp (uniformly
    2-way conflicted, 4 wavefronts/instr measured). Two STS.32 that store
    the pair in per-lane order slot = h, 1-h with h = (L>>1)&1 make the bank
    a bijection on all 32 lanes: 2x1 conflict-free wavefronts instead. Each
    thread still writes only its own two values to the same cells, so the
    smem image (and TMA) is unchanged; the reorder costs one select per
    stored word on the otherwise-idle ALU pipe.

    Costs a handful of extra registers (w fragments + second address base);
    only use in epilogues with register headroom — gemm_add's fp32-C
    epilogue sits exactly at the setmaxnreg cap and spills.
    """
    from cutlass.cute.tensor import TensorSSA

    assert isinstance(src.iterator, cute.Pointer) and src.memspace == cute.AddressSpace.rmem
    if const_expr(src.element_type != dst.element_type):
        src = src.to(dst.element_type, loc=loc, ip=ip)
    assert const_expr(dst.element_type.width == 32)
    src_flat = cute.make_tensor(src.iterator, cute.flatten(src.layout))
    dst_flat = cute.make_tensor(dst.iterator, cute.flatten(dst.layout))
    assert const_expr(src_flat.layout.shape == dst_flat.layout.shape)
    coord0, coord1 = _pair_leaf_slices(dst_flat.layout)
    v0, v1 = src_flat[coord0].load(), src_flat[coord1].load()
    h = (tidx >> 1) & 1
    hb = h != 0
    w0 = cute.make_rmem_tensor(v0.shape, dst.element_type)
    w1 = cute.make_rmem_tensor(v1.shape, dst.element_type)
    w0.store(TensorSSA(cutlass.select_(hb, v1, v0), v0.shape, dst.element_type))  # slot h
    w1.store(TensorSSA(cutlass.select_(hb, v0, v1), v1.shape, dst.element_type))  # slot 1 - h
    dst0 = dst_flat[coord0]
    # dst leaves are stride >= 2 after slicing off the pair, so these can
    # never re-vectorize into a 64-bit store
    cute.autovec_copy(w0, cute.make_tensor(dst0.iterator + h, dst0.layout))
    cute.autovec_copy(w1, cute.make_tensor(dst0.iterator + (1 - h), dst0.layout))


@dsl_user_op
def sr_cvt_copy(
    tiled_copy: cute.TiledCopy,
    src: cute.Tensor,
    dst: cute.Tensor,
    seed: Int32,
    tidx: Int32,
    *,
    retile: bool = False,
    loc=None,
    ip=None,
) -> None:
    """Like cvt_copy but uses stochastic rounding for FP32 -> BF16/FP16 conversion."""
    assert isinstance(src.iterator, cute.Pointer) and src.memspace == cute.AddressSpace.rmem
    from torch._vendor.quack.rounding import convert_f32_to_bf16_sr, convert_f32_to_f16_sr
    from cutlass.cute.tensor import TensorSSA

    assert const_expr(dst.element_type in (cutlass.BFloat16, cutlass.Float16)), (
        "stochastic rounding supports BF16/FP16 output only"
    )
    convert_sr = (
        convert_f32_to_bf16_sr
        if const_expr(dst.element_type == cutlass.BFloat16)
        else convert_f32_to_f16_sr
    )
    src_cvt = cute.make_rmem_tensor_like(src, dst.element_type)
    src_vec = src.load()
    raw_vec = convert_sr(src_vec, seed, tidx, loc=loc, ip=ip)
    src_cvt.store(TensorSSA(raw_vec, src_vec.shape, dst.element_type))
    src = src_cvt
    if const_expr(retile):
        src = tiled_copy.retile(src)
    cute.copy(tiled_copy, src, dst, loc=loc, ip=ip)


@dsl_user_op
def load_s2r(src: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor:
    dst = cute.make_rmem_tensor_like(src, src.element_type, loc=loc, ip=ip)
    cute.autovec_copy(src, dst, loc=loc, ip=ip)
    return dst


@dsl_user_op
def contiguous(src: cute.Tensor, *, loc=None, ip=None) -> cute.Tensor:
    dst = cute.make_rmem_tensor(src.shape, src.element_type, loc=loc, ip=ip)
    cute.autovec_copy(src, dst, loc=loc, ip=ip)
    return dst


@dsl_user_op
def load_s2r_retile(
    tiled_copy: cute.TiledCopy,
    src: cute.Tensor,
    dst_shape: cute.Tensor | cute.Shape,
    *,
    loc=None,
    ip=None,
) -> cute.Tensor:
    # Will also accept dst_shape being a tensor, in which case we write into that tensor
    if const_expr(not isinstance(dst_shape, cute.Tensor)):
        dst = cute.make_rmem_tensor(dst_shape, src.element_type, loc=loc, ip=ip)
    else:
        dst = dst_shape
    cute.copy(tiled_copy, src, tiled_copy.retile(dst), loc=loc, ip=ip)
    return dst


@dsl_user_op
def load_t2r(
    tiled_copy: cute.TiledCopy,
    src: cute.Tensor,
    *,
    fence: bool = False,
    loc=None,
    ip=None,
) -> cute.Tensor:
    """Load one tmem tile partition into rmem, deriving the rmem shape from `src`.

    `src` should already be indexed to the tile being copied, with any
    stage/subtile modes removed.
    """
    dst = tmem_reg_frag(tiled_copy, src, loc=loc, ip=ip)
    cute.copy(tiled_copy, src, dst, loc=loc, ip=ip)
    if const_expr(fence):
        cute.arch.fence_view_async_tmem_load()
    return dst


@dsl_user_op
def get_copy_atom(
    dtype: Type[cutlass.Numeric], num_copy_elems: int, is_async: bool = False, *, loc=None, ip=None
) -> cute.CopyAtom:
    num_copy_bits = const_expr(min(128, num_copy_elems * dtype.width))
    copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp()
    return cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits)


@dsl_user_op
def copy(
    src: cute.Tensor,
    dst: cute.Tensor,
    *,
    pred: Optional[cute.Tensor] = None,
    is_async: bool = False,
    loc=None,
    ip=None,
    **kwargs,
) -> None:
    num_copy_elems = src.shape[0][0]
    copy_atom = get_copy_atom(src.element_type, num_copy_elems, is_async)
    cute.copy(copy_atom, src, dst, pred=pred, loc=loc, ip=ip, **kwargs)


def tiled_copy_1d(
    dtype: Type[cutlass.Numeric], num_threads: int, num_copy_elems: int = 1, is_async: bool = False
) -> cute.TiledCopy:
    num_copy_bits = num_copy_elems * dtype.width
    copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp()
    copy_atom = cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits)
    thr_layout = cute.make_layout(num_threads)
    val_layout = cute.make_layout(num_copy_elems)
    return cute.make_tiled_copy_tv(copy_atom, thr_layout, val_layout)


def vectorized_thread_partition(
    tensor: cute.Tensor,
    tidx: Int32,
    num_threads: int,
    num_copy_elems: int = 1,
    *,
    is_source: bool,
) -> Tuple[cute.ThrCopy, cute.Tensor]:
    """Partition a flat tensor into adjacent per-thread vectors.

    At a fixed vector iteration, thread ``t`` owns
    ``[t * num_copy_elems, (t + 1) * num_copy_elems)``. Returns the thread
    copy together with its source or destination partition of ``tensor``.
    """
    thr_copy = tiled_copy_1d(tensor.element_type, num_threads, num_copy_elems).get_slice(tidx)
    partition = (
        thr_copy.partition_S(tensor) if const_expr(is_source) else thr_copy.partition_D(tensor)
    )
    return thr_copy, partition


def tiled_copy_2d(
    dtype: Type[cutlass.Numeric],
    threads_per_row: int,
    num_threads: int,
    num_copy_elems: int = 1,
    is_async: bool = False,
) -> cute.TiledCopy:
    num_copy_bits = num_copy_elems * dtype.width
    copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp()
    copy_atom = cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits)
    assert num_threads % threads_per_row == 0
    thr_layout = cute.make_ordered_layout(
        (num_threads // threads_per_row, threads_per_row),
        order=(1, 0),
    )
    val_layout = cute.make_layout((1, num_copy_elems))
    return cute.make_tiled_copy_tv(copy_atom, thr_layout, val_layout)


@cute.jit
def predicate_k(tAcA: cute.Tensor, limit: Int32) -> cute.Tensor:
    # Only compute predicates for the "k" dimension. For the mn dimension, we will use "if"
    tApA = cute.make_rmem_tensor(
        cute.make_layout(
            (cute.size(tAcA, mode=[0, 1]), cute.size(tAcA, mode=[1]), cute.size(tAcA, mode=[2])),
            stride=(cute.size(tAcA, mode=[2]), 0, 1),
        ),
        Boolean,
    )
    for rest_v in cutlass.range_constexpr(tApA.shape[0]):
        for rest_k in cutlass.range_constexpr(tApA.shape[2]):
            tApA[rest_v, 0, rest_k] = cute.elem_less(tAcA[(0, rest_v), 0, rest_k][1], limit)
    return tApA


# def tiled_copy_2d(
#     dtype: Type[cutlass.Numeric], major_mode_size: int, num_threads: int, is_async: bool = False
# ) -> cute.TiledCopy:
#     num_copy_bits = math.gcd(major_mode_size, 128 // dtype.width) * dtype.width
#     copy_elems = num_copy_bits // dtype.width
#     copy_op = cpasync.CopyG2SOp() if is_async else cute.nvgpu.CopyUniversalOp()
#     copy_atom = cute.make_copy_atom(copy_op, dtype, num_bits_per_copy=num_copy_bits)
#     gmem_threads_per_row = major_mode_size // copy_elems
#     assert num_threads % gmem_threads_per_row == 0
#     thr_layout = cute.make_ordered_layout(
#         (num_threads // gmem_threads_per_row, gmem_threads_per_row),
#         order=(1, 0),
#     )
#     val_layout = cute.make_layout((1, copy_elems))
#     return cute.make_tiled_copy_tv(copy_atom, thr_layout, val_layout)


# Ragged tensor trick for TMA: encodes variable-length sequences into a higher-rank
# tensor so that TMA's out-of-bounds checking handles sequence boundaries.
#
# Given a tensor T with a ragged dimension (variable-length across batches), we create
# a higher-rank tensor where the ragged dim is replaced with a fixed size `big_int`, and
# extra dim(s) are appended. When indexing into a specific sequence at (offset, length),
# `offset_ragged_tensor` computes coordinates such that:
#   ragged_coord = big_int - length   (OOB check clamps reads past the sequence end)
#   extra_coord(s) = f(offset, length) (selects the correct memory region)
#
# ptr_shift=True: 1-extra-dim approach (adds 1 dim, supports up to 4D input):
#   Shape:  (*before, big_int, *after, max_int)
#   Stride: (*original_strides, stride_r)     where stride_r = T.stride[ragged_dim]
#   Pointer shifted backward by big_int * stride_r elements.
#   Address for coords (big_int - length) in ragged dim, (offset + length) in extra dim:
#     addr = (base - big_int * s_r) + (big_int - length) * s_r + (offset + length) * s_r
#          = base + offset * s_r                                                      [correct]
#   Works for epilogue TMA store. Does NOT work for TMA load with large big_int
#   — the shifted pointer must land in physically mapped GPU memory.
#
# ptr_shift=False: 2-extra-dim approach (adds 2 dims, supports up to 3D input):
#   Shape:  (*before, big_int, *after, max_int, max_int)
#   Stride: (*before_strides, stride_r, *after_strides, 2^34 - stride_r, stride_r)
#   No pointer shift. Uses 64-bit address wraparound to cancel the ragged offset.
#   Let W = 2^34 - stride_r. Address for coords (big_int - length) in ragged dim,
#   big_int in extra dim 0, (offset + length) in extra dim 1:
#     addr = base + (big_int - length) * s_r + big_int * W + (offset + length) * s_r
#          = base + big_int * (s_r + W) - length * s_r + (offset + length) * s_r
#          = base + big_int * 2^34 + offset * s_r
#   Since big_int = 2^30: big_int * 2^34 = 2^64 ≡ 0 (mod 2^64), so:
#     addr = base + offset * s_r                                                      [correct]
#   Works for all TMA paths since the base pointer is never shifted.
#
# Ragged tensor was adapted from the implementation from Triton, but here we have an option that
# only needs 1 extra dimension instead of 2.
# https://github.com/triton-lang/triton/blob/main/python/triton/tools/ragged_tma.py
BIG_INT = 2**30
MAX_INT = 2**31 - 1
BIG_INT_INV = 2**64 // BIG_INT


@dsl_user_op
def create_ragged_tensor_for_tma(
    T: cute.Tensor,
    ragged_dim: int = 0,
    ptr_shift: bool = False,
    *,
    loc=None,
    ip=None,
) -> cute.Tensor:
    rank = cute.rank(T)
    if ragged_dim < 0:
        ragged_dim += rank
    if ptr_shift:
        assert rank <= 4, "ptr_shift ragged tensor only supports up to 4 dimensions"
        new_shape = T.shape[:ragged_dim] + (BIG_INT,) + T.shape[ragged_dim + 1 :] + (MAX_INT,)
        new_stride = T.stride + (T.stride[ragged_dim],)
        ptr_offset = (None,) * ragged_dim + (-BIG_INT,) + (None,) * (rank - ragged_dim - 1)
        new_ptr = cute.domain_offset(ptr_offset, T).iterator
        return cute.make_tensor(new_ptr, cute.make_layout(new_shape, stride=new_stride))
    else:
        assert rank <= 3, "non-ptr_shift ragged tensor only supports up to 3 dimensions"
        stride_r = T.stride[ragged_dim]
        new_shape = (
            T.shape[:ragged_dim] + (BIG_INT,) + T.shape[ragged_dim + 1 :] + (MAX_INT, MAX_INT)
        )
        new_stride = (
            T.stride[:ragged_dim]
            + (stride_r,)
            + T.stride[ragged_dim + 1 :]
            + (BIG_INT_INV - stride_r, stride_r)
        )
        return cute.make_tensor(T.iterator, cute.make_layout(new_shape, stride=new_stride))


@dsl_user_op
def offset_ragged_tensor(
    T: cute.Tensor,
    offset: Int32,
    length: Int32,
    ragged_dim: int = 0,
    ptr_shift: bool = False,
    *,
    loc=None,
    ip=None,
) -> cute.Tensor:
    rank = cute.rank(T)
    if ragged_dim < 0:
        ragged_dim += rank
    big_int = cute.size(T, mode=[ragged_dim])
    offset_val = big_int - length
    if ptr_shift:
        # 1-extra-dim: rank = original_rank + 1
        assert rank >= ragged_dim + 2
        offset_tuple = (None,) * ragged_dim + (offset_val,) + (None,) * (rank - ragged_dim - 2)
        index_tuple = (None,) * (rank - 1) + (offset + length,)
    else:
        # 2-extra-dim: rank = original_rank + 2, last 2 modes are the wraparound dims
        assert rank >= ragged_dim + 3
        offset_tuple = (None,) * ragged_dim + (offset_val,) + (None,) * (rank - ragged_dim - 3)
        index_tuple = (None,) * (rank - 2) + (big_int, offset + length)
    return cute.domain_offset(offset_tuple, T[index_tuple])


def swizzle_int(ptr_int: Int32, b: int, m: int, s: int) -> Int32:
    bit_msk = (1 << b) - 1
    yyy_msk = bit_msk << (m + s)
    return ptr_int ^ ((ptr_int & yyy_msk) >> s)


def swizzle_ptr(ptr: cute.Pointer):
    swz = ptr.type.swizzle_type
    ptr_int = swizzle_int(ptr.toint(), swz.num_bits, swz.num_base, swz.num_shift)
    return cute.make_ptr(ptr.dtype, ptr_int, ptr.memspace, assumed_align=ptr.alignment)


def as_position_independent_swizzle_tensor(tensor: cute.Tensor) -> cute.Tensor:
    outer = tensor.layout
    width = tensor.element_type.width
    swizzle_type = tensor.iterator.type.swizzle_type
    inner = cute.make_swizzle(swizzle_type.num_bits, swizzle_type.num_base, swizzle_type.num_shift)
    # Need to recast the swizzle from byte (e.g. <3, 4, 3> to element units (e.g. <3, 3, 3> for
    # for 16 bits and <3, 2, 3> for 32 bits)
    new_layout = cute.recast_layout(
        width, 8, cute.make_composed_layout(inner, 0, cute.recast_layout(8, width, outer))
    )
    # recast_ptr to remove the pointer swizzle
    return cute.make_tensor(cute.recast_ptr(tensor.iterator, dtype=tensor.element_type), new_layout)


def partition_D_position_independent(thr_copy: cute.ThrCopy, tensor: cute.Tensor) -> cute.Tensor:
    return cute.make_tensor(
        swizzle_ptr(thr_copy.partition_D(tensor).iterator),
        thr_copy.partition_D(as_position_independent_swizzle_tensor(tensor)).layout,
    )


def partition_S_position_independent(thr_copy: cute.ThrCopy, tensor: cute.Tensor) -> cute.Tensor:
    return cute.make_tensor(
        swizzle_ptr(thr_copy.partition_S(tensor).iterator),
        thr_copy.partition_S(as_position_independent_swizzle_tensor(tensor)).layout,
    )


@dsl_user_op
def sm90_get_smem_load_op(
    layout_c: cutlass.utils.LayoutEnum,
    elem_ty_c: Type[cutlass.Numeric],
    *,
    loc=None,
    ip=None,
) -> cute.CopyAtom:
    """
    Selects the largest vectorized smem load atom available subject to constraint of gmem layout.

    Parameters:
    -----------
    layout_c : LayoutEnum
        The layout enum of the output tensor D.

    elem_ty_c : Type[Numeric]
        The element type for output tensor D.

    Returns:
    --------
    Either SmemLoadMatrix or SimtSyncCopy, based on the input parameters.
    """

    if not isinstance(elem_ty_c, cutlass.cutlass_dsl.NumericMeta):
        raise TypeError(f"elem_ty_c must be a Numeric, but got {elem_ty_c}")
    is_m_major = layout_c.is_m_major_c()
    if elem_ty_c.width == 16:
        return cute.make_copy_atom(warp.LdMatrix8x8x16bOp(is_m_major, 4), elem_ty_c, loc=loc, ip=ip)
    else:
        return cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), elem_ty_c, loc=loc, ip=ip)


def get_smem_store_atom(
    element_type: Type[cute.Numeric],
    transpose: bool = False,
    major_mode_size: Optional[int] = None,
) -> cute.CopyAtom:
    arch = cutlass.base_dsl.BaseDSL._get_dsl().get_arch_enum()
    if const_expr(arch < Arch.sm_90 or element_type.width != 16):
        return cute.make_copy_atom(
            cute.nvgpu.CopyUniversalOp(),
            element_type,
            num_bits_per_copy=(2 if not transpose else 1) * element_type.width,
        )
    else:
        num_matrices = (
            4
            if major_mode_size is None or major_mode_size % 16 == 0
            else (2 if major_mode_size % 8 == 0 else 1)
        )
        return cute.make_copy_atom(
            warp.StMatrix8x8x16bOp(transpose=transpose, num_matrices=num_matrices),
            element_type,
        )


def get_smem_load_atom(
    element_type: Type[cute.Numeric],
    transpose: bool = False,
    major_mode_size: Optional[int] = None,
) -> cute.CopyAtom:
    arch = cutlass.base_dsl.BaseDSL._get_dsl().get_arch_enum()
    if const_expr(arch < Arch.sm_90 or element_type.width != 16):
        return cute.make_copy_atom(
            cute.nvgpu.CopyUniversalOp(),
            element_type,
            num_bits_per_copy=(2 if not transpose else 1) * element_type.width,
        )
    else:
        num_matrices = (
            4
            if major_mode_size is None or major_mode_size % 16 == 0
            else (2 if major_mode_size % 8 == 0 else 1)
        )
        return cute.make_copy_atom(
            warp.LdMatrix8x8x16bOp(transpose=transpose, num_matrices=num_matrices),
            element_type,
        )


def get_smem_store_C(
    tiled_mma: cute.TiledMma | cute.TiledCopy,
    sC: cute.Tensor,
    tidx: Int32,
    transpose: bool = False,
    position_independent=False,
    major_mode_size: Optional[int] = None,
) -> Tuple[Callable, cute.TiledCopy, cute.Tensor]:
    dtype = sC.element_type
    if const_expr(isinstance(tiled_mma, cute.TiledCopy)):
        tiled_copy_t2r = tiled_mma
        layout = LayoutEnum.COL_MAJOR if const_expr(transpose) else LayoutEnum.ROW_MAJOR
        copy_atom = sm100_utils.get_smem_store_op(
            layout, dtype, tiled_copy_t2r.value_type, tiled_copy_t2r
        )
        tiled_copy = cute.make_tiled_copy_D(copy_atom, tiled_copy_t2r)
    else:
        copy_atom = get_smem_store_atom(dtype, transpose, major_mode_size=major_mode_size)
        tiled_copy = cute.make_tiled_copy_C(copy_atom, tiled_mma)
    thr_copy = tiled_copy.get_slice(tidx)
    if const_expr(not position_independent):
        tRS_sC = thr_copy.partition_D(sC)
    else:
        tRS_sC = partition_D_position_independent(thr_copy, sC)

    def copy_fn(src: cute.Tensor, dst_idx: Optional[Int32] = None, fence=False, **new_kwargs):
        dst_tensor = tRS_sC if const_expr(dst_idx is None) else tRS_sC[..., dst_idx]
        cvt_copy(tiled_copy, src, dst_tensor, retile=True, **new_kwargs)
        if const_expr(fence):
            cute.arch.fence_view_async_shared()

    return copy_fn, thr_copy, tRS_sC


def get_smem_load_C(
    tiled_mma: cute.TiledMma,
    sC: cute.Tensor,
    tidx: Int32,
    transpose: bool = False,
    position_independent=False,
) -> Tuple[Callable, cute.TiledCopy, cute.Tensor]:
    dtype = sC.element_type
    copy_atom = get_smem_load_atom(dtype, transpose)
    tiled_copy = cute.make_tiled_copy_C(copy_atom, tiled_mma)
    thr_copy = tiled_copy.get_slice(tidx)
    if const_expr(not position_independent):
        tSR_sC = thr_copy.partition_S(sC)
    else:
        tSR_sC = partition_S_position_independent(thr_copy, sC)
    copy_atom_RS = get_smem_store_atom(dtype, transpose)
    thr_copy_RS = cute.make_tiled_copy_C(copy_atom_RS, tiled_mma).get_slice(tidx)
    tRS_shape = thr_copy_RS.partition_S(cute.make_identity_tensor(sC.shape[:2])).shape

    def copy_fn(src_idx: Optional[Int32] = None, **new_kwargs):
        src_tensor = tSR_sC if const_expr(src_idx is None) else tSR_sC[None, None, None, src_idx]
        return load_s2r_retile(tiled_copy, src_tensor, dst_shape=tRS_shape, **new_kwargs)

    return copy_fn, thr_copy, tSR_sC


def epilog_smem_copy_atom(
    tiled_mma: cute.TiledMma, epi_tile: cute.Shape, transpose: bool = False
) -> cute.TiledCopy:
    arch = cutlass.base_dsl.BaseDSL._get_dsl().get_arch_enum()
    if const_expr(arch < Arch.sm_90):
        copy_atom_C = cute.make_copy_atom(
            cute.nvgpu.CopyUniversalOp(),
            cutlass.Float16,  # this is just to get the right source layout
            num_bits_per_copy=(2 if not transpose else 1) * cutlass.Float16.width,
        )
    else:
        copy_atom_C = cute.make_copy_atom(
            warp.StMatrix8x8x16bOp(transpose, num_matrices=4 if epi_tile[1] % 16 == 0 else 2),
            cutlass.Float16,  # this is just to get the right source layout
        )
    tiled_copy_C_atom = cute.make_tiled_copy_C_atom(copy_atom_C, tiled_mma)
    return tiled_copy_C_atom


def get_smem_store_epi(
    tiled_mma: cute.TiledMma,
    epi_tile: cute.Shape,
    sC: Optional[cute.Tensor],
    tidx: Int32,
    transpose: bool = False,
    position_independent=False,
) -> Tuple[Callable, cute.TiledCopy, cute.Tensor, cute.Tensor]:
    dtype = sC.element_type if const_expr(sC is not None) else cutlass.Float16
    copy_atom = get_smem_store_atom(dtype, transpose)
    tiled_copy_C_atom = epilog_smem_copy_atom(tiled_mma, epi_tile)
    tiled_copy = cute.make_tiled_copy_S(copy_atom, tiled_copy_C_atom)
    thr_copy = tiled_copy.get_slice(tidx)
    tRS_sC = None
    if const_expr(sC is not None):
        if const_expr(not position_independent):
            tRS_sC = thr_copy.partition_D(sC)
        else:
            tRS_sC = partition_D_position_independent(thr_copy, sC)
    sC_shape = sC.shape[:2] if sC is not None else epi_tile
    # (R2S, R2S_M, R2S_N, PIPE_C)
    tRS_rC_shape = thr_copy.partition_S(cute.make_identity_tensor(sC_shape)).shape
    tRS_rC = cute.make_rmem_tensor(tRS_rC_shape, tiled_mma.op.acc_dtype)

    def copy_fn(src: cute.Tensor, dst_idx: Int32, **new_kwargs):
        cvt_copy(tiled_copy, src, tRS_sC[None, None, None, dst_idx], **new_kwargs)

    return copy_fn if const_expr(sC is not None) else None, thr_copy, tRS_sC, tRS_rC


def get_smem_store_A(
    tiled_mma: cute.TiledMma, sA: cute.Tensor, tidx: Int32, position_independent=False
) -> Tuple[Callable, cute.TiledCopy, cute.Tensor]:
    dtype = sA.element_type
    transpose = tiled_mma.op.a_major_mode == cute.nvgpu.OperandMajorMode.MN
    copy_atom = get_smem_store_atom(dtype, transpose)
    tiled_copy = cute.make_tiled_copy_A(copy_atom, tiled_mma)
    thr_copy = tiled_copy.get_slice(tidx)
    if const_expr(not position_independent):
        tRS_sA = thr_copy.partition_D(sA)
    else:
        tRS_sA = partition_D_position_independent(thr_copy, sA)

    def copy_fn(src: cute.Tensor, dst_idx: Int32, **new_kwargs):
        cvt_copy(tiled_copy, src, tRS_sA[None, None, None, dst_idx], retile=True, **new_kwargs)

    return copy_fn, thr_copy, tRS_sA


def get_smem_load_A(
    tiled_mma: cute.TiledMma,
    sA: cute.Tensor,
    tidx: Int32,
    with_dst_tensor: bool = False,
    position_independent=False,
) -> Tuple[Callable, cute.TiledCopy, cute.Tensor]:
    dtype = sA.element_type
    transpose = tiled_mma.op.a_major_mode == cute.nvgpu.OperandMajorMode.MN
    copy_atom = get_smem_load_atom(dtype, transpose)
    tiled_copy = cute.make_tiled_copy_A(copy_atom, tiled_mma)
    thr_copy = tiled_copy.get_slice(tidx)
    if const_expr(not position_independent):
        tSR_sA = thr_copy.partition_S(sA)
    else:
        tSR_sA = partition_S_position_independent(thr_copy, sA)
    tRS_shape = tiled_mma.partition_shape_A(sA.shape[:2])

    def copy_fn(src_idx: Int32, **new_kwargs):
        return load_s2r_retile(
            tiled_copy, tSR_sA[None, None, None, src_idx], dst_shape=tRS_shape, **new_kwargs
        )

    def copy_fn_w_dst_tensor(src_idx: Int32, dst: cute.Tensor, **new_kwargs):
        return load_s2r_retile(tiled_copy, tSR_sA[None, None, None, src_idx], dst, **new_kwargs)

    return copy_fn if not with_dst_tensor else copy_fn_w_dst_tensor, thr_copy, tSR_sA


def _cpasync_reduction_kind_name(reduction_kind: Any) -> str:
    name = (
        reduction_kind.lower() if isinstance(reduction_kind, str) else reduction_kind.name.lower()
    )
    assert name in {"add", "min", "max", "inc", "dec", "and", "or", "xor"}, (
        f"Unsupported cp.reduce.async.bulk reduction kind: {reduction_kind}"
    )
    return name


def _cpasync_bulk_reduce_suffix(
    reduction_kind: Any,
    dtype: Type[cutlass.Numeric],
) -> str:
    op = _cpasync_reduction_kind_name(reduction_kind)
    if dtype is cutlass.Float16:
        assert op in {"add", "min", "max"}, f"{op} is not supported for f16 bulk reduce"
        return f"{op}.noftz.f16" if op == "add" else f"{op}.f16"
    if dtype is cutlass.BFloat16:
        assert op in {"add", "min", "max"}, f"{op} is not supported for bf16 bulk reduce"
        return f"{op}.noftz.bf16" if op == "add" else f"{op}.bf16"
    if dtype is cutlass.Float32:
        assert op == "add", f"{op} is not supported for f32 bulk reduce"
        return "add.f32"
    if dtype is cutlass.Float64:
        assert op == "add", f"{op} is not supported for f64 bulk reduce"
        return "add.f64"

    signed = getattr(dtype, "signed", None)
    width = getattr(dtype, "width", None)
    if signed is not None:
        assert width in (32, 64), f"Unsupported integer bulk-reduce width: {width}"
        if op in {"and", "or", "xor"}:
            return f"{op}.b{width}"
        if op in {"min", "max", "add"}:
            return f"{op}.{'s' if signed else 'u'}{width}"
        assert op in {"inc", "dec"} and dtype is cutlass.Uint32, (
            f"{op} bulk reduce is only supported for u32"
        )
        return f"{op}.u32"

    raise TypeError(f"Unsupported cp.reduce.async.bulk dtype: {dtype}")


@dsl_user_op
def cpasync_bulk_s2g(
    smem_ptr: cute.Pointer,
    gmem_ptr: cute.Pointer,
    store_bytes: int | Int32,
    *,
    reduction_kind: Optional[Any] = None,
    dtype: Optional[Type[cutlass.Numeric]] = None,
    loc=None,
    ip=None,
):
    smem_ptr_i32 = smem_ptr.toint(loc=loc, ip=ip)
    if reduction_kind is None:
        ptx = "cp.async.bulk.global.shared::cta.bulk_group [{$r0}], [{$r1}], {$r2};"
    else:
        assert dtype is not None, "dtype is required for cp.reduce.async.bulk"
        ptx = (
            "cp.reduce.async.bulk.global.shared::cta.bulk_group."
            f"{_cpasync_bulk_reduce_suffix(reduction_kind, dtype)} "
            "[{$r0}], [{$r1}], {$r2};"
        )
    cute.arch.inline_ptx(
        ptx,
        read_only_args=[gmem_ptr.llvm_ptr, smem_ptr_i32, Int32(store_bytes)],
        loc=loc,
        ip=ip,
    )


@dsl_user_op
def cpasync_reduce_bulk_add_f32(
    smem_ptr: cute.Pointer,
    gmem_ptr: cute.Pointer,
    store_bytes: int | Int32,
    *,
    loc=None,
    ip=None,
):
    cpasync_bulk_s2g(
        smem_ptr,
        gmem_ptr,
        store_bytes,
        reduction_kind=cpasync.ReductionOp.ADD,
        dtype=cutlass.Float32,
        loc=loc,
        ip=ip,
    )


@dsl_user_op
def get_tma_desc_addr(tma_atom: cute.CopyAtom, *, loc=None, ip=None) -> cute.Pointer:
    """
    Get the address of the TMA descriptor embedded in a TMA Copy Atom.

    Extracts the constant memory address of the TMA descriptor for use with
    custom PTX instructions.

    :param tma_atom: TMA Copy Atom from make_tiled_tma_atom
    :return: Pointer to TMA descriptor in constant memory

    Example:
        >>> desc_ptr = get_tma_descriptor_address(tma_atom)
    """
    exec_atom = _cute_nvgpu_ir.atom_make_exec_tma(tma_atom._trait.value, loc=loc, ip=ip)
    tma_desc_ptr_type = ir.Type.parse(
        "!cute.ptr<!cute_nvgpu.tma_descriptor_tiled, generic, align<128>>"
    )
    return _cute_nvgpu_ir.get_tma_desc_addr(tma_desc_ptr_type, exec_atom, loc=loc, ip=ip)


@dsl_user_op
def tma_gather4_load(
    tma_desc_ptr: cute.Pointer,
    dst_smem_ptr: cute.Pointer,
    mbarrier_ptr: cute.Pointer,
    col_idx: Int32,
    row_indices: Sequence[Int32],
    *,
    num_cta: int = 1,
    multicast_mask=None,
    loc=None,
    ip=None,
) -> None:
    """
    Perform TMA gather4 load from global memory to shared memory.

    Issues PTX instruction:
    cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes
        [dstMem], [tensorMap, {col_idx, row0, row1, row2, row3}], [smem_bar];

    This loads 4 rows (specified by row_indices) from a 2D tensor at the given
    column index into shared memory, using the TMA descriptor.

    :param tma_desc_ptr: Pointer to TMA descriptor in constant memory (128-byte aligned)
    :type tma_desc_ptr:  Pointer
    :param dst_smem_ptr: Destination address in shared memory
    :type dst_smem_ptr:  Pointer
    :param mbarrier_ptr: Pointer to mbarrier in shared memory for completion tracking
    :type mbarrier_ptr:  Pointer
    :param col_idx:      Column index
    :type col_idx:       Int32
    :param row_indices:  Sequence of exactly 4 row indices
    :type row_indices:   Sequence[Int32]
    :param num_cta:      Number of CTAs participating (default: 1)
    :type num_cta:       int
    :param multicast_mask: Optional multicast mask
    :type multicast_mask: Int16

    Requirements:
        - row_indices must contain exactly 4 elements
        - Compute capability >= SM_100 (Blackwell)
        - TMA descriptor must be properly initialized for 2D tensor

    Example:
        >>> from cutlass.cute.nvgpu import cpasync
        >>> from cutlass.cute import core
        >>>
        >>> # Create TMA descriptor
        >>> tma_atom, tma_tensor = cpasync.make_tiled_tma_atom(...)
        >>> tma_desc_ptr = get_tma_descriptor_address(tma_atom)
        >>>
        >>> # Compute indices (typically from kernel logic)
        >>> col_idx = core.get(...) or 5  # Int32 value
        >>> row_indices = [core.get(...) for _ in range(4)]  # 4 Int32 values
        >>>
        >>> # Gather 4 rows at computed column
        >>> tma_gather4_load(
        ...     tma_desc_ptr=tma_desc_ptr,
        ...     dst_smem_ptr=smem_ptr,
        ...     mbarrier_ptr=barrier_ptr,
        ...     col_idx=col_idx,
        ...     row_indices=row_indices
        ... )
    """
    if len(row_indices) != 4:
        raise ValueError(f"gather4 requires exactly 4 row indices, got {len(row_indices)}")
    col_val = Int32(col_idx)
    row_vals = [Int32(row_idx) for row_idx in row_indices]
    # Convert pointers to integer addresses
    desc_addr = tma_desc_ptr.toint(loc=loc, ip=ip)
    dst_addr = dst_smem_ptr.toint(loc=loc, ip=ip)
    mbar_addr = mbarrier_ptr.toint(loc=loc, ip=ip)
    if num_cta > 1:
        # Executed by both CTAs. Set peer bit to 0 so that the
        # transaction bytes will update CTA0's barrier.
        mbar_addr = mbar_addr & Sm100MmaPeerBitMask
    mbar_addr = Int32(mbar_addr)
    # Handle multicast_mask - may already be ir.Value or Python int
    multicast_mask_val = None
    if multicast_mask is not None:
        multicast_mask_val = Int16(multicast_mask)
    assert multicast_mask_val is None, "multicast is not supported yet"
    # Emit inline PTX for TMA gather4
    # PTX: cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes
    #      [dstMem], [tensorMap, {col, row0, row1, row2, row3}], [smem_bar];
    ptx = (
        "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes."
        f"cta_group::{num_cta} "
        "[{$r0}], [{$r1}, {{$r2}, {$r3}, {$r4}, {$r5}, {$r6}}], [{$r7}];"
    )

    cute.arch.inline_ptx(
        ptx,
        read_only_args=[
            dst_addr,
            desc_addr,
            col_val,
            row_vals[0],
            row_vals[1],
            row_vals[2],
            row_vals[3],
            mbar_addr,
        ],
        loc=loc,
        ip=ip,
    )


def cpasync_bulk_get_copy_fn(
    src_tensor: cute.Tensor,
    dst_tensor: cute.Tensor,
    single_stage: bool = False,
    reduction_kind: Optional[cute.nvgpu.cpasync.ReductionKind] = None,
    **kwargs,
) -> Callable:
    src_is_smem = const_expr(
        isinstance(src_tensor.iterator, cute.Pointer)
        and src_tensor.memspace == cute.AddressSpace.smem
    )
    dst_is_smem = const_expr(
        isinstance(dst_tensor.iterator, cute.Pointer)
        and dst_tensor.memspace == cute.AddressSpace.smem
    )
    if const_expr(reduction_kind is not None):
        assert src_is_smem and not dst_is_smem, "cp.reduce.async.bulk only supports SMEM -> GMEM"
    group_rank_src = const_expr(cute.rank(src_tensor) - (1 if not single_stage else 0))
    group_rank_dst = const_expr(cute.rank(dst_tensor) - (1 if not single_stage else 0))
    # ((atom_v, rest_v), STAGE), ((atom_v, rest_v), RestK)
    src = cute.group_modes(src_tensor, 0, group_rank_src)
    dst = cute.group_modes(dst_tensor, 0, group_rank_dst)

    if const_expr(src_is_smem and not dst_is_smem):

        def copy_bulk_s2g(src_idx, dst_idx, **new_kwargs):
            store_bytes = const_expr(cute.size(src.shape[:-1]) * src.element_type.width // 8)
            with cute.arch.elect_one():
                cpasync_bulk_s2g(
                    src[None, src_idx].iterator,
                    dst[None, dst_idx].iterator,
                    store_bytes,
                    reduction_kind=reduction_kind,
                    dtype=src.element_type,
                    **new_kwargs,
                    **kwargs,
                )

        def copy_bulk_s2g_single_stage(**new_kwargs):
            store_bytes = const_expr(cute.size(src.shape) * src.element_type.width // 8)
            with cute.arch.elect_one():
                cpasync_bulk_s2g(
                    src.iterator,
                    dst.iterator,
                    store_bytes,
                    reduction_kind=reduction_kind,
                    dtype=src.element_type,
                    **new_kwargs,
                    **kwargs,
                )

        return copy_bulk_s2g if const_expr(not single_stage) else copy_bulk_s2g_single_stage

    def copy_bulk(src_idx, dst_idx, tma_bar_ptr: cute.Pointer, **new_kwargs):
        assert dst_is_smem and not src_is_smem, "cp.async.bulk G2S expects GMEM -> SMEM"
        atom = cute.make_copy_atom(cpasync.CopyBulkG2SOp(), src.element_type)
        with cute.arch.elect_one():
            cute.copy(
                atom,
                src[None, src_idx],
                dst[None, dst_idx],
                mbar_ptr=tma_bar_ptr,
                **new_kwargs,
                **kwargs,
            )

    def copy_bulk_single_stage(tma_bar_ptr: cute.Pointer, **new_kwargs):
        assert dst_is_smem and not src_is_smem, "cp.async.bulk G2S expects GMEM -> SMEM"
        atom = cute.make_copy_atom(cpasync.CopyBulkG2SOp(), src.element_type)
        with cute.arch.elect_one():
            cute.copy(atom, src, dst, mbar_ptr=tma_bar_ptr, **new_kwargs, **kwargs)

    return copy_bulk if const_expr(not single_stage) else copy_bulk_single_stage


def cpasync_bulk_get_store_or_add_fn(
    src_tensor: cute.Tensor,
    dst_tensor: cute.Tensor,
    store_first_contribution: bool,
    single_stage: bool = False,
    **kwargs,
) -> Callable:
    assert not single_stage, "store-or-add helper only supports staged SMEM -> GMEM tensors"
    src_is_smem = const_expr(
        isinstance(src_tensor.iterator, cute.Pointer)
        and src_tensor.memspace == cute.AddressSpace.smem
    )
    dst_is_smem = const_expr(
        isinstance(dst_tensor.iterator, cute.Pointer)
        and dst_tensor.memspace == cute.AddressSpace.smem
    )
    assert src_is_smem and not dst_is_smem, "store-or-add helper only supports SMEM -> GMEM"
    group_rank_src = const_expr(cute.rank(src_tensor))
    group_rank_dst = const_expr(cute.rank(dst_tensor))
    src = cute.group_modes(src_tensor, 0, group_rank_src - 1)
    dst = cute.group_modes(dst_tensor, 0, group_rank_dst - 1)

    @cute.jit
    def copy_bulk_s2g_store_or_add(src_idx, dst_idx, idx, **new_kwargs):
        store_bytes = const_expr(cute.size(src.shape[:-1]) * src.element_type.width // 8)
        src_ptr = src[None, src_idx].iterator
        dst_ptr = dst[None, dst_idx].iterator
        with cute.arch.elect_one():
            if const_expr(store_first_contribution):
                if idx == 0:
                    cpasync_bulk_s2g(
                        src_ptr,
                        dst_ptr,
                        store_bytes,
                        reduction_kind=None,
                        **new_kwargs,
                        **kwargs,
                    )
                else:
                    cpasync_bulk_s2g(
                        src_ptr,
                        dst_ptr,
                        store_bytes,
                        reduction_kind=cpasync.ReductionOp.ADD,
                        dtype=src.element_type,
                        **new_kwargs,
                        **kwargs,
                    )
            else:
                cpasync_bulk_s2g(
                    src_ptr,
                    dst_ptr,
                    store_bytes,
                    reduction_kind=cpasync.ReductionOp.ADD,
                    dtype=src.element_type,
                    **new_kwargs,
                    **kwargs,
                )

    return copy_bulk_s2g_store_or_add


@dsl_user_op
def tma_get_copy_fn(
    atom: cute.CopyAtom,
    cta_coord: cute.Coord,
    cta_layout: cute.Layout,
    src_tensor: cute.Tensor,
    dst_tensor: cute.Tensor,
    filter_zeros: bool = False,
    single_stage: bool = False,
    *,
    loc=None,
    ip=None,
    **kwargs,
) -> Callable:
    src_is_smem = const_expr(
        isinstance(src_tensor.iterator, cute.Pointer)
        and src_tensor.memspace == cute.AddressSpace.smem
    )
    smem_tensor, gmem_tensor = (src_tensor, dst_tensor) if src_is_smem else (dst_tensor, src_tensor)
    group_rank_smem = const_expr(cute.rank(smem_tensor) - (1 if not single_stage else 0))
    group_rank_gmem = const_expr(cute.rank(gmem_tensor) - (1 if not single_stage else 0))
    # ((atom_v, rest_v), STAGE), ((atom_v, rest_v), RestK)
    s, g = cpasync.tma_partition(
        atom,
        cta_coord,
        cta_layout,
        cute.group_modes(smem_tensor, 0, group_rank_smem),
        cute.group_modes(gmem_tensor, 0, group_rank_gmem),
        loc=loc,
        ip=ip,
    )
    if const_expr(filter_zeros):
        s = cute.filter_zeros(s)
        g = cute.filter_zeros(g)
    src, dst = (s, g) if src_is_smem else (g, s)

    @dsl_user_op
    def copy_tma(src_idx, dst_idx, *, loc=None, ip=None, **new_kwargs):
        cute.copy(
            atom, src[None, src_idx], dst[None, dst_idx], **new_kwargs, **kwargs, loc=loc, ip=ip
        )

    @dsl_user_op
    def copy_tma_single_stage(*, loc=None, ip=None, **new_kwargs):
        cute.copy(atom, src, dst, **new_kwargs, **kwargs, loc=loc, ip=ip)

    return (copy_tma if const_expr(not single_stage) else copy_tma_single_stage), s, g


@dsl_user_op
def tma_get_block_copy_fn(
    atom: cute.CopyAtom,
    src_tensor: cute.Tensor,
    dst_tensor: cute.Tensor,
    tma_multicast: Optional[dict] = None,
    single_stage: bool = False,
    *,
    loc=None,
    ip=None,
    **kwargs,
) -> Callable:
    src_is_smem = const_expr(
        isinstance(src_tensor.iterator, cute.Pointer)
        and src_tensor.memspace == cute.AddressSpace.smem
    )
    if const_expr(tma_multicast is not None and "use_2cta_mma_inst" not in tma_multicast):
        op = atom.op if const_expr(hasattr(atom, "op")) else atom
        tma_multicast = {
            **tma_multicast,
            "use_2cta_mma_inst": getattr(op, "cta_group", None) == tcgen05.CtaGroup.TWO,
        }
    smem_tensor, gmem_tensor = (src_tensor, dst_tensor) if src_is_smem else (dst_tensor, src_tensor)
    group_rank_smem = const_expr(cute.rank(smem_tensor) - (1 if not single_stage else 0))
    group_rank_gmem = const_expr(cute.rank(gmem_tensor) - (1 if not single_stage else 0))
    s = cute.group_modes(smem_tensor, 0, group_rank_smem)
    g = cute.group_modes(gmem_tensor, 0, group_rank_gmem)
    src, dst = (s, g) if src_is_smem else (g, s)

    @dsl_user_op
    def copy_tma(src_idx, dst_idx, *, loc=None, ip=None, **new_kwargs):
        src_cur = src[None, src_idx]
        dst_cur = dst[None, dst_idx]
        if const_expr(tma_multicast is None):
            block_copy(atom, src_cur, dst_cur, **new_kwargs, **kwargs, loc=loc, ip=ip)
        else:
            block_copy(
                atom,
                src_cur,
                dst_cur,
                tma_multicast=tma_multicast,
                **new_kwargs,
                **kwargs,
                loc=loc,
                ip=ip,
            )

    @dsl_user_op
    def copy_tma_single_stage(*, loc=None, ip=None, **new_kwargs):
        if const_expr(tma_multicast is None):
            block_copy(atom, src, dst, **new_kwargs, **kwargs, loc=loc, ip=ip)
        else:
            block_copy(
                atom,
                src,
                dst,
                tma_multicast=tma_multicast,
                **new_kwargs,
                **kwargs,
                loc=loc,
                ip=ip,
            )

    return copy_tma if const_expr(not single_stage) else copy_tma_single_stage


def s2t_get_copy_fn(
    src_tensor: cute.Tensor,
    dst_tensor: cute.Tensor,
    cta_group: tcgen05.CtaGroup,
) -> Callable:
    """
    Make tiledCopy for smem to tmem load, then return a copy function over stages.

    :param src_tensor: The source tensor in smem
    :param dst_tensor: The destination tensor in tmem
    """
    assert src_tensor.element_type == dst_tensor.element_type
    # (MMA, MMA_MN, MMA_K, STAGE)
    src_compact = cute.filter_zeros(src_tensor)
    # (MMA, MMA_MN, MMA_K)
    dst_compact = cute.filter_zeros(dst_tensor)
    # Make S2T CopyAtom and tiledCopy.
    copy_atom = cute.make_copy_atom(tcgen05.Cp4x32x128bOp(cta_group), dst_tensor.element_type)
    tiled_copy = tcgen05.make_s2t_copy(copy_atom, dst_compact)
    thr_copy = tiled_copy.get_slice(0)
    # ((ATOM_V, REST_V), Rest_Tiler, MMA_MN, MMA_K, STAGE)
    src_partition = tcgen05.get_s2t_smem_desc_tensor(tiled_copy, thr_copy.partition_S(src_compact))
    # ((ATOM_V, REST_V), Rest_Tiler, MMA_MN, MMA_K)
    dst_partition = thr_copy.partition_D(dst_compact)

    @dsl_user_op
    def copy_s2t(stage_idx, *, loc=None, ip=None, **new_kwargs):
        # Stage slice of partitioned source tensor: ((ATOM_V, REST_V), Rest_Tiler, MMA_MN, MMA_K)
        stage_coord = (None, None, None, None, stage_idx)
        cute.copy(
            tiled_copy, src_partition[stage_coord], dst_partition, loc=loc, ip=ip, **new_kwargs
        )

    return copy_s2t


# tcgen05 TMEM <-> RMEM helpers (t2r loads / r2t stores).
#
# The register-side fragment of a tmem copy is derivable by layout algebra,
# with no reference to the original (pre-partition) tile tensor:
# - the per-thread register VALUE shape is mode 1 of the tiled copy's
#   register-side TV layout (`layout_dst_tv_tiled` for loads,
#   `layout_src_tv_tiled` for stores). The tmem-side partition can't supply
#   it: tmem partitioning is warp-collective, so its value mode counts tmem
#   cells across the whole warp, not per-thread register elements.
# - the tile-iteration modes are shared between partition_S and partition_D
#   (same tiler over the same tile extent), so they can be read off whichever
#   side was already partitioned.
# This kills the make-a-fake/identity-tensor-and-partition_D dance previously
# needed at every t2r site.


def tmem_reg_frag(
    tiled_copy: cute.TiledCopy,
    partitioned: cute.Tensor,
    num_extra_modes: int = 0,
    dtype: Optional[Type[cutlass.Numeric]] = None,
    *,
    loc=None,
    ip=None,
) -> cute.Tensor:
    """Allocate the per-thread register fragment for ONE tile of a tcgen05
    tmem copy, given any partitioned view of it (tmem or otherwise).

    `partitioned` is (V, iter..., extra...) as produced by partition_S/_D;
    the trailing `num_extra_modes` modes (stage, epi-subtile, ...) are
    excluded from the fragment and indexed at copy time instead. `dtype`
    defaults to `partitioned.element_type`.
    The register side is inferred from the tcgen05 load/store op: destination
    for t2r loads, source for r2t stores."""
    tv = _tmem_copy_reg_tv_layout(tiled_copy)
    val_shape = tv.shape[1]
    rank = cute.rank(partitioned.shape)
    iters = tuple(partitioned.shape[i] for i in range(1, rank - num_extra_modes))
    frag_dtype = partitioned.element_type if const_expr(dtype is None) else dtype
    return cute.make_rmem_tensor((val_shape, *iters), frag_dtype, loc=loc, ip=ip)


def coord_frag(tiled_copy: cute.TiledCopy, tidx: Int32, shape) -> cute.Tensor:
    """Per-thread (row, col) coordinates aligned with a tiled copy's register
    fragments (`tmem_reg_frag` / `load_t2r`): the register-side partition of
    an identity tensor over `shape`. Deliberately partition_D — partition_S of
    a TMEM tiled copy keeps whole warp-addressed atom tiles instead of
    distributing elements over lanes."""
    return tiled_copy.get_slice(tidx).partition_D(cute.make_identity_tensor(shape))


def r2s_partition_from_t2r(
    tiled_copy_t2r: cute.TiledCopy,
    s: cute.Tensor,
    tidx: Int32,
    transpose: bool = False,
    position_independent=False,
) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]:
    """SMEM-store (r2s) side chained off a tmem-load tiled copy: the r2s copy
    inherits the t2r copy's per-thread value ownership via make_tiled_copy_D,
    so the loaded fragment can be stored (post-conversion) without a shuffle.
    By default the store atom is selected like SM100 GEMM epilogues:
    `get_smem_store_op(layout, dst_dtype, tiled_copy_t2r.value_type, tiled_copy_t2r)`,
    so the stmatrix shape follows the tmem-load atom. `transpose=True` maps to
    COL_MAJOR, otherwise ROW_MAJOR.

    `s` is the staged SMEM tile; its trailing stage mode is excluded from the
    register fragment. `position_independent=True` partitions through a
    position-independent swizzle view, matching `get_smem_store_C`.
    Returns `(tiled_copy, tRS_r, tRS_s)`; store via
    `cute.copy(tiled_copy, tRS_r, tRS_s[..., idx])`."""
    dtype = s.element_type
    layout = LayoutEnum.COL_MAJOR if const_expr(transpose) else LayoutEnum.ROW_MAJOR
    copy_atom = sm100_utils.get_smem_store_op(
        layout, dtype, tiled_copy_t2r.value_type, tiled_copy_t2r
    )
    tiled_copy = cute.make_tiled_copy_D(copy_atom, tiled_copy_t2r)
    thr_copy = tiled_copy.get_slice(tidx)
    if const_expr(not position_independent):
        tRS_s = thr_copy.partition_D(s)
    else:
        tRS_s = partition_D_position_independent(thr_copy, s)
    rank = cute.rank(tRS_s.shape)
    frag_shape = tuple(tRS_s.shape[i] for i in range(rank - 1))
    tRS_r = cute.make_rmem_tensor(frag_shape, dtype)
    return tiled_copy, tRS_r, tRS_s


def s2r_partition_from_t2r(
    tiled_copy_t2r: cute.TiledCopy,
    s: cute.Tensor,
    tidx: Int32,
    r_layout: cute.Layout,
    copy_atom: Optional[cute.CopyAtom] = None,
    transpose: bool = False,
    position_independent=False,
) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor, cute.Tensor]:
    """SMEM-load (s2r) counterpart of `r2s_partition_from_t2r` (ldmatrix vs
    stmatrix), for reading an epilogue input that was TMA-staged into an
    epi-tile SMEM buffer (e.g. C in gemm, z in ssd) into registers
    element-aligned with the t2r fragments. The register fragment is
    allocated with `r_layout` (pass the r2s fragment's layout) so its linear
    element order matches the t2r/r2s fragments; `tSR_r` is its retiled view
    for the collective copy. Per-warp SMEM footprints of this load and the
    chained r2s store coincide, so reusing one buffer for input then output
    is warp-local (no inter-warp hazard).

    `position_independent=True` partitions through a position-independent
    swizzle view, matching `get_smem_load_C`.

    Returns `(tiled_copy, tRS_r, tSR_r, tSR_s)`; load via
    `cute.copy(tiled_copy, tSR_s[..., idx], tSR_r)` then read `tRS_r`."""
    dtype = s.element_type
    if const_expr(copy_atom is None):
        copy_atom = cute.make_copy_atom(
            warp.LdMatrix8x8x16bOp(transpose=transpose, num_matrices=4), dtype
        )
    tiled_copy = cute.make_tiled_copy_D(copy_atom, tiled_copy_t2r)
    thr_copy = tiled_copy.get_slice(tidx)
    if const_expr(not position_independent):
        tSR_s = thr_copy.partition_S(s)
    else:
        tSR_s = partition_S_position_independent(thr_copy, s)
    tRS_r = cute.make_rmem_tensor(r_layout, dtype)
    tSR_r = tiled_copy.retile(tRS_r)
    return tiled_copy, tRS_r, tSR_r, tSR_s


def tma_producer_copy_fn(copy: Callable, pipeline: cutlass.pipeline.PipelineAsync):
    def copy_fn(src_idx, producer_state: cutlass.pipeline.PipelineState, **new_kwargs):
        copy(
            src_idx=src_idx,
            dst_idx=producer_state.index,
            tma_bar_ptr=pipeline.producer_get_barrier(producer_state),
            **new_kwargs,
        )

    return copy_fn


def chain_tma_producer_copy_fns(copy_fns: Sequence[Optional[Callable]]):
    if not any(fn is not None for fn in copy_fns):
        return None

    def copy_fn(src_idx, producer_state: cutlass.pipeline.PipelineState, **new_kwargs):
        for fn in copy_fns:
            if const_expr(fn is not None):
                fn(src_idx=src_idx, producer_state=producer_state, **new_kwargs)

    return copy_fn


@cute.jit
def gather_m_get_copy_fn(
    thr_copy_A: cute.ThrCopy,
    mA: cute.Tensor,  # (whatever, K)
    sA: cute.Tensor,  # (tile_M, tile_K, STAGE)
    gsAIdx: cute.Tensor,  # (tile_M), either gmem or smem
    limit_m: Int32,
    limit_k: Int32,
) -> Callable:
    tile_M, tile_K = cute.size(sA, mode=[0]), cute.size(sA, mode=[1])
    tAsA = partition_D_position_independent(thr_copy_A, sA)
    # k-major
    assert tAsA.shape[2] == 1
    tAsA = cute.group_modes(cute.slice_(tAsA, (None, None, 0, None)), 0, 2)

    is_even_m_smem = tile_M % thr_copy_A.tiler_mn[0].shape == 0
    if const_expr(not is_even_m_smem):
        limit_m = min(limit_m, tile_M)
    elems_per_load = cute.size(tAsA.shape[0][0])
    cA = cute.make_identity_tensor((tile_M, tile_K))
    tAcA = thr_copy_A.partition_S(cA)
    t0AcA = thr_copy_A.get_slice(0).partition_S(cA)
    # Instead of comparing tAcA to limit_m, we instead compare t0AcA to limit_m - tAcA[0][0]
    # since we know that tAcA[m][0] = t0AcA[m][0] + tAcA[0][0].
    # This is so that when we do the comparison, t0AcA is known at compile time.
    limit_m = limit_m - tAcA[0][0]
    limit_k = limit_k - tAcA[0][1]
    # Read and cache indices for A
    rows_per_thread = const_expr(cute.size(tAcA.shape, mode=[1]))
    cols_per_thread = const_expr(cute.size(tAcA.shape, mode=[2]))
    tApA_m = cute.make_rmem_tensor(rows_per_thread, Boolean)
    for m in cutlass.range(rows_per_thread, unroll_full=True):
        tApA_m[m] = t0AcA[0, m, 0][0] < limit_m
    m_idx = cute.make_rmem_tensor(rows_per_thread, Int32)
    for m in cutlass.range(rows_per_thread, unroll_full=True):
        row_idx = tAcA[0, m, 0][0]
        if tApA_m[m]:
            m_idx[m] = gsAIdx[row_idx]
        else:
            m_idx[m] = 0  # It's ok to load row 0 in the case of OOB

    mA_k = cute.logical_divide(mA, (None, tile_K))

    def copy_fn(src_idx, dst_idx, pred: cutlass.Constexpr[bool] = False):
        tApA_k = None
        if const_expr(pred):
            tApA_k = cute.make_rmem_tensor(cols_per_thread, Boolean)
            limit_k_cur = limit_k - src_idx * tile_K
            for k in cutlass.range(cols_per_thread, unroll_full=True):
                tApA_k[k] = t0AcA[0, 0, k][1] < limit_k_cur
        mA_cur = mA_k[None, (None, src_idx)]
        for m in cutlass.range_constexpr(tAcA.shape[1]):
            # cute.tiled_divide(mA_cur[m_idx[m], None], (elems_per_load,)) would give shape
            # ((elems_per_load), thread_per_row)
            # But we actually want shape ((elems_per_load, 1), thread_per_row) to match tAsA
            # So we append 1s to the last dimension and then do tiled_divide, then slice.
            mA_row = cute.tiled_divide(
                cute.append_ones(mA_cur[m_idx[m], None], up_to_rank=2), (elems_per_load, 1)
            )[None, None, 0]
            if const_expr(is_even_m_smem) or tApA_m[m]:
                # There's only 1 load per row
                assert cute.size(tAcA.shape, mode=[2]) == 1
                ki = tAcA[0, 0, 0][1] // elems_per_load
                cute.copy(thr_copy_A, mA_row[None, ki], tAsA[(None, m), dst_idx], pred=tApA_k)

    return copy_fn


@cute.jit
def gather_k_get_copy_fn(
    thr_copy_A: cute.ThrCopy,
    mA: cute.Tensor,  # (tile_M, whatever)
    sA: cute.Tensor,  # (tile_M, tile_K, STAGE)
    gsAIdx: cute.Tensor,  # (tile_K, RestK), either gmem or smem
    limit_m: Int32,
    limit_k: Int32,
) -> Callable:
    gAIdx, sAIdx = None, None
    if const_expr(gsAIdx.memspace == cute.AddressSpace.gmem):
        gAIdx = gsAIdx
    else:
        assert gsAIdx.memspace == cute.AddressSpace.smem
        sAIdx = gsAIdx
    tile_shape_mk = (cute.size(sA, mode=[0]), cute.size(sA, mode=[1]))
    # (atom_v, CPY_M, 1, STAGE)
    tAsA = thr_copy_A.partition_D(sA)
    # m-major
    tAsA = cute.group_modes(tAsA, 0, 3)

    is_even_m_smem = tile_shape_mk[0] % thr_copy_A.tiler_mn[0].shape == 0
    if const_expr(not is_even_m_smem):
        limit_m = min(limit_m, tile_shape_mk[0])
    elems_per_load = cute.size(tAsA.shape[0][0])
    cA = cute.make_identity_tensor(tile_shape_mk)
    tAcA = thr_copy_A.partition_S(cA)
    t0AcA = thr_copy_A.get_slice(0).partition_S(cA)
    # Instead of comparing tAcA to limit_m, we instead compare t0AcA to limit_m - tAcA[0][0]
    # since we know that tAcA[m][0] = t0AcA[m][0] + tAcA[0][0].
    # This is so that when we do the comparison, t0AcA is known at compile time.
    limit_m = limit_m - tAcA[0][0]
    limit_k = limit_k - tAcA[0][1]
    # Read and cache indices for A
    rows_per_thread = const_expr(cute.size(tAcA.shape, mode=[1]))
    cols_per_thread = const_expr(cute.size(tAcA.shape, mode=[2]))
    tApA_m = cute.make_rmem_tensor(rows_per_thread, Boolean)
    for m in cutlass.range(rows_per_thread, unroll_full=True):
        tApA_m[m] = t0AcA[0, m, 0][0] < limit_m
    threads_per_col = const_expr(thr_copy_A.tiler_mn[0].shape // elems_per_load)
    # This is very convoluted but idk a better way
    # for tile_M=128, flat_divide gives (8, 16, K),
    # then logical_divide gives ((8, 1), (8, 2), K).
    tidx = thr_copy_A.thr_idx
    tAmA = cute.logical_divide(
        cute.flat_divide(mA, (elems_per_load,)), (elems_per_load, threads_per_col)
    )[None, (tidx % threads_per_col, None), None]  # ((8, 1), 2, K)

    def prefetch_from_gmem_fn(src_idx, pred: bool = False) -> Tuple[cute.Tensor, cute.Tensor]:
        # Prefetch mAIdx early, even before smem is free
        tApA_k = None
        if const_expr(pred):
            tApA_k = cute.make_rmem_tensor(cols_per_thread, Boolean)
            limit_k_cur = limit_k - src_idx * tile_shape_mk[1]
            for k in cutlass.range(cols_per_thread, unroll_full=True):
                tApA_k[k] = t0AcA[0, 0, k][1] < limit_k_cur
        gAIdx_cur = gAIdx[None, src_idx]
        k_idx = cute.make_rmem_tensor(cols_per_thread, Int32)
        for k in cutlass.range(cols_per_thread):
            col_idx = tAcA[0, 0, k][1]
            if const_expr(not pred):
                k_idx[k] = gAIdx_cur[col_idx]
            else:
                if tApA_k[k]:
                    k_idx[k] = gAIdx_cur[col_idx]
                else:
                    k_idx[k] = -1
        return k_idx, tApA_k

    def prefetch_from_smem_fn(
        a_prefetch_pipeline, src_idx, dst_idx, a_prefetch_consumer_state, pred: bool = False
    ) -> Tuple[cute.Tensor, cute.Tensor]:
        tApA_k = None
        if const_expr(pred):
            tApA_k = cute.make_rmem_tensor(cols_per_thread, Boolean)
            limit_k_cur = limit_k - src_idx * tile_shape_mk[1]
            for k in cutlass.range(cols_per_thread, unroll_full=True):
                tApA_k[k] = t0AcA[0, 0, k][1] < limit_k_cur
        a_prefetch_pipeline.consumer_wait(a_prefetch_consumer_state)
        sAIdx_cur = sAIdx[None, dst_idx]
        k_idx = cute.make_rmem_tensor(cols_per_thread, Int32)
        for k in cutlass.range(cols_per_thread):
            col_idx = tAcA[0, 0, k][1]
            k_idx[k] = sAIdx_cur[col_idx]
        a_prefetch_pipeline.consumer_release(a_prefetch_consumer_state)
        return k_idx, tApA_k

    def copy_fn(
        src_idx, dst_idx, k_idx_tApA_k: Tuple[cute.Tensor, cute.Tensor], pred: bool = False
    ):
        k_idx, tApA_k = k_idx_tApA_k
        tApA_k_pred = None
        if const_expr(pred):
            tApA_k_pred = cute.prepend_ones(tApA_k, up_to_rank=2)  # (1, cols_per_thread)
        for k in cutlass.range_constexpr(tAcA.shape[2]):
            # copy_A(tAmA[None, None, k_idx[k]], tAsA[(None, None, k), smem_idx], pred=cute.prepend_ones(tApA_m, up_to_rank=2))
            for m in cutlass.range_constexpr(tAcA.shape[1]):
                if tApA_m[m]:
                    cute.copy(
                        thr_copy_A,
                        tAmA[None, m, k_idx[k]],
                        tAsA[(None, m, k), dst_idx],
                        pred=None if const_expr(tApA_k_pred is None) else tApA_k_pred[None, k],
                    )

    return copy_fn, prefetch_from_gmem_fn if const_expr(
        gAIdx is not None
    ) else prefetch_from_smem_fn


@cute.jit
def gather_m_get_tma_copy_fn(
    tma_atom: cute.CopyAtom,
    mA: cute.Tensor,  # (whatever, K)
    sA: cute.Tensor,  # ((4, 32), (64, 1), STAGE)
    sAIdx: cute.Tensor,  # (tile_M),
    warp_idx: Int32,
    num_warps: int,
    num_cta: int = 1,
) -> Callable:
    tile_M = cute.size(sAIdx, mode=[0])
    tile_K = cute.size(sA[None, None, 0]) // tile_M
    assert tile_M % 4 == 0
    # cta_group = 1 if tma_atom.op.cta_group == CtaGroup.ONE else 2
    cta_group = num_cta  # Somehow all tma_atom has CtaGroup.ONE inside the kernel

    copy_AIdx_s2r = cute.make_tiled_copy_tv(
        cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), Int32, num_bits_per_copy=128),
        cute.make_layout(num_warps),  # thr_layout
        cute.make_layout(4),  # val_layout
    )
    warp_copy_AIdx_s2r = copy_AIdx_s2r.get_slice(warp_idx)
    tSR_sAIdx = warp_copy_AIdx_s2r.partition_S(sAIdx)
    # ((4, 1), 8, (64, 1), STAGE)
    tSR_sA = warp_copy_AIdx_s2r.partition_S(sA)
    tSR_rAIdx = load_s2r(tSR_sAIdx)
    tma_desc_ptr = get_tma_desc_addr(tma_atom)
    tma_gather4_load_fn = partial(tma_gather4_load, tma_desc_ptr, num_cta=cta_group)

    def copy_fn(src_idx, dst_idx, tma_bar_ptr: cute.Pointer):
        tSR_sA_cur = tSR_sA[None, None, None, dst_idx]
        col_idx = tile_K * src_idx
        for m in cutlass.range(cute.size(tSR_rAIdx, mode=[1]), unroll_full=True):
            row_indices = [tSR_rAIdx[v, m] for v in range(4)]
            smem_ptr = tSR_sA_cur[None, m, None].iterator
            with cute.arch.elect_one():
                tma_gather4_load_fn(smem_ptr, tma_bar_ptr, col_idx, row_indices)

    return copy_fn


@cute.jit
def gather_k_get_tma_copy_fn(
    tma_atom: cute.CopyAtom,
    sA: cute.Tensor,  # ((4, tile_K/4), (tile_M,), STAGE) — K-grouped load layout
    sAIdx: cute.Tensor,  # (tile_K, a_prefetch_stage) — K indices in smem
    col_idx: Int32,  # M offset in global tensor (contiguous dim for M-major)
    warp_idx: Int32,
    num_warps: int,
    num_cta: int = 1,
) -> Tuple[Callable, Callable]:
    """Build a copy function for TMA gather4 in K dimension (M-major A).

    Each gather4 instruction loads 4 K-columns × tile_M contiguous M-elements.
    col_idx is the absolute M position in the global tensor.
    K indices come from sAIdx (prefetched to smem by the scheduler warp).

    Returns copy_fn(src_idx, dst_idx, tma_bar_ptr) which:
      Issues gather4 calls with those K indices as row_indices
    """
    tile_K = cute.size(sAIdx, mode=[0])
    assert tile_K % 4 == 0
    cta_group = num_cta

    # Tiled copy for loading K indices from smem to registers (4 per vector, across warps)
    copy_AIdx_s2r = cute.make_tiled_copy_tv(
        cute.make_copy_atom(cute.nvgpu.CopyUniversalOp(), Int32, num_bits_per_copy=128),
        cute.make_layout(num_warps),  # thr_layout
        cute.make_layout(4),  # val_layout — 4 K indices per gather4
    )
    warp_idx = cute.arch.make_warp_uniform(warp_idx)
    warp_copy_AIdx_s2r = copy_AIdx_s2r.get_slice(warp_idx)
    tSR_sAIdx = warp_copy_AIdx_s2r.partition_S(sAIdx)  # (((4,1),4,4))
    # ((4,1),4,(64,2),(1,4)):((64,0),1024,(1,4096),(0,8192))
    tSR_sA = warp_copy_AIdx_s2r.partition_S(layout_utils.transpose_view(sA))
    tma_desc_ptr = get_tma_desc_addr(tma_atom)
    tma_gather4_load_fn = partial(tma_gather4_load, tma_desc_ptr, num_cta=cta_group)

    def prefetch_from_smem_fn(
        a_prefetch_pipeline,
        src_idx,
        dst_idx,
        a_prefetch_consumer_state,
    ) -> cute.Tensor:
        a_prefetch_pipeline.consumer_wait(a_prefetch_consumer_state)
        tSR_rAIdx = load_s2r(tSR_sAIdx[None, None, dst_idx])
        a_prefetch_pipeline.consumer_release(a_prefetch_consumer_state)
        return tSR_rAIdx

    def copy_fn(src_idx, dst_idx, tSR_rAIdx, tma_bar_ptr: cute.Pointer):
        # Issue gather4: col_idx = M position, row_indices = 4 K positions
        tSR_sA_cur = tSR_sA[None, None, None, dst_idx]
        gather_dim = cute.size(tSR_sA_cur, mode=[2, 0])  # Typically 64
        for k in cutlass.range(cute.size(tSR_rAIdx, mode=[1]), unroll_full=True):
            row_indices = [tSR_rAIdx[v, k] for v in range(4)]
            for m in cutlass.range(cute.size(tSR_sA_cur, mode=[2, 1]), unroll_full=True):
                smem_ptr = tSR_sA_cur[None, k, (None, m)].iterator
                with cute.arch.elect_one():
                    tma_gather4_load_fn(
                        smem_ptr, tma_bar_ptr, col_idx + m * gather_dim, row_indices
                    )

    return copy_fn, prefetch_from_smem_fn


# ---------------------------------------------------------------------------
# Store helpers
# ---------------------------------------------------------------------------


@dsl_user_op
@cute.jit
def store(
    ptr: cute.Pointer,
    val,
    pred: Optional[Boolean] = None,
    cop: cutlass.Constexpr = None,
    *,
    loc=None,
    ip=None,
):
    """Store a scalar value via cute.arch.store.

    ptr:  cute.Pointer (any address space).
    val:  DSL Numeric value.
    pred: None → unconditional.  DSL Boolean → skipped when pred == 0.
    cop:  Cache operator — "wb" (default), "cg", "cs" (streaming), "wt".
    """
    if const_expr(pred is None):
        cute.arch.store(ptr.llvm_ptr, type(val)(val), cop=cop, loc=loc, ip=ip)
    else:
        if pred:
            cute.arch.store(ptr.llvm_ptr, type(val)(val), cop=cop, loc=loc, ip=ip)


@dsl_user_op
@cute.jit
def store_v2(
    ptr: cute.Pointer,
    v0,
    v1,
    pred: Optional[Boolean] = None,
    cop: cutlass.Constexpr = None,
    *,
    loc=None,
    ip=None,
):
    """Vectorized store of 2 elements via cute.arch.store.

    Packs v0, v1 into an MLIR <2 x T> vector.
    ptr:  cute.Pointer (any address space, must be aligned for vector width).
    cop:  Cache operator — "wb" (default), "cg", "cs" (streaming), "wt".
    """
    vec = make_vector(type(v0), v0, v1, loc=loc, ip=ip)
    if const_expr(pred is None):
        cute.arch.store(ptr.llvm_ptr, vec, cop=cop, loc=loc, ip=ip)
    else:
        if pred:
            cute.arch.store(ptr.llvm_ptr, vec, cop=cop, loc=loc, ip=ip)


@dsl_user_op
@cute.jit
def store_v4(
    ptr: cute.Pointer,
    v0,
    v1,
    v2,
    v3,
    pred: Optional[Boolean] = None,
    cop: cutlass.Constexpr = None,
    *,
    loc=None,
    ip=None,
):
    """Vectorized store of 4 elements via cute.arch.store.

    Packs v0–v3 into an MLIR <4 x T> vector.
    ptr:  cute.Pointer (any address space, must be aligned for vector width).
    cop:  Cache operator — "wb" (default), "cg", "cs" (streaming), "wt".
    """
    vec = make_vector(type(v0), v0, v1, v2, v3, loc=loc, ip=ip)
    if const_expr(pred is None):
        cute.arch.store(ptr.llvm_ptr, vec, cop=cop, loc=loc, ip=ip)
    else:
        if pred:
            cute.arch.store(ptr.llvm_ptr, vec, cop=cop, loc=loc, ip=ip)
