+
    õäûjõË  ã                   óN  € ^ RI t ^RIHt ^RIHt ^RIHt ^RIHt	 ^RIH
t ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHtHtHtHt ^RIHt ^RIHt ^RIHtHt ^RIHtH t  ^RI!H"t"H#t#H$t$ ^RI%H&t&H't' ^RI%Ht( ^RI%H)t* ^RI%H+t, ^RI%H-t-H.t. ^RI%H/t0 ^RI%H1t2 ^RI3H3t3 ^RI4H5t5H6t6H7t7H8t8 ^RI4Ht9 ^RI:H;t;H<t<H=t= ^RI>H?t?H@t@HAtA ]B! ] P†                  RR4      '       dL   ^RIHDtE ]3P6                  ! R4      '       d-   ]3PŒ                  ! . RüO4       ]EP                  t]EP                  tM]3PŽ                  ! RR4       . RNRNR NR!NR"NR#NR$NR%NR&NR'NR(NR)NR*NR+NR,NR-NR.NR/NR0NR1NR2NR3NR4NR5NR6NR7NR8NR9NR:NR;NR<NR=NR>NR?NR@NRANRBNRCNRDNRENRFNRGNRHNRINRJNRKNRLNRMNRNNRONRPNRQNRRNRSNRTNRUNRVNRWNRXNRYNRZNR[NR\NR]NR^NR_NR`NRaNRbNRcNRdNReNRfNRgNRhNRiNRjNRkNRlNRmNRnNRoNRpNRqNRrNRsNRtNRuNRvNRwNRxNRyNRzNR{NtHR| R} ltIRýR€ R� lltJR‚ Rƒ ltKRþR„ R… lltLRÿR† R‡ lltMRÿRˆ R‰ lltNER RŠ R‹ lltOERRŒ R� lltPERRŽ R� lltQERR‘ R’ lltRERR“ R” lltSERR• R– lltT] Pª                  3R— R˜ lltV] P®                  3R™ Rš lltX] Pª                  3R› Rœ lltYERR� Rž lltZ] P®                  R3RŸ R  llt[ERR¡ R¢ llt\ERR£ R¤ llt]] P®                  3R¥ R¦ llt^RÿR§ R¨ llt_ERR© Rª llt`ER	R« R¬ lltaER
R­ R® lltbR¯ R° ltcR± R² ltdERR³ R´ llteERRµ R¶ lltfERR· R¸ lltgERR¹ Rº llthERR» R¼ lltiERR½ R¾ lltjERR¿ RÀ lltkERRÁ RÂ lltlRÃ RÄ ltmRÅ RÆ ltnRÇ RÈ ltoRÉ RÊ ltpRË RÌ ltqRÍ RÎ ltrERRÏ RÐ lltsERRÑ RÒ llttRÓ RÔ ltuRÕ RÖ ltERR× RØ lltvERRÙ RÚ lltwRÛ RÜ ltxERRÝ RÞ lltyERRß Rà lltzERRá Râ llt{Rã Rä lt|Rå Ræ lt}]~! Rç4      3Rè Ré lltR~]~! Rç4      R�3Rê Rë llt€Rì Rí lt�ERRî Rï llt‚Rð Rñ ltƒRò Ró lt„Rô Rõ lt…Rö R÷ lt†Rø Rù lt‡Rú Rû ltˆR# (  é    N)Úset_allocation_context)Úcuda)Úascend)Úeager)Útriton)Úsol_attn_chunked)Úmemmove_)ÚDTYPE_TO_CODE)Úmm_int8)ÚBackendErrorÚBackendNotFoundErrorÚBackendNotImplementedErrorÚNoCapableBackendError)Úflash_attention_decode)Úis_available)Úflash_attention_decode_gqaÚ'flash_attention_decode_gqa_is_available)Ú!flash_attention_decode_step_mergeÚ!flash_attention_decode_tree_merge)Úfrom_blockedÚswap_nibblesÚ
to_blocked)Údeltanet_conv_stepÚgated_delta_decode_fused)Údeferred_is_available)Údeferred_buffers)Údeltanet_conv_step_deferredÚgated_delta_decode_deferred)ÚCTL_INTS)ÚSLOT_MAX)Úregistry)ÚPrequantizedInt8AttentionÚint8_attentionÚ int8_attention_from_prequantizedÚprequantize_int8_attention)Úconvrot_w4a4_linearÚdequantize_convrot_w4a4_weightÚquantize_convrot_w4a4_weight)Údequantize_w4a8_int8_weightÚquantize_w4a8_int8_weightÚw4a8_int8_linearÚhip)r,   z&PyTorch ROCm/HIP runtime not availableÚadalnÚrms_gated_residualÚfp16_conv3dÚgroup_norm_silu_pad3dÚgroup_norm_stats3dÚ	rms_adalnr"   r#   r$   Úint8_attention_is_availabler%   r   r   r   r   r   r   r   Úgated_delta_decode_is_availableÚ!gated_delta_deferred_is_availableÚgated_delta_deferred_buffersr   r   Úgated_delta_ctl_intsÚgated_delta_slot_maxÚ#flash_attention_decode_is_availableÚna2dÚna3dÚwindow_attentionÚsol_attnr   Úsol_attn_is_availableÚquantize_per_tensor_fp8Údequantize_per_tensor_fp8Úquantize_nvfp4Údequantize_nvfp4Úquantize_mxfp8Údequantize_mxfp8Úquantize_svdquant_w4a4r(   r*   Úquantize_int8_rowwiseÚquantize_int8_tensorwiseÚdequantize_int8_simpleÚscaled_mm_nvfp4Úscaled_mm_mxfp8Úscaled_mm_svdquant_w4a4r&   r'   r)   Úgemv_awq_w4a16Úfp16_linearÚint8_linearÚint8_linear_modulatedÚint8_linear_rms_modulatedr+   Ú
apply_ropeÚapply_rope_Úapply_rope1Úapply_rope1_Úapply_rope_split_halfÚapply_rope_split_half_Úapply_rope_split_half1Úapply_rope_split_half1_Úrms_ropeÚ	rms_rope_Ú	rms_rope1Ú
rms_rope1_Úrms_rope_split_halfÚrms_rope_split_half_Úrms_rope_split_half1Úrms_rope_split_half1_Ú
delaunay3dÚmesh_bvhÚclosest_point_on_meshÚclosest_hit_on_meshÚconnected_componentsÚedge_collapse_checksÚmin_cutr   r   r   r   ÚmemmoveÚlist_backendsÚset_backend_priorityÚenable_backendÚdisable_backendÚstochastic_rounding_fp8Úuse_backendr   r   r   r   c                ó0   € V ^8„  d   QhR\         R\        /# )é   ÚnameÚkwargs)ÚstrÚdict)Úformats   "ÚRC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\__init__.pyÚ__annotate__rw   ²   s   € ÷ ñ œð ¤Tñ ó    c                ó@   € \         P                  ! WR7      pV! R/ VB # )zDDispatch a thin public wrapper whose signature matches its backends.©rr   © ©r!   Úget_implementation)rq   rr   Úimpls   && rv   Ú_call_backendr   ²   s   € ä×&Ò& tÔ;€DÙ‰>�&‰>Ðrx   ç        Tc                óÌ  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R,          R\        \        ,          R,          R\        \        ,          R,          R	\         P                  R,          R
\        R\
        R\         P                  R,          R\         P                  R,          R\        R\         P                  /# )rp   ÚqÚkÚvÚtauÚscaleNÚsink_blocksÚsink_qÚkey_biasÚ
topk_ratioÚtailÚ	block_lenÚcoarse_gateÚ	token_augÚreturn)ÚtorchÚTensorÚfloatÚlistÚintÚbool)ru   s   "rv   rw   rw   ¸   så   € ÷ @ñ @Ü‡|�|ð@ä‡|�|ð@ô ‡|�|ð@ô 
ð	@ô
 �4�<ð@ô ”c•˜TÕ!ð@ô ”�I˜Õð@ô �l‰l˜TÕ!ð@ô ð@ô ð@ô �|‰|˜dÕ"ð@ô —‘ Õ$ð@ô ð@ô ‡\�\ñ@rx   c                óâ   € \         P                  P                  P                  YY#TVf   ^ ^ .M
\	        V4      Vf   ^ ^ .M
\	        V4      V\        V4      \        V	4      V
V\        V4      4      # )a€  Sol-Attn training-free sparse attention (arXiv 2607.24027).

Each 64-token query block attends a routed subset of key blocks exactly and
covers the rest with one pooled term per block, so the full sequence still
contributes to the softmax denominator. The win grows with sequence length;
below roughly 12k tokens dense or a fused attention is usually faster.

Args:
    q, k, v: ``(B, T, H, 128)`` tensors, same shape and dtype. The fused
        backends take bfloat16 or float16; head_dim is fixed at 128.
    tau: Routing threshold in sigmas of the proxy row. Higher routes fewer
        blocks exactly: cheaper and less accurate.
    scale: Score scale; None means ``head_dim ** -0.5``.
    sink_blocks: ``[start, end)`` key blocks always attended exactly by every
        query -- conditioning rows, typically.
    sink_q: ``[start, end)`` query blocks that attend everything exactly.
    key_bias: Per-key additive logit bias in natural log, ``(T,)``,
        ``(B, T)`` or an SDPA-style ``(B|1, 1, 1, T)`` float or bool mask.
        Honoured by the exact branch only, so biased blocks must be
        covered by ``sink_blocks``.
    topk_ratio: > 0 selects SLA-style top-k instead of the tau threshold:
        keep this fraction of key blocks per query block (the selection the
        lightx2v SLA LoRAs were distilled against). tau is ignored then.
    tail: False drops the pooled term so the softmax runs over the routed
        blocks only (VSA / SLA fine stage).
    block_len: int32 ``(ceil(T/64),)`` live tokens at the front of each
        64-token block, for zero-padded tiles. Values are clamped to
        ``[1, rows in the block]``; dead rows are never keys and their
        output rows are unspecified.
    coarse_gate: ``(B, T, H, 128)`` per-token gate for VSA's coarse branch:
        ``gate * softmax(q_mean k_mean^T * scale) v_mean`` is added per block.
    token_aug: 0, or a multiple of 64 up to 256: up to that many tokens per
        query block are routed individually, the highest-scoring ones outside
        the routed blocks, and attended exactly. The eager reference ignores
        it.

Returns:
    ``(B, T, H, 128)`` attention output.
)r�   ÚopsÚcomfy_kitchenr=   r“   r’   r•   r”   )r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   r‹   rŒ   r�   rŽ   s   &&&&&&&&&&&&&rv   r=   r=   ¸   sl   € ôl �9‰9×"Ñ"×+Ñ+Ø	ˆa�eØÒ%ˆˆA‰¬4°Ó+<Ø’.ˆˆA‰¤d¨6£lØÜˆjÓÜˆT‹
ØØÜˆI‹ó
ð 
rx   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )rp   ÚdstÚsrc©r�   r‘   )ru   s   "rv   rw   rw   û   s"   € ÷ ñ ”—‘ð ¤E§L¡Lñ rx   c                ó  € V P                   P                  R8X  dJ   \        P                  ! \        P
                  P                  '       d   RMR4      '       d   \        W4      # V P                  VP                  4       4       R# )z?Copy src's bytes into dst in place; the two ranges may overlap.r   r,   N)
ÚdeviceÚtyper!   r   r�   Úversionr,   Ú	_memmove_Úcopy_Úclone)rš   r›   s   &&rv   rh   rh   û   sQ   € à
‡z�z‡�˜&Ô ¤X×%:Ò%:ÄEÇMÁM×DU×DUÐDU¹5Ð[a×%bÒ%bÜ˜Ó"Ð"Ø‡I�Iˆc�i‰i‹kÖrx   c                óR   € V ^8„  d   QhR\         P                  R,          R\        /# )rp   rž   Nr�   )r�   rž   r•   )ru   s   "rv   rw   rw     s'   € ÷ Zñ Z¤%§,¡,°Õ"5ð ZÄñ Zrx   c                ó€  € V e   V P                   R8w  d   R# \        P                  P                  4       '       g   R# \	        \        P
                  RR4      '       d8   \        P                  ! R4      ;'       d    \        P                  ! RR4      RJ# \        P                  ! RR4      p\	        \        RR4      p\        P                  ! R4      ;'       de    \        P                  ;'       dM    \        VR4      ;'       d9    VRJ;'       d-    \        P                  P                  V 4      VP                  8¬  # )zÑWhether the compiled Sol-Attn kernels can run on ``device``: the CUDA
backend on sm_80+, or the HIP backend on a GPU with matrix cores. The
per-call rules (bf16/fp16, head_dim 128, matching q/k/v) still apply.Nr   Fr,   r=   Ú_C)rŸ   r�   r   r   Úgetattrr    r!   Úget_constraintsÚ_cuda_backendÚ_EXT_AVAILABLEÚhasattrÚget_device_capabilityÚmin_compute_capability)rž   ÚrulesÚexts   &  rv   r>   r>     s  € ð Ò˜fŸk™k¨VÔ3ÙÜ�:‰:×"Ñ"×$Ò$ÙÜŒu�}‰}˜e T×*Ò*ô ×$Ò$ UÓ+×gÐg´×0HÒ0HÈÐPZÓ0[ÐcgÐ0gÐgÜ×$Ò$ V¨ZÓ8€EÜ
”-  tÓ
,€CÜ×!Ò! &Ó)÷ Yð Y¬m×.JÑ.J÷ Yð YÌwÐWZÐ\fÓOg÷ Yð YØ˜TÐ!÷Yð Yä—
‘
×0Ñ0°Ó8¸E×<XÑ<XÑXðZrx   c                ó0  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \        \        ,          ,          R\        \        \        ,          ,          R,          R\
        R,          R\         P                  /# ©	rp   r‚   rƒ   r„   Úkernel_sizeÚ	is_causalNr†   r�   ©r�   r‘   r”   r“   r•   r’   )ru   s   "rv   rw   rw     s�   € ÷ "Pñ "PÜ‡|�|ð"Pä‡|�|ð"Pô ‡|�|ð"Pô ”tœC•y•ð	"Pô
 ”dœ4•jÕ  4Õ'ð"Pô �4�<ð"Pô ‡\�\ñ"Prx   c                óê   € \        V\        4      '       d   V.^,          pVf   . ROpM \        V\        4      '       d   V.^,          p\        P                  P
                  P                  WW#WE4      # )av  3D neighborhood attention (NATTEN ``na3d`` semantics, dilation 1).

Per non-causal axis each query attends a window of exactly
``kernel_size`` positions centered on it, shifted inward at grid
boundaries (kernels larger than an axis clamp to that axis); per causal
axis it attends the ``min(i + 1, kernel_size)`` nearest previous
positions. RoPE/normalization are the caller's responsibility.

Args:
    q, k, v: ``(B, T, H, W, num_heads, head_dim)`` tensors, same shape/dtype.
    kernel_size: Per-axis window sizes ``[k_t, k_h, k_w]``; a bare int
        repeats across the axes (NATTEN convention).
    is_causal: Per-axis causal flags; a bare bool repeats across the axes;
        None means non-causal everywhere.
    scale: Score scale; None means ``head_dim ** -0.5``. Pass 1.0 for
        pre-scaled queries.

Returns:
    ``(B, T, H, W, num_heads, head_dim)`` attention output.
)FFF)Ú
isinstancer”   r•   r�   r—   r˜   r;   )r‚   rƒ   r„   r²   r³   r†   s   &&&&&&rv   r;   r;     s_   € ô8 �+œs×#Ò#Ø"�m aÕ'ˆØÒÚ)‰	Ü	�Iœt×	$Ò	$Ø�K !•Oˆ	Ü�9‰9×"Ñ"×'Ñ'¨¨a¸iÓOÐOrx   c                ó0  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \        \        ,          ,          R\        \        \        ,          ,          R,          R\
        R,          R\         P                  /# r±   r´   )ru   s   "rv   rw   rw   :  sx   € ÷ ñ Ü‡|�|ðä‡|�|ðô ‡|�|ðô ”tœC•y•ð	ô
 ”dœ4•jÕ  4Õ'ðô �4�<ðô ‡\�\ñrx   c                ó  € \        V\        4      '       d   V.^,          pVf   RR.pM \        V\        4      '       d   V.^,          p\        V4      ^8w  d   \	        R\        V4       24      h\        V4      ^8w  d   \	        R\        V4       24      h\
        P                  P                  P                  V P                  ^4      VP                  ^4      VP                  ^4      ^.VOR.VOV4      pVP                  ^4      # )zÉ2D neighborhood attention over ``(B, H, W, num_heads, head_dim)``
tensors; equivalent to ``na3d`` with a singleton, non-causal T axis.
Bare-scalar ``kernel_size``/``is_causal`` repeat across both axes.Fz+na2d kernel_size must have 2 elements, got z)na2d is_causal must have 2 elements, got )r¶   r”   r•   ÚlenÚ
ValueErrorr�   r—   r˜   r;   Ú	unsqueezeÚsqueeze)r‚   rƒ   r„   r²   r³   r†   Úouts   &&&&&& rv   r:   r:   :  sî   € ô �+œs×#Ò#Ø"�m aÕ'ˆØÒØ˜E�N‰	Ü	�Iœt×	$Ò	$Ø�K !•Oˆ	ä
ˆ;Ó˜1ÔÜÐFÄsÈ;ÓGWÐFXÐYÓZÐZÜ
ˆ9ƒ~˜ÔÜÐDÄSÈÃ^ÐDTÐUÓVÐVÜ
�)‰)×
!Ñ
!×
&Ñ
&Ø	�‰�A‹˜Ÿ™ A›¨¯©°A«Ø	
Ðˆ[Ð˜EÐ. IÐ.°ó€Cð �;‰;�q‹>Ðrx   c                ó<  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \        ,          \        \        \        ,          ,          ,          R\        R,          R\        R\
        R,          R	\         P                  /# )
rp   r‚   rƒ   r„   ÚwindowsÚtemporal_windowNÚclampr†   r�   )r�   r‘   r“   r”   r•   r’   )ru   s   "rv   rw   rw   W  s‹   € ÷ &dñ &dÜ‡|�|ð&dä‡|�|ð&dô ‡|�|ð&dô ”#�Yœœd¤3�i�Õ(ð	&dô
 ˜4•Zð&dô ð&dô �4�<ð&dô ‡\�\ñ&drx   c           
     óÈ  € V'       d!   \        V^ ,          \        4      '       d   V.p\        \        \	        \
        V4      4      4      ^8X  d
   VR,          pV UUu. uF  w  rx\        V4      NK  	  p	ppV UU
u. uF  w  rŠ\        V
4      NK  	  ppp
Vf   VP                  ^,          p\        P                  P                  P                  WW)W´WV4      # u uppi u up
pi )aî  Attention of query planes over local 2D windows of a stack of key planes.

The Tq query planes are the last Tq of the Tk key planes. Query plane i
attends key plane i + Tk - Tq and the ``temporal_window - 1`` before it
(all earlier planes when None). In key plane p a query at (y, x) attends
the ``windows[p]`` = (wh, ww) keys centered on (y, x), shifted inward at
the frame border when ``clamp`` is set (NATTEN) or cut off by it. With
Tq == Tk, one window and clamp this is ``na3d`` with a causal T axis.

Args:
    q: ``(B, Tq, H, W, num_heads, head_dim)``.
    k, v: ``(B, Tk, H, W, kv_heads, head_dim)``, kv_heads dividing
        num_heads (grouped-query attention).
    windows: One (wh, ww) for every key plane, or Tk of them.
    temporal_window: Key planes each query plane attends, counting its own.
    clamp: Shift border windows inward instead of cutting them off.
    scale: Score scale; None means ``head_dim ** -0.5``.

Returns:
    ``(B, Tq, H, W, num_heads, head_dim)`` attention output.
:Né   N)r¶   r”   r¹   ÚsetÚmapÚtupleÚshaper�   r—   r˜   r<   )r‚   rƒ   r„   r¿   rÀ   rÁ   r†   ÚwhÚ_ÚkhÚwwÚkws   &&&&&&&     rv   r<   r<   W  s±   € ÷< ”:˜g a�j¬#×.Ò.Ø�)ˆÜ
Œ3Œs”5˜'Ó"Ó#Ó$¨Ô)Ø˜"•+ˆÙ&Ô	'™w‘e�bŒ#ˆbŽ'™w€BÑ	'Ù&Ô	'™w‘e�aŒ#ˆbŽ'™w€BÑ	'ØÒØŸ'™' !�*ˆÜ�9‰9×"Ñ"×3Ñ3°A¸!ÀÐV[ÓcÐcùó	 
(ùÛ	's   ÁCÁ<Cc          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# ©rp   Úxr†   ÚshiftÚepsr�   ©r�   r‘   r’   )ru   s   "rv   rw   rw   €  sJ   € ÷ ?ñ ?Ü‡|�|ð?ä�<‰<ð?ô �<‰<ð?ô 
ð	?ô
 ‡\�\ñ?rx   c                óV   € \         P                  P                  P                  WW#4      # )aK  Fused Adaptive Layer Normalization: layernorm(x) * (1 + scale) + shift.

Args:
    x: Input tensor of any shape (..., D)
    scale: Modulation scale, broadcastable to x's shape
    shift: Modulation shift, broadcastable to x's shape
    eps: LayerNorm epsilon

Returns:
    Normalized and modulated tensor with the same shape as x
)r�   r—   r˜   r-   ©rÏ   r†   rÐ   rÑ   s   &&&&rv   r-   r-   €  s!   € ô" �9‰9×"Ñ"×(Ñ(¨°5Ó>Ð>rx   c                óP  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\        \        \        \        \        3,          ,          R\         P                  R,          R\         P                  /# )	rp   rÏ   ÚweightÚbiasNÚresidualÚstrider½   r�   ©r�   r‘   r”   rÆ   )ru   s   "rv   rw   rw   ”  s…   € ÷ ñ Ü‡|�|ðä�L‰Lðô �,‰,˜Õ
ðô �l‰l˜TÕ!ð	ô
 ”%œœS¤#˜Õ&Õ&ðô 
�‰˜Õ	ðô ‡\�\ñrx   c                ó  € \        V\        4      '       d   V.^,          M
\        V4      pVf,   \        P                  P
                  P                  WW#V4      # \        P                  P
                  P                  WW#WE4       V# )a:  fp16-accumulate conv3d with bias and residual fused into the epilogue.

x [N, C, D, H, W], weight [K, C, T, R, S], zero padding only. Same opt-in
numerics as fp16_linear; shapes the kernel declines run torch's conv. x and out may be
NDHWC-ordered views of larger tensors, so a tiled conv needs no per-tile copies.
)r¶   r”   r“   r�   r—   r˜   r/   Úfp16_conv3d_out)rÏ   rÖ   r×   rØ   rÙ   r½   s   &&&&&&rv   r/   r/   ”  sf   € ô (¨´×4Ò4ˆfˆX˜Ž\¼$¸v»,€FØ
‚{Ü�y‰y×&Ñ&×2Ñ2°1¸dÈfÓUÐUÜ	‡I�I×Ñ×+Ñ+¨A°tÀvÔSØ€Jrx   Fc                óŒ  € V ^8„  d   QhR\         P                  R\         P                  R,          R\         P                  R,          R\        R\        R\        \        \        \        \        \        3,          R\
        R	\
        R
\         P                  R,          R\         P                  R,          R\         P                  /# )rp   rÏ   rÖ   Nr×   Ú
num_groupsrÑ   ÚpadÚsiluÚzero_padr½   Ústatsr�   )r�   r‘   r”   r’   rÆ   r•   )ru   s   "rv   rw   rw   ©  s²   € ÷ ñ Ü‡|�|ðä�L‰L˜4Õðô �,‰,˜Õ
ðô ð	ô
 
ðô 
Œs”Cœœc¤3Ð&Õ	'ðô ðô ðô 
�‰˜Õ	ðô �<‰<˜$Õðô ‡\�\ñrx   c
                óê   € Vf8   \         P                  P                  P                  WW#V\	        V4      WgV	4	      # \         P                  P                  P                  WW#V\	        V4      WgW‰4
       V# )a   Per-frame GroupNorm, SiLU and causal conv3d padding in one pass.

x [B, C, T, H, W]; pad is (left, right, top, bottom, front). The spatial border
reflects, or is zero when zero_pad; the front frames are always zero. weight=None is
pad-only. stats from group_norm_stats3d normalizes with those per-frame statistics
instead of x's own, so x may be a row strip of the frame they describe. Output is
channels_last_3d, into out when given, which is returned; for a batch of one out may
be a frame-offset view, leaving room for a caller's real halo.
)r�   r—   r˜   r0   r“   Úgroup_norm_silu_pad3d_out)
rÏ   rÖ   r×   rÞ   rÑ   rß   rà   rá   r½   râ   s
   &&&&&&&&&&rv   r0   r0   ©  sh   € ð* ‚{Ü�y‰y×&Ñ&×<Ñ<Ø�t¨¬d°3«i¸ÈóPð 	Pä	‡I�I×Ñ×5Ñ5Ø	�4 S¬$¨s«)°TÀSôQà€Jrx   c                óp   € V ^8„  d   QhR\         P                  R\        R\        R\         P                  /# )rp   rÏ   rÞ   rÑ   r�   )r�   r‘   r”   r’   )ru   s   "rv   rw   rw   Æ  s9   € ÷ Jñ Jœ%Ÿ,™,ð J´Cð JÄ5ð JÔTY×T`ÑT`ñ Jrx   c                óV   € \         P                  P                  P                  WV4      # )zŽPer-frame GroupNorm statistics of x [B, C, T, H, W]: float32 [B*T, num_groups, 2]
of (mean, rstd), for group_norm_silu_pad3d's stats argument.)r�   r—   r˜   r1   )rÏ   rÞ   rÑ   s   &&&rv   r1   r1   Æ  s!   € ô �9‰9×"Ñ"×5Ñ5°aÀSÓIÐIrx   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# rÎ   rÒ   )ru   s   "rv   rw   rw   Ì  sQ   € ÷ Cñ CÜ‡|�|ðCä�<‰<ðCô �<‰<ðCô 
ð	Cô
 ‡\�\ñCrx   c                óV   € \         P                  P                  P                  WW#4      # )ué  Fused Adaptive Layer Normalization with RMSNorm: rmsnorm(x) * (1 + scale) + shift.

Same modulation as :func:`adaln`, but normalizing by the root mean square
instead of subtracting the mean â€” the form used by LTX/LTX2-style DiTs.

Args:
    x: Input tensor of any shape (..., D)
    scale: Modulation scale, broadcastable to x's shape
    shift: Modulation shift, broadcastable to x's shape
    eps: RMSNorm epsilon

Returns:
    Normalized and modulated tensor with the same shape as x
)r�   r—   r˜   r2   rÔ   s   &&&&rv   r2   r2   Ì  s!   € ô( �9‰9×"Ñ"×,Ñ,¨Q°uÓBÐBrx   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# ©rp   rÏ   r†   Úoutput_typer�   ©r�   r‘   Údtype)ru   s   "rv   rw   rw   ã  sF   € ÷ Fñ FÜ‡|�|ðFä�<‰<ðFô —‘ðFô ‡\�\ñ	Frx   c                óp   € \         V,          p\        P                  P                  P	                  WV4      # )zÐQuantize tensor to FP8 format with per-tensor scaling.

Args:
    x: Input tensor
    scale: Scale tensor (scalar)
    output_type: FP8 dtype (float8_e4m3fn or float8_e5m2)

Returns:
    Quantized FP8 tensor
)r
   r�   r—   r˜   Úquantize_fp8©rÏ   r†   rë   Ú
dtype_codes   &&& rv   r?   r?   ã  s+   € ô ˜{Õ+€JÜ�9‰9×"Ñ"×/Ñ/°¸*ÓEÐErx   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# rê   rì   )ru   s   "rv   rw   rw   ö  sF   € ÷ Hñ HÜ‡|�|ðHä�<‰<ðHô —‘ðHô ‡\�\ñ	Hrx   c                óp   € \         V,          p\        P                  P                  P	                  WV4      # )a  Dequantize tensor from FP8 format with per-tensor scaling.

Args:
    x: Input FP8 tensor (float8_e4m3fn or float8_e5m2)
    scale: Scale tensor (scalar)
    output_type: Target dtype (float32, float16, or bfloat16)

Returns:
    Dequantized tensor in specified output format
)r
   r�   r—   r˜   Údequantize_fp8rð   s   &&& rv   r@   r@   ö  s+   € ô ˜{Õ+€JÜ�9‰9×"Ñ"×1Ñ1°!¸JÓGÐGrx   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# )rp   rÏ   Úrngrë   r�   rì   )ru   s   "rv   rw   rw   	  s@   € ÷ ñ Ü‡|�|ðä	�‰ðô —‘ðô ‡\�\ñ	rx   c                óR   € RV RVRV/p\         P                  ! RVR7      pV! R/ VB # )zæStochastically round tensor to FP8 format.

Args:
    x: Input tensor
    rng: Random uint8 tensor with the same shape as x
    output_type: FP8 dtype (float8_e4m3fn or float8_e5m2)

Returns:
    Stochastically rounded FP8 tensor
rÏ   rö   rë   rm   rz   r{   r|   )rÏ   rö   rë   rr   r~   s   &&&  rv   rm   rm   	  s5   € ð �1�e˜S -°Ð=€FÜ×&Ò&Ð'@ÈÔP€DÙ‰>�&‰>Ðrx   c                óÒ   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\        R\        \         P                  \         P                  3,          /# )rp   rÏ   Úper_tensor_scaleÚepsilonÚpad_16xÚhi_firstr�   ©r�   r‘   r’   r•   rÆ   )ru   s   "rv   rw   rw     sf   € ÷ cñ cÜ‡|�|ðcä—l‘lðcô ðcô ð	cô
 ðcô Œ5�<‰<œŸ™Ð%Õ&ñcrx   c                óX   € \         P                  P                  P                  WW#V4      # )a   Quantize tensor to NVFP4 format with block-wise scaling.

Args:
    x: Input tensor (2D)
    per_tensor_scale: Global scale factor
    epsilon: Epsilon for numerical stability
    pad_16x: If True, implicit zero-padding is applied to make dimensions divisible by 16
    hi_first: Nibble packing order. If True (default), the even-indexed element
              is stored in the high nibble of each packed byte. If False, the
              even-indexed element is stored in the low nibble.

Returns:
    Tuple of (quantized_tensor, block_scales)
)r�   r—   r˜   rA   )rÏ   rù   rú   rû   rü   s   &&&&&rv   rA   rA     s$   € ô* �9‰9×"Ñ"×1Ñ1°!ÀwÐYaÓbÐbrx   c                óÄ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  /# )rp   Úqxrù   Úblock_scalesrë   rü   r�   )r�   r‘   rí   r•   )ru   s   "rv   rw   rw   5  s`   € ÷ nñ nÜ�‰ðnä—l‘lðnô —,‘,ðnô —‘ð	nô
 ðnô ‡\�\ñnrx   c                ór   € \         V,          p\        P                  P                  P	                  WW%V4      # )a  Dequantize tensor from NVFP4 format with block-wise scaling.

Args:
    qx: Quantized FP4 tensor (packed as uint8)
    per_tensor_scale: Global scale factor
    block_scales: Block scales in swizzled layout (float8_e4m3fn)
    output_type: Target output dtype (float32, float16, or bfloat16)
    hi_first: Nibble packing order. Must match the packing order used
              during quantization. If True (default), the even-indexed
              element is in the high nibble.

Returns:
    Dequantized tensor in specified output format
)r
   r�   r—   r˜   rB   )r   rù   r  rë   rü   rñ   s   &&&&& rv   rB   rB   5  s.   € ô* ˜{Õ+€JÜ�9‰9×"Ñ"×3Ñ3°BÈ,ÐdlÓmÐmrx   c                ó‚  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\         P                  R,          R
\         P                  R,          R\         P                  /
# )rp   ÚaÚbÚtensor_scale_aÚtensor_scale_bÚblock_scale_aÚblock_scale_br×   NÚ	out_dtypeÚalphar�   rì   )ru   s   "rv   rw   rw   N  s£   € ÷ #ñ #Ü‡|�|ð#ä‡|�|ð#ô —L‘Lð#ô —L‘Lð	#ô
 —<‘<ð#ô —<‘<ð#ô �,‰,˜Õ
ð#ô �{‰{˜TÕ!ð#ô �<‰<˜$Õð#ô ‡\�\ñ#rx   c	                óž   € Vf   \         P                  p\        V,          p	\         P                  P                  P                  WW#WEWiV4	      # )aB  Matrix multiplication with NVFP4 quantized inputs.

Computes: y = (a @ b.T) * (tensor_scale_a * tensor_scale_b) + bias

Args:
    a: Quantized matrix A (M, K//2) in uint8 format
    b: Quantized matrix B (N, K//2) in uint8 format
    tensor_scale_a: Global scale for A
    tensor_scale_b: Global scale for B
    block_scale_a: Block-wise scales for A
    block_scale_b: Block-wise scales for B
    bias: Optional bias vector
    out_dtype: Output dtype (defaults to bfloat16)
    alpha: Output scale (tensor_scale_a * tensor_scale_b)

Returns:
    Result tensor of shape (M, N)
)r�   Úbfloat16r
   r—   r˜   rI   )
r  r  r  r  r  r	  r×   r
  r  rñ   s
   &&&&&&&&& rv   rI   rI   N  sF   € ð: ÒÜ—N‘Nˆ	Ü˜yÕ)€JÜ�9‰9×"Ñ"×2Ñ2Ø	ˆnØ d¸óð rx   c                óš   € V ^8„  d   QhR\         P                  R\        R\        \         P                  \         P                  3,          /# )rp   rÏ   Úpad_32xr�   )r�   r‘   r•   rÆ   )ru   s   "rv   rw   rw   t  s<   € ÷ >ñ >Ü‡|�|ð>äð>ô Œ5�<‰<œŸ™Ð%Õ&ñ>rx   c                óT   € \         P                  P                  P                  W4      # )a³  Quantize tensor to MXFP8 format with block-wise E8M0 scaling.

MXFP8 uses block size 32 with power-of-2 (E8M0) block scales.

Args:
    x: Input tensor (2D, shape M x K, K must be divisible by 32)
    pad_32x: If True, pad dimensions to be divisible by 32

Returns:
    Tuple of (quantized_fp8_tensor, block_scales_e8m0)
    - quantized_fp8_tensor: FP8 E4M3 data of shape (M, K)
    - block_scales_e8m0: E8M0 scales in swizzled layout
)r�   r—   r˜   rC   )rÏ   r  s   &&rv   rC   rC   t  s   € ô" �9‰9×"Ñ"×1Ñ1°!Ó=Ð=rx   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# )rp   r   r  rë   r�   rì   )ru   s   "rv   rw   rw   ˆ  sF   € ÷ Rñ RÜ�‰ðRä—,‘,ðRô —‘ðRô ‡\�\ñ	Rrx   c                óp   € \         V,          p\        P                  P                  P	                  WV4      # )a"  Dequantize tensor from MXFP8 format.

Args:
    qx: Quantized FP8 tensor (float8_e4m3fn)
    block_scales: E8M0 block scales in swizzled layout (float8_e8m0fnu)
    output_type: Target output dtype (float32, float16, or bfloat16)

Returns:
    Dequantized tensor in specified output format
)r
   r�   r—   r˜   rD   )r   r  rë   rñ   s   &&& rv   rD   rD   ˆ  s+   € ô ˜{Õ+€JÜ�9‰9×"Ñ"×3Ñ3°BÀjÓQÐQrx   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  /# )	rp   r  r  r  r	  r×   Nr
  r�   rì   )ru   s   "rv   rw   rw   ›  st   € ÷ ñ Ü‡|�|ðä‡|�|ðô —<‘<ðô —<‘<ð	ô
 �,‰,˜Õ
ðô �{‰{˜TÕ!ðô ‡\�\ñrx   c                óš   € Vf   \         P                  p\        V,          p\         P                  P                  P                  WW#WF4      # )aœ  Matrix multiplication with MXFP8 quantized inputs.

Computes: y = a @ b.T + bias

Args:
    a: Quantized FP8 matrix A (M, K)
    b: Quantized FP8 matrix B (N, K)
    block_scale_a: E8M0 block scales for A in swizzled layout
    block_scale_b: E8M0 block scales for B in swizzled layout
    bias: Optional bias vector
    out_dtype: Output dtype (defaults to bfloat16)

Returns:
    Result tensor of shape (M, N)
)r�   r  r
   r—   r˜   rJ   )r  r  r  r	  r×   r
  rñ   s   &&&&&& rv   rJ   rJ   ›  sA   € ð. ÒÜ—N‘Nˆ	Ü˜yÕ)€JÜ�9‰9×"Ñ"×2Ñ2Ø	ˆm¨Dóð rx   c                ó2  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R\         P                  R,          R\        \         P                  \         P                  \         P                  3,          /# )	rp   rÏ   ÚsmoothÚ	lora_downÚpad_sizeÚact_unsignedÚlora_xNr�   )r�   r‘   r”   r•   rÆ   )ru   s   "rv   rw   rw   º  s{   € ÷ ñ Ü‡|�|ðä�L‰Lðô �|‰|ðô ð	ô
 ðô �L‰L˜4Õðô Œ5�<‰<œŸ™¤u§|¡|Ð3Õ4ñrx   c                óX   € \         P                  P                  P                  WW#WE4      # )u÷  Quantize activations to int4 with smoothing + LoRA down projection.

Args:
    x: (M, K) bf16/fp16 main-path input (caller pre-shifts if unsigned path).
    smooth: (K,) smoothing factor applied before quantization.
    lora_down: (K, R) low-rank down projection weight.
    pad_size: pad M to multiple of this value (default 256).
    act_unsigned: if True, quantize into uint4 [0, 15] (scale=max/15) for u4
        MMA downstream. Caller must ensure x is non-negative â€” the shift
        constant is a model-topology concern, not part of this op.
    lora_x: (M, K) optional pre-shift activation for LoRA. Defaults to x.
        Pass raw (un-shifted) x when x has been pre-shifted for unsigned path.

Returns:
    (quantized_x uint8 [M_pad, K//2], ascales [K//64, M_pad], lora_act [M_pad, R])

Note: eager returns fp32 lora_act as a high-precision reference. The CUDA
backend returns lora_act in x.dtype because the runtime epilogue consumes it
as bf16/fp16; this avoids an otherwise redundant cast/allocation.
)r�   r—   r˜   rE   )rÏ   r  r  r  r  r  s   &&&&&&rv   rE   rE   º  s(   € ô8 �9‰9×"Ñ"×9Ñ9Ø	�9¨óð rx   c                ó2  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\        R
\         P                  /	# )rp   ÚactÚwgtÚascalesÚwscalesÚlora_act_inÚlora_upr×   Nr  r�   )r�   r‘   r•   )ru   s   "rv   rw   rw   Û  s‡   € ÷ $ñ $Ü	�‰ð$ä	�‰ð$ô �\‰\ð$ô �\‰\ð	$ô
 —‘ð$ô �\‰\ð$ô �,‰,˜Õ
ð$ô ð$ô ‡\�\ñ$rx   c           
     óZ   € \         P                  P                  P                  WW#WEWg4      # )aF  SVDQuant W4A4 int4 GEMM + LoRA-up + bias.

Computes out = int4_matmul(act, wgt, ascales, wscales) + lora_act_in @ lora_up^T + bias.
The CUDA backend performs int4 MMA + per-group dequant + bias in one
kernel and, when lora_act_in/proj_up layout and dtype allow it, fuses
LoRA-up into the same writeback epilogue with bf16/fp16 tensor-core MMA.
Unsupported combinations fall back to the wrapper's bf16/fp16 addmm_ path.

Args:
    act: (M, K//2) uint8 packed activations from quantize_svdquant_w4a4.
    wgt: (N, K//2) int8 packed weights (natural row-major), or backend
        specific tile-packed storage.
    ascales: (K//64, M) activation scales.
    wscales: (K//64, N) weight scales.
    lora_act_in: (M, R) LoRA activations from quantize step.
    lora_up: (N, R) LoRA up projection weight, or matching tile-packed
        storage for tile-packed weights.
    bias: optional (N,) bias.
    act_unsigned: if True, activations are interpreted as unsigned [0,15] by
        u4.s4 MMA (for post-GELU+shift fc2). Caller pre-shifts.

Returns:
    (M, N) output tensor (same dtype as lora_up).
)r�   r—   r˜   rK   )r  r  r  r   r!  r"  r×   r  s   &&&&&&&&rv   rK   rK   Û  s+   € ôD �9‰9×"Ñ"×:Ñ:Ø�' K¸$óð rx   c                óò   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\         P                  /# )	rp   rÏ   Úqweightr   Úwzerosr×   NÚ
group_sizer�   ©r�   r‘   r”   )ru   s   "rv   rw   rw     sk   € ÷ ñ Ü‡|�|ðä�\‰\ðô �\‰\ðô �L‰Lð	ô
 �,‰,˜Õ
ðô ðô ‡\�\ñrx   c                óX   € \         P                  P                  P                  WW#WE4      # )a|  AWQ W4A16 quantized GEMV (for modulation-style layers called with small batch).

Args:
    x: (..., K) bf16/fp16 input.
    qweight: (N//4, K//2) int32 packed weight.
    wscales: (K//group_size, N) per-group scales.
    wzeros: (K//group_size, N) per-group zero points.
    bias: optional (N,) bias.
    group_size: quantization group size.

Returns:
    (..., N) output tensor.
)r�   r—   r˜   rL   )rÏ   r%  r   r&  r×   r'  s   &&&&&&rv   rL   rL     s(   € ô* �9‰9×"Ñ"×1Ñ1Ø	�G Tóð rx   c          
      óÎ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# ©rp   ÚxqÚxkÚ	freqs_cisr�   ©r�   r‘   rÆ   )ru   s   "rv   rw   rw     sT   € ÷ Añ AÜ�‰ðAä�‰ðAô �|‰|ðAô Œ5�<‰<œŸ™Ð%Õ&ñ	Arx   c                óV   € \         P                  P                  P                  WV4      # )a  Apply Rotary Position Embedding (RoPE) to query and key tensors.

Interleaved layout: pair k uses adjacent elements [2k, 2k+1].

Args:
    xq: Query tensor
    xk: Key tensor
    freqs_cis: Precomputed frequency tensor

Returns:
    Tuple of (transformed_query, transformed_key)
)r�   r—   r˜   rQ   ©r,  r-  r.  s   &&&rv   rQ   rQ     s!   € ô" �9‰9×"Ñ"×-Ñ-¨b°iÓ@Ð@rx   c          
      óÎ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# r+  r/  )ru   s   "rv   rw   rw   0  óJ   € ÷ ñ Ü�‰ðÜŸ,™,ðÜ38·<±<ðä
Œ5�<‰<œŸ™Ð%Õ&ñrx   c                ó\   € \         P                  P                  P                  WV4       W3# ©z1Apply interleaved RoPE in place (inference only).)r�   r—   r˜   rR   r1  s   &&&rv   rR   rR   0  s%   € ô 
‡I�I×Ñ×'Ñ'¨°	Ô:Øˆ6€Mrx   c                ó(  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        \         P                  \         P                  3,          /# ©	rp   r‚   rƒ   r.  Úq_scaleÚk_scaleNrú   r�   ©r�   r‘   r’   rÆ   )ru   s   "rv   rw   rw   8  s‚   € ÷ Xñ XÜ‡|�|ðXä‡|�|ðXô �|‰|ðXô �\‰\ð	Xô
 �\‰\˜DÕ ðXô ðXô Œ5�<‰<œŸ™Ð%Õ&ñXrx   c                óX   € \         P                  P                  P                  WW#WE4      # )a  Apply per-head RMSNorm followed by interleaved RoPE to query and key tensors.

Interleaved layout: pair k uses adjacent elements [2k, 2k+1].

Args:
    q: Query tensor.
    k: Key tensor. Its head count may differ from q for grouped-query attention.
    freqs_cis: Precomputed frequency tensor.
    q_scale: Per-dimension RMSNorm scale for q.
    k_scale: Optional per-dimension RMSNorm scale for k. Defaults to q_scale.
    epsilon: RMSNorm numerical-stability epsilon.

Returns:
    Tuple of normalized and rotated (query, key) tensors.
)r�   r—   r˜   rY   ©r‚   rƒ   r.  r8  r9  rú   s   &&&&&&rv   rY   rY   8  s#   € ô. �9‰9×"Ñ"×+Ñ+¨A°)ÀgÓWÐWrx   c                ó(  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        \         P                  \         P                  3,          /# r7  r:  )ru   s   "rv   rw   rw   R  ss   € ÷ ñ Ü‡|�|ðÜŸ™ðÜ16·±ðä�\‰\ðä$)§L¡L°4Õ$7ðô ðô Œ5�<‰<œŸ™Ð%Õ&ñ	rx   c                ó^   € \         P                  P                  P                  WW#WE4       W3# ©z=Apply RMSNorm and interleaved RoPE in place (inference only).)r�   r—   r˜   rZ   r<  s   &&&&&&rv   rZ   rZ   R  s'   € ô 
‡I�I×Ñ×%Ñ% a¨IÀÔQØˆ4€Krx   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# ©rp   rÏ   r.  r†   rú   r�   rÒ   )ru   s   "rv   rw   rw   \  sQ   € ÷ Kñ KÜ‡|�|ðKä�|‰|ðKô �<‰<ðKô ð	Kô
 ‡\�\ñKrx   c                óV   € \         P                  P                  P                  WW#4      # )zGApply per-head RMSNorm followed by interleaved RoPE to a single tensor.)r�   r—   r˜   r[   ©rÏ   r.  r†   rú   s   &&&&rv   r[   r[   \  s!   € ô �9‰9×"Ñ"×,Ñ,¨Q¸5ÓJÐJrx   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# rA  rÒ   )ru   s   "rv   rw   rw   f  óF   € ÷ ñ Ü‡|�|ðÜ %§¡ðÜ5:·\±\ðäðô ‡\�\ñrx   c                óZ   € \         P                  P                  P                  WW#4       V # r?  )r�   r—   r˜   r\   rC  s   &&&&rv   r\   r\   f  s#   € ô
 
‡I�I×Ñ×&Ñ& q°UÔDØ€Hrx   c                ó4  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R	\        \         P                  \         P                  3,          /# ©
rp   r‚   rƒ   r.  r8  r9  Nrú   Úrot_dimr�   ©r�   r‘   r’   r”   rÆ   )ru   s   "rv   rw   rw   o  s�   € ÷ lñ lÜ‡|�|ðlä‡|�|ðlô �|‰|ðlô �\‰\ð	lô
 �\‰\˜DÕ ðlô ðlô ðlô Œ5�<‰<œŸ™Ð%Õ&ñlrx   c           	     óZ   € \         P                  P                  P                  WW#WEV4      # )a  Apply per-head RMSNorm and split-half RoPE to query and key tensors.

Split-half layout: pair i uses elements [i] and [i + rot_dim//2]. rot_dim
restricts the rotation to a head-dim prefix (partial rotary; the norm
always spans the full head_dim); 0 rotates everything.
)r�   r—   r˜   r]   ©r‚   rƒ   r.  r8  r9  rú   rI  s   &&&&&&&rv   r]   r]   o  s'   € ô �9‰9×"Ñ"×6Ñ6°q¸YÐQXÐcjÓkÐkrx   c                ó4  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R	\        \         P                  \         P                  3,          /# rH  rJ  )ru   s   "rv   rw   rw   �  s}   € ÷ ñ Ü‡|�|ðÜŸ™ðÜ16·±ðä�\‰\ðä$)§L¡L°4Õ$7ðô ðô %(ðô Œ5�<‰<œŸ™Ð%Õ&ñ	rx   c           	     ó`   € \         P                  P                  P                  WW#WEV4       W3# )zÃApply RMSNorm and split-half RoPE in place (inference only).

rot_dim restricts the rotation to a head-dim prefix (partial rotary; the
norm always spans the full head_dim); 0 rotates everything.
)r�   r—   r˜   r^   rL  s   &&&&&&&rv   r^   r^   �  s.   € ô 
‡I�I×Ñ×0Ñ0Ø	ˆi '°Gôð ˆ4€Krx   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# rA  rÒ   )ru   s   "rv   rw   rw   ‘  sQ   € ÷ 
Vñ 
VÜ‡|�|ð
Vä�|‰|ð
Vô �<‰<ð
Vô ð	
Vô
 ‡\�\ñ
Vrx   c                óV   € \         P                  P                  P                  WW#4      # )z‹Apply per-head RMSNorm followed by split-half RoPE to a single tensor.

Split-half layout: pair k uses elements [k] and [k + head_dim//2].
)r�   r—   r˜   r_   rC  s   &&&&rv   r_   r_   ‘  s!   € ô �9‰9×"Ñ"×7Ñ7¸ÀeÓUÐUrx   c          
      ó¤   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\         P                  /# rA  rÒ   )ru   s   "rv   rw   rw   ž  rE  rx   c                óZ   € \         P                  P                  P                  WW#4       V # )z<Apply RMSNorm and split-half RoPE in place (inference only).)r�   r—   r˜   r`   rC  s   &&&&rv   r`   r`   ž  s#   € ô
 
‡I�I×Ñ×1Ñ1°!ÀÔOØ€Hrx   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# ©rp   rÏ   r.  r�   rœ   )ru   s   "rv   rw   rw   §  s2   € ÷ =ñ =Ü‡|�|ð=ä�|‰|ð=ô ‡\�\ñ=rx   c                óT   € \         P                  P                  P                  W4      # )zâApply Rotary Position Embedding (RoPE) to a single tensor.

Interleaved layout: pair k uses adjacent elements [2k, 2k+1].

Args:
    x: Input tensor
    freqs_cis: Precomputed frequency tensor

Returns:
    Transformed tensor
)r�   r—   r˜   rS   ©rÏ   r.  s   &&rv   rS   rS   §  s   € ô �9‰9×"Ñ"×.Ñ.¨qÓ<Ð<rx   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# rT  rœ   )ru   s   "rv   rw   rw   ¹  s-   € ÷ ñ ”E—L‘Lð ¬U¯\©\ð ¼e¿l¹lñ rx   c                óX   € \         P                  P                  P                  W4       V # r5  )r�   r—   r˜   rT   rV  s   &&rv   rT   rT   ¹  s   € ä	‡I�I×Ñ×(Ñ(¨Ô6Ø€Hrx   c          
      óÎ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# r+  r/  )ru   s   "rv   rw   rw   ¿  sT   € ÷ Lñ LÜ�‰ðLä�‰ðLô �|‰|ðLô Œ5�<‰<œŸ™Ð%Õ&ñ	Lrx   c                óV   € \         P                  P                  P                  WV4      # )a"  Apply Rotary Position Embedding (RoPE) to query and key tensors.

Split-half layout: pair k uses elements [k] and [k + head_dim//2].
Matches the formula:
    t_ = t.reshape(*t.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2).to(freqs.dtype)
    t_out = freqs[..., 0] * t_[..., 0] + freqs[..., 1] * t_[..., 1]
    t_out.movedim(-1, -2).reshape(*t.shape).type_as(t)

Args:
    xq: Query tensor
    xk: Key tensor
    freqs_cis: Precomputed frequency tensor shape (..., head_dim//2, 2, 2)

Returns:
    Tuple of (transformed_query, transformed_key)
)r�   r—   r˜   rU   r1  s   &&&rv   rU   rU   ¿  s!   € ô* �9‰9×"Ñ"×8Ñ8¸ÀÓKÐKrx   c          
      óÎ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# r+  r/  )ru   s   "rv   rw   rw   ×  r3  rx   c                ó\   € \         P                  P                  P                  WV4       W3# ©z0Apply split-half RoPE in place (inference only).)r�   r—   r˜   rV   r1  s   &&&rv   rV   rV   ×  s%   € ô 
‡I�I×Ñ×2Ñ2°2¸9ÔEØˆ6€Mrx   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# rT  rœ   )ru   s   "rv   rw   rw   ß  s7   € ÷ Hñ HÜ‡|�|ðHä�|‰|ðHô ‡\�\ñHrx   c                óT   € \         P                  P                  P                  W4      # )aí  Apply Rotary Position Embedding (RoPE) to a single tensor.

Split-half layout: pair k uses elements [k] and [k + head_dim//2].
Matches the formula:
    t_ = t.reshape(*t.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2).to(freqs.dtype)
    t_out = freqs[..., 0] * t_[..., 0] + freqs[..., 1] * t_[..., 1]
    t_out.movedim(-1, -2).reshape(*t.shape).type_as(t)

Args:
    x: Input tensor
    freqs_cis: Precomputed frequency tensor shape (..., head_dim//2, 2, 2)

Returns:
    Transformed tensor
)r�   r—   r˜   rW   rV  s   &&rv   rW   rW   ß  s   € ô& �9‰9×"Ñ"×9Ñ9¸!ÓGÐGrx   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# rT  rœ   )ru   s   "rv   rw   rw   õ  s0   € ÷ ñ Ü‡|�|ðÜ %§¡ðä
‡\�\ñrx   c                óX   € \         P                  P                  P                  W4       V # r]  )r�   r—   r˜   rX   rV  s   &&rv   rX   rX   õ  s!   € ô 
‡I�I×Ñ×3Ñ3°AÔAØ€Hrx   c          
      ó  € V ^8„  d   QhR\         P                  R\         P                  \        ,          \        ,          R,          R\        R,          R\
        \         P                  \         P                  3,          /# )rp   rÏ   r†   NÚstochastic_roundingr�   )r�   r‘   r’   rs   r”   rÆ   )ru   s   "rv   rw   rw   ý  s]   € ÷ ñ Ü‡|�|ðä�<‰<œ%Õ¤#Õ%¨Õ,ðô ˜t�ðô Œ5�<‰<œŸ™Ð%Õ&ñ	rx   c                óR   € RV RVRV/p\         P                  ! RVR7      pV! R/ VB # )z5Quantize tensor to INT8 with single tensorwise scale.rÏ   r†   rc  rG   rz   r{   r|   )rÏ   r†   rc  rr   r~   s   &&&  rv   rG   rG   ý  s7   € ð �1�g˜uÐ&;Ð=PÐQ€FÜ×&Ò&Ð'AÈ&ÔQ€DÙ‰>�&‰>Ðrx   c                ó¨   € V ^8„  d   QhR\         P                  R\        R,          R\        \         P                  \         P                  3,          /# )rp   rÏ   rc  Nr�   rÚ   )ru   s   "rv   rw   rw     s@   € ÷ ñ Ü‡|�|ðä˜t�ðô Œ5�<‰<œŸ™Ð%Õ&ñrx   c                óN   € RV RV/p\         P                  ! RVR7      pV! R/ VB # )z,Quantize tensor to INT8 with per-row scales.rÏ   rc  rF   rz   r{   r|   )rÏ   rc  rr   r~   s   &&  rv   rF   rF     s3   € ð
 �1Ð+Ð-@ÐA€FÜ×&Ò&Ð'>ÀvÔN€DÙ‰>�&‰>Ðrx   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )rp   r‚   r†   r�   rœ   )ru   s   "rv   rw   rw     s2   € ÷ Dñ DœeŸl™lð D´5·<±<ð DÄEÇLÁLñ Drx   c                óT   € \         P                  P                  P                  W4      # )z"Dequantize INT8 tensor with scale.)r�   r—   r˜   rH   )r‚   r†   s   &&rv   rH   rH     s   € ä�9‰9×"Ñ"×9Ñ9¸!ÓCÐCrx   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )rp   r  r  r�   rœ   )ru   s   "rv   rw   rw     s-   € ÷ ñ Œu�|‰|ð ¤§¡ð ´·±ñ rx   c                ó   € \        W4      # )z5INT8 matrix multiplication: C[M,N] = A[M,K] @ B[K,N].)Ú_mm_int8)r  r  s   &&rv   r   r     s   € ä�A‹>Ðrx   c                óÄ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  /# )rp   Ú
activationÚnorm_weightrØ   ÚgaterÑ   r�   rÒ   )ru   s   "rv   rw   rw     sX   € ÷ ñ Ü—‘ðä—‘ðô �l‰lðô �,‰,ð	ô
 
ðô ‡\�\ñrx   c                ó.   € \        RRV RVRVRVRV/4      # )z‰Compute ``residual + gate * rms_norm(activation, norm_weight)``.

Each step is rounded to the input dtype as in the unfused PyTorch ops.
r.   rm  rn  rØ   ro  rÑ   )r   )rm  rn  rØ   ro  rÑ   s   &&&&&rv   r.   r.     s2   € ô Øà˜*Ø˜;Ø˜Ø�DØ�3ð	
ó	ð 	rx   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  R,          R\         P                  /# )rp   rÏ   rÖ   r×   NrØ   Úresidual_scaler�   rœ   )ru   s   "rv   rw   rw   3  sk   € ÷ ñ Ü‡|�|ðä�L‰Lðô �,‰,˜Õ
ðô �l‰l˜TÕ!ð	ô
 —L‘L 4Õ'ðô ‡\�\ñrx   c           
     ó
  € \        V P                  RR
 P                  4       VP                  ^ ,          VP                  ^,          4      '       g‰   \        P                  P
                  P                  WV4      pVf   V# Vf   \        R4      h\        P                  ! VP                  VP                  4      WTP                  VP                  4      4      # RV RVRVRVRV/p\        P                  ! RVR	7      pV! R/ VB # )zÔfp16-accumulate linear, optionally with ``residual + residual_scale * out`` fused.

Same numerics as ``torch.backends.cuda.matmul.allow_fp16_accumulation``, so
route here only when the user opted into that mode.
Nz-fp16_linear: residual requires residual_scalerÏ   rÖ   r×   rØ   rr  rM   rz   éÿÿÿÿr{   )Ú_fp16_linear_fills_gpurÇ   Únumelr�   ÚnnÚ
functionalÚlinearrº   ÚaddcmulÚtorí   r!   r}   )rÏ   rÖ   r×   rØ   rr  r½   rr   r~   s   &&&&&   rv   rM   rM   3  sß   € ô " !§'¡'¨#¨2 ,×"4Ñ"4Ó"6¸¿¹ÀQ½ÈÏÉÐVWÍ×YÒYô �h‰h×!Ñ!×(Ñ(¨°DÓ9ˆØÒØˆJØÒ!ÜÐLÓMÐMÜ�}Š}˜XŸ[™[¨¯©Ó3°S×:KÑ:KÈCÏIÉIÓ:VÓWÐWàˆQØ�&Ø�Ø�HØ˜.ð€Fô ×&Ò& }¸VÔD€DÙ‰>�&‰>Ðrx   c                óH   € V ^8„  d   QhR\         R\         R\         R\        /# )rp   ÚmÚnrƒ   r�   )r”   r•   )ru   s   "rv   rw   rw   S  s.   € ÷ Cñ Cœcð C¤cð C¬cð C´dñ Crx   c                óà   € VR8”  d)   V ^,           ^€,          V^,           ^€,          ,          ^ 8¬  # VR8:  g   VR8”  d   RM^€pV ^,           ^€,          W,           ^,
          V,          ,          ^`8¬  # )zhMirror of the launcher's tile gate (cutlass_gemm_fp16.cu), so small
launches skip the dispatch entirely.i   i   i    é   r{   )r}  r~  rƒ   Útile_ns   &&& rv   ru  ru  S  sd   € ð 	ˆ4„xØ�S•˜SÕ  a¨#¥g°#Õ%5Õ6¸"Ñ<Ð<Ø˜”I  T¤‰S¨s€FØ��W˜Õ !¥*¨q¥.°VÕ!;Õ<ÀÑBÐBrx   c                ó¼  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\        R\        R	\
        R,          R
\         P                  R,          R\        R\         P                  R,          R\         P                  R,          R\         P                  /# )rp   rÏ   rÖ   Úweight_scaler×   Nr
  ÚconvrotÚconvrot_groupsizeÚ	input_actÚinput_act_weightÚinput_act_epsrØ   rr  r�   )r�   r‘   rí   r•   r”   rs   r’   )ru   s   "rv   rw   rw   \  sË   € ÷ 9ñ 9Ü‡|�|ð9ä�L‰Lð9ô —,‘,ð9ô �,‰,˜Õ
ð	9ô
 �{‰{˜TÕ!ð9ô ð9ô ð9ô �T�zð9ô —l‘l TÕ)ð9ô ð9ô �l‰l˜TÕ!ð9ô —L‘L 4Õ'ð9ô ‡\�\ñ9rx   c                óž   € Vf   \         P                  pRV RVRVRVRVRVRVRVR	VR
V	RV
RV/p\        P                  ! RVR7      pV! R/ VB # )a^  INT8 linear layer dynamically quantized.

Args:
    x: Input tensor.
    weight: INT8 weight tensor.
    weight_scale: Scalar weight scale.
    bias: Optional bias.
    out_dtype: Output dtype.
    convrot: If True, apply online activation rotation.
    convrot_groupsize: Group size for Hadamard rotation.
    input_act: Optional activation applied to x before quantization
        ("gelu_tanh", "swiglu", "situ", "rms_norm", or None). When
        the fused ConvRot quantizer handles the shape it is folded in, so
        an MLP's ``linear(act(proj(x)))`` or a pre-norm block's
        ``linear(rms_norm(x))`` never writes the intermediate to HBM;
        every other path applies it eagerly for identical results.
    input_act_weight: K-element norm weight, required for "rms_norm".
    input_act_eps: Norm eps for "rms_norm".
    residual: Optional [..., N] tensor; the result becomes
        ``residual + residual_scale * linear(x)`` (a pre-norm block's
        addcmul), fused into the GEMM epilogue where supported.
    residual_scale: Per-channel [N] scale for the residual form.

Returns:
    Result tensor.
rÏ   rÖ   rƒ  r×   r
  r„  r…  r†  r‡  rˆ  rØ   rr  rN   rz   r{   )r�   r  r!   r}   )rÏ   rÖ   rƒ  r×   r
  r„  r…  r†  r‡  rˆ  rØ   rr  rr   r~   s   &&&&&&&&&&&&  rv   rN   rN   \  sy   € ðP ÒÜ—N‘Nˆ	àˆQØ�&Ø˜Ø�Ø�YØ�7ØÐ.Ø�YØÐ,Ø˜Ø�HØ˜.ð€Fô ×&Ò& }¸VÔD€DÙ‰>�&‰>Ðrx   c                ó,  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\        R	\        R
\         P                  /	# )rp   rÏ   Úmodulation_scalerÖ   rƒ  r×   Nr
  r„  r…  r�   )r�   r‘   rí   r•   r”   )ru   s   "rv   rw   rw   ˜  sˆ   € ÷ ñ Ü‡|�|ðä—l‘lðô �L‰Lðô —,‘,ð	ô
 �,‰,˜Õ
ðô �{‰{˜TÕ!ðô ðô ðô ‡\�\ñrx   c                ób   € Vf   \         P                  p\        RRV RVRVRVRVRVRVR	V/4      # )
z2Batch-one modulation with fused INT8 quantization.rO   rÏ   r‹  rÖ   rƒ  r×   r
  r„  r…  ©r�   r  r   )rÏ   r‹  rÖ   rƒ  r×   r
  r„  r…  s   &&&&&&&&rv   rO   rO   ˜  sR   € ð ÒÜ—N‘Nˆ	ÜØà�ØÐ 0Ø�fØ˜LØ�DØ˜Ø�wØÐ!2ð		
óð rx   c                óX  € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\         P                  R,          R
\        R\
        R\         P                  /# )rp   rÏ   rn  Únorm_epsr‹  rÖ   rƒ  r×   Nr
  r„  r…  r�   )r�   r‘   r’   rí   r•   r”   )ru   s   "rv   rw   rw   ´  s    € ÷ ñ Ü‡|�|ðä—‘ðô ðô —l‘lð	ô
 �L‰Lðô —,‘,ðô �,‰,˜Õ
ðô �{‰{˜TÕ!ðô ðô ðô ‡\�\ñrx   c
                ój   € Vf   \         P                  p\        RRV RVRVRVRVRVRVR	VR
VRV	/
4      # )zFINT8 projection with RMSNorm and batch-one modulation in its producer.rP   rÏ   rn  r�  r‹  rÖ   rƒ  r×   r
  r„  r…  r�  )
rÏ   rn  r�  r‹  rÖ   rƒ  r×   r
  r„  r…  s
   &&&&&&&&&&rv   rP   rP   ´  s\   € ð ÒÜ—N‘Nˆ	ÜØ#à�Ø˜;Ø˜ØÐ 0Ø�fØ˜LØ�DØ˜Ø�wØÐ!2ð	
óð rx   c                ó¬   € V ^8„  d   QhR\         P                  R\        \         P                  \         P                  \         P                  3,          /# )rp   Úpointsr�   r/  )ru   s   "rv   rw   rw   Ô  s6   € ÷ ñ ”u—|‘|ð ¬¬e¯l©l¼E¿L¹LÌ%Ï,É,Ð.VÕ(Wñ rx   c                óJ   € RV /p\         P                  ! RVR7      pV! R/ VB # )aö  3D Delaunay tetrahedralization of distinct points (N, 3) float32/float64.

Returns (vertices (N + 4, 3) float64, tets (M, 4) int32, neighbors (M, 4) int32).
Vertices N..N+3 are the corners of a large bounding tet; tets using them fill the space
outside the convex hull. Tets satisfy (v1 - v0) . ((v2 - v0) x (v3 - v0)) > 0, and
neighbors[i, k] is the tet across the face opposite vertex k, or -1 on the bounding
tet's outer faces.

Like qhull's QJ, points are joggled by up to 1e-10 of their extent or largest
coordinate, whichever is larger, so no four are exactly coplanar; the returned vertices
are the joggled ones. A small fraction of faces may stay non-Delaunay, mostly on nearly
cospherical input, but the result is always a valid tetrahedralization.
r’  ra   rz   r{   r|   )r’  rr   r~   s   &  rv   ra   ra   Ô  s,   € ð ˜Ð€FÜ×&Ò& |¸FÔC€DÙ‰>�&‰>Ðrx   c                ó8   € V ^8„  d   QhR\         P                  /# )rp   Ú	trianglesrœ   )ru   s   "rv   rw   rw   ç  s   € ÷ ñ œŸ™ñ rx   c                óJ   € RV /p\         P                  ! RVR7      pV! R/ VB # )zÃLinear BVH over triangles (F, 3, 3) float32 for closest_point_on_mesh and closest_hit_on_mesh.

Returns a reusable tuple (triangles in Morton order, their original ids, child links,
node boxes).
r•  rb   rz   r{   r|   )r•  rr   r~   s   &  rv   rb   rb   ç  s,   € ð ˜9Ð%€FÜ×&Ò& z¸&ÔA€DÙ‰>�&‰>Ðrx   Úinfc          	      ó¸   € V ^8„  d   QhR\         P                  R\        R\        \         P                  \         P                  \         P                  3,          /# )rp   r’  Úmax_distr�   r:  )ru   s   "rv   rw   rw   ò  s@   € ÷ ñ Ü—‘ðÜ).ðä
Œ5�<‰<œŸ™¤u§|¡|Ð3Õ4ñrx   c                óR   € RV RVRV/p\         P                  ! RVR7      pV! R/ VB # )a$  Exact closest point on the `bvh` mesh to each of points (N, 3) float32.

Returns (distance (N,) float32, closest point (N, 3) float32, triangle id (N,) int64
into the triangles given to mesh_bvh). Points with nothing within `max_dist` get
distance max_dist, the point itself and triangle -1.
Úbvhr’  r™  rc   rz   r{   r|   )r›  r’  r™  rr   r~   s   &&&  rv   rc   rc   ò  s5   € ð �S˜( F¨J¸ÐA€FÜ×&Ò&Ð'>ÀvÔN€DÙ‰>�&‰>Ðrx   c                óÒ   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\        R\        \         P                  \         P                  3,          /# )rp   ÚoriginsÚ
directionsÚtminÚtmaxÚcull_backfacer�   rý   )ru   s   "rv   rw   rw      sX   € ÷ ñ Ü—,‘,ðÜ,1¯L©LðÜ@EðÜSXðäðô Œ5�<‰<œŸ™Ð%Õ&ñrx   c                ó^   € RV RVRVRVRVRV/p\         P                  ! RVR7      pV! R	/ VB # )
au  Nearest hit on the `bvh` mesh of each ray origins + t * directions, (N, 3) float32, for t in (tmin, tmax).

Triangles are double-sided unless cull_backface, which keeps those whose winding normal faces the ray; edges
count as inside. Returns (t (N,) float32, triangle id (N,) int64 into the triangles given to mesh_bvh); rays
that hit nothing get t = tmax and triangle -1.
r›  r�  rž  rŸ  r   r¡  rd   rz   r{   r|   )r›  r�  rž  rŸ  r   r¡  rr   r~   s   &&&&&&  rv   rd   rd      sG   € ð �S˜) W¨l¸JÈÐPTÐV\Ð^bØ˜}ð.€Fä×&Ò&Ð'<ÀVÔL€DÙ‰>�&‰>Ðrx   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )rp   ÚedgesÚ	num_nodesr�   r(  )ru   s   "rv   rw   rw     s)   € ÷ ñ ¤§¡ð ¼ð ÄÇÁñ rx   c                óN   € RV RV/p\         P                  ! RVR7      pV! R/ VB # )z«Connected components of an undirected graph with edges (E, 2) int32/int64 in
[0, num_nodes). Returns (num_nodes,) int64: the smallest node index in each node's
component.
r¤  r¥  re   rz   r{   r|   )r¤  r¥  rr   r~   s   &&  rv   re   re     s1   € ð
 �u˜k¨9Ð5€FÜ×&Ò&Ð'=ÀfÔM€DÙ‰>�&‰>Ðrx   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        R\        \         P                  \         P                  \         P                  3,          /# )rp   ÚverticesÚfacesr¤  Ú	positionsÚcos_thresholdr�   r:  )ru   s   "rv   rw   rw     sl   € ÷ ñ Ü�l‰lðä�<‰<ðô �<‰<ðô �|‰|ð	ô
 ðô Œ5�<‰<œŸ™¤u§|¡|Ð3Õ4ñrx   c           
     óZ   € RV RVRVRVRV/p\         P                  ! RVR7      pV! R/ VB # )	u–  Checks for collapsing each edge i = (a, b) into one vertex at positions[i].

vertices (V, 3) float32, faces (F, 3) int32/int64 with distinct corners, edges (E, 2)
int32/int64, positions (E, 3) float32. Returns per edge, over the faces that keep a or b:

- flips (E,) int32: faces with cos(old normal, new normal) < cos_threshold, skipping
  faces with |normal| <= 1e-6 x their sum of squared edge lengths before or after.
- skinny (E,) float32: mean of 1 - clamp(4âˆš3Â·area / sum of squared edge lengths, 0, 1),
  0 for equilateral faces.
- link_ok (E,) bool: a and b share at most as many neighbours as faces on the edge, so
  the collapse keeps the topology.
r¨  r©  r¤  rª  r«  rf   rz   r{   r|   )r¨  r©  r¤  rª  r«  rr   r~   s   &&&&&  rv   rf   rf     sA   € ð& ˜( G¨U°G¸UÀKÐQZØ˜}ð.€Fä×&Ò&Ð'=ÀfÔM€DÙ‰>�&‰>Ðrx   c          
      ó¸   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  /# )rp   ÚnbrÚcapÚs_capÚt_capr�   rœ   )ru   s   "rv   rw   rw   3  sF   € ÷ ñ ”—‘ð ¤E§L¡Lð ¼¿¹ð ÌeÏlÉlð Ô_d×_kÑ_kñ rx   c                óV   € RV RVRVRV/p\         P                  ! RVR7      pV! R/ VB # )aZ  Minimum s-t cut of a graph given as neighbour slots: nbr (N, K) int32/int64, -1 for an
empty slot, symmetric (j lists i whenever i lists j); cap (N, K) float32 symmetric edge
capacities; s_cap / t_cap (N,) float32 capacities to the source and the sink. Returns
(N,) bool, True on the source side (the largest source side among the minimum cuts).
r®  r¯  r°  r±  rg   rz   r{   r|   )r®  r¯  r°  r±  rr   r~   s   &&&&  rv   rg   rg   3  s8   € ð �S˜%  g¨u°g¸uÐE€FÜ×&Ò& y¸Ô@€DÙ‰>�&‰>Ðrx   c                ó>   € V ^8„  d   QhR\         \        ,          RR/# )rp   Úpriorityr�   N)r“   rs   )ru   s   "rv   rw   rw   C  s   € ÷ $ñ $¤4¬¥9ð $°ñ $rx   c                ó2   € \         P                  ! V 4       R# )z²Set the priority order for backend selection.

Args:
    priority: List of backend names in order of preference
             Example: ["cuda", "eager"] to prefer CUDA over Torch
N)r!   Úset_priority)r´  s   &rv   rj   rj   C  s   € ô ×Ò˜(Ö#rx   c                ó(   € V ^8„  d   QhR\         RR/# ©rp   rq   r�   N©rs   )ru   s   "rv   rw   rw   M  s   € ÷ ñ œ#ð  $ñ rx   c                ó2   € \         P                  ! V 4       R# )zoDisable a backend, preventing its use.

Args:
    name: Backend name to disable ("eager", "cuda", or "triton")
N)r!   Údisable©rq   s   &rv   rl   rl   M  s   € ô ×Ò�TÖrx   c                ó(   € V ^8„  d   QhR\         RR/# r¸  r¹  )ru   s   "rv   rw   rw   V  s   € ÷ ñ œð  ñ rx   c                ó2   € \         P                  ! V 4       R# )zpRe-enable a previously disabled backend.

Args:
    name: Backend name to enable ("eager", "cuda", or "triton")
N)r!   Úenabler¼  s   &rv   rk   rk   V  s   € ô ‡O‚O�DÖrx   c                ó$   € V ^8„  d   QhR\         /# )rp   r�   )rt   )ru   s   "rv   rw   rw   _  s   € ÷ $ñ $”tñ $rx   c                 ó,   € \         P                  ! 4       # )a+  Get status information for all backends.

Returns:
    Dictionary mapping backend names to their status:
    {
        "backend_name": {
            "available": bool,
            "disabled": bool,
            "unavailable_reason": str or None,
            "capabilities": list[str]
        }
    }
)r!   ri   r{   rx   rv   ri   ri   _  s   € ô ×!Ò!Ó#Ð#rx   c                ó$   € V ^8„  d   QhR\         /# )rp   rq   r¹  )ru   s   "rv   rw   rw   p  s   € ÷ 
&ñ 
&”cñ 
&rx   c                ó.   € \         P                  ! V 4      # )zçContext manager to temporarily use a specific backend.

Args:
    name: Backend name to use within the context

Example:
    with comfy_kitchen.use_backend("eager"):
        result = comfy_kitchen.quantize_per_tensor_fp8(x, scale)
)r!   rn   r¼  s   &rv   rn   rn   p  s   € ô ×Ò Ó%Ð%rx   )r,   r   r   r   )
g      ð?NNNNr€   TNNr   )N)NN)NTN)ç�íµ ÷Æ°>)NNrÃ   N)	NNé    rÄ  )r   r   r   r   r   TFNN)rÅ  rÄ  )r€   FT)NNN)F)r€  FN)NF)Né@   )NrÄ  )NrÄ  r   )Nr   )r   )gñhãˆµøä>)	NNFr€  NNr€   NN)NNFr€  )r€   )‰r�   Ú
allocationr   Úbackendsr   r©   r   Ú_ascend_backendr   Ú_eager_backendr   Ú_triton_backendÚbackends.cudar   r	   r¡   Úbackends.eager.quantizationr
   r   rk  Ú
exceptionsr   r   r   r   Úflash_attentionr   r   r9   r   r   r   r   Úfloat_utilsr   r   r   Úgated_deltar   r   r4   r   r5   r   r6   r   r   r   r7   r    r8   r!   Úsage_attentionr"   r#   r$   r%   r3   Útensor.convrot_w4a4r&   r'   r(   Útensor.w4a8_int8r)   r*   r+   r§   r    r,   Ú_hip_backendr¶  Úmark_unavailableÚ__all__r   r=   rh   r>   r;   r:   r<   r-   r/   r0   r1   r2   Úfloat8_e4m3fnr?   r  r@   rm   rA   rB   rI   rC   rD   rJ   rE   rK   rL   rQ   rR   rY   rZ   r[   r\   r]   r^   r_   r`   rS   rT   rU   rV   rW   rX   rG   rF   rH   r.   rM   ru  rN   rO   rP   ra   rb   r’   rc   rd   re   rf   rg   rj   rl   rk   ri   rn   r{   rx   rv   Ú<module>rÙ     s¬  ðÛ å .Ý +õ 0Ý -Ý /Ý +Ý 0Ý 6Ý <÷ó õ 4Ý Pß `ß aß ?Ñ ?ß EÝ HÝ SÝ Iß QÝ 9Ý 9Ý ÷ó õ H÷ñ ÷
ñ ñ ˆ5�=‰=˜% ×&Ò&Ý-ð ×Ò˜U×#Ò#Ø×ÒÒ@ÔAØ'×8Ñ8ÐØ ×)Ñ)ˆ	øà×Ò˜eÐ%MÔNðhàðhð ðhð ð	hð
 ðhð ðhð ðhð  ðhð ðhð 'ðhð "ðhð !ðhð ðhð !ðhð  .ð!hð" (ð#hð$ (ð%hð& ð'hð( ð)hð* &ð+hð, (ð-hð. #ð/hð0 "ð1hð2 "ð3hð4 ð5hð6 ð7hð8 *ð9hð: ð;hð< ð=hð> ð?hð@ ðAhðB ðChðD ðEhðH ðIhðJ  ðKhðL ðMhðN ðOhðP ðQhðR ðShðT ðUhðV #ðWhðX  ðYhðZ ð[hð\ ð]hð^ ð_hðb ðchðd ðehðf ðghðh ðihðj %ðkhðl "ðmhðn ðohðp ðqhðr ðshðt ðuhðv  ðwhðx ðyhð| ð}hð~ ðhð@ ðAhðB ðChðD ðEhðF ðGhðH ðIhðJ ðKhðL ðMhðN ðOhðP ðQhðR ðShðT ðUhðV ðWhðX ðYhðZ ð[hð^ ð_hð` ðahðb ðchðd ðehðf ðghðh ðihðj ðkhðn ðohðp ðqhðr ðshðt ðuhðv ðwhðz ð{hð| ð}hð~ ðhð@ ðAhðB ðChðD ðEhðH ðIhðJ ðKhðL !ðMhðN ðOh€õ`÷@õF÷Z÷&"P÷J÷:&dð &d÷R?ð ?÷(ð ÷*ð ÷:Jð J÷Cð Cð4  %×2Ñ2÷Fð,  %Ÿ~™~÷Hð,  %×2Ñ2÷÷(cð cð8  %Ÿ~™~Ø÷n÷2#ð #÷L>ð >ð.  %Ÿ~™~÷R÷&÷>ð ÷B$ð $÷Nð õ4Aõ(÷Xð X÷4ð ÷Kð K÷ð ÷lð l÷$ð ÷ 
Vð 
V÷ð õ=õ$õLõ0õHõ,÷ð ÷ð õDõ
÷
ð ÷.ð õ@C÷9ð 9÷xð ÷8ð õ@õ&ñ 27°u³÷ð ILÑ[`ÐafÓ[gØ÷õ ÷ð õ2õ $õõõ$÷"
&rx   