+
    öäûjËZ  ã                  ó¢  € R t ^ RIHt ^ RIt^ RIHt ^ RIt^ RIHt ^RI	H
t ^RIHt ]! ]P                  RR4      '       d   ^RI	Ht MRt^@t^€t]P(                  ]P*                  ]P,                  3tR-t]! R	R	R
7       ! R R4      4       t]3R R lltR R ltRRRR/R R lltR.R R lltR R ltRRRR/R R lltRRRR/R R llt R  R! lt!]PD                  PG                  R"R/R#7      R$ R% l4       t$]$PJ                  R& 4       t&]PD                  PG                  R'R/R#7      R( R) l4       t']'PJ                  R* 4       t(RRRR/R+ R, llt)R# )0zCPure INT8 scaled dot-product attention for NVIDIA tensor-core GPUs.)ÚannotationsN)Ú	dataclass)Ú
functional)Úcuda)ÚDTYPE_TO_CODEÚhip)r   T)ÚfrozenÚslotsc                  óŠ   € ] tR t^t$ RtR]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R	]R
&   R]R&   R]R&   R	]R&   R]R&   RtR# )ÚPrequantizedInt8Attentionaî  Packed Q/K/V and immutable launch metadata for split INT8 attention.

Instances own only the quantized tensors, their scales, and an optional
attention mask in its active layout: expanded 4D values or a prepared 3D
buffer containing key values and tile metadata, or a HIP 5D dense tile
buffer. Prepared masks replace the original mask. Instances never retain
the floating-point Q, K, or V inputs.
Create instances with :func:`prequantize_int8_attention` rather than
constructing them directly.
útorch.TensorÚqÚkÚvÚq_scaleÚk_scaleÚv_scaleÚintÚoriginal_head_dimztorch.dtypeÚinput_dtypeÚfloatÚattention_scaleÚcta_kútorch.Tensor | NoneÚ	attn_mask© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú__static_attributes__r   ó    ÚXC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\sage_attention.pyr   r      sG   ‡ ñ	ð ƒOØƒOØƒOØÓØÓØÓØÓØÓØÓØƒJØ"×"r#   r   c               ó$   € V ^8„  d   QhRRRRRR/# )é   Úlengthr   r   Úreturnr   )Úformats   "r$   Ú__annotate__r*   5   s!   € ÷ 3ñ 3˜#ð 3 cð 3°cñ 3r#   c                ó<   € W,           ^,
          V,          V,          # )é   r   )r'   r   s   &&r$   Ú_pad_to_cta_kr-   5   s   € Ø�^˜aÕ EÕ)¨UÕ2Ð2r#   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r&   Úkernel_head_dimr   Ú	kv_lengthÚhas_maskÚboolr(   r   )r)   s   "r$   r*   r*   9   s0   € ÷ ñ Øðàðð ð	ð
 	ñr#   c               óF   € V'       g   V ^€8¼  d   VR8”  d   \         # \        # )é€   é   )ÚLARGE_CTA_KÚCTA_K)r/   r0   r1   s   &&$r$   Ú_select_cta_kr8   9   s    € ÷ ˜¨3Ô.°9¸tÔ3CÜÐÜ€Lr#   Úfuse_short_key_maskFÚfuse_short_dense_maskc          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )	r&   r   r   r   r   r9   r2   r:   r(   r   )r)   s   "r$   r*   r*   D   sA   € ÷ @ñ @Ø"ð@àð@ð ð	@ð
  ð@ð ñ@r#   c               óÜ  € V'       d'   \         e   \         P                  ! V 4      '       d   V # V'       dx   \         ep   V el   V P                  ^,          R8:  dT   ^@V P                  ^,          u;8  d   R8:  d4   M M0V P                  ^4      ^ 8X  g   V P                  ^,          ^8X  d   V # V eP   V P                  R,          \         e   ^@MR8:  g.   \         f   V P                  ^4      ^ 8w  g   \         f
   V^ 8:  d   V # V P                  ^ 4      ^ 8X  d   ^MV P                  ^ ,          pV P                  ^4      ^ 8X  d   ^MV P                  ^,          p\         eÐ   V P                  ^4      ^ 8w  dº   V P                  ^,          ^8”  d¢   V RV1RV13,          p\         P                  ! V 4      p\         P
                  P                  \         P                  ! V4      \         P                  ! V4      \        P                  P                  V P                  4      P                  4       V# V RV1RV1RR3,          p\         e   \         P                  M\        p\         e   \         M\        p	\         e   V	P                  MV	P                   p
V P                  R,          V,           ^,
          V,          pW¸^,           ,          ^,           ^,          ^,          p\        P"                  ! WEV3\        P$                  V P                  R7      pV	P
                  P'                  V
! V4      V
! V4      \        P                  P                  V P                  4      P                  4       V# )	zIPrepare masks for the native tile layout or let a short fused call do it.Né   i   r5   :Nr,   N:NNN©ÚdtypeÚdeviceéÿÿÿÿ)Ú_hip_backendÚ_sage_can_fuse_dense_maskÚshapeÚstrideÚ_sage_dense_mask_bufferÚ_CÚsage_prepare_dense_maskÚ_dlÚtorchr   Úcurrent_streamr@   Úcuda_streamÚ_SAGE_CTA_Kr6   Ú_cuda_backendÚ_wrap_for_dlpackÚemptyÚfloat32Úsage_prepare_key_mask)r   r   r9   r:   Ú
mask_batchÚ
mask_headsÚcompact_maskÚpackedÚtile_kÚbackendÚwrapÚtilesÚpacked_widths   &&$$         r$   Ú_prepare_attn_maskr\   D   s«  € ÷ 	ÜÒ$Ü×2Ò2°9×=Ò=àÐçÜÒ$ØÒ!Ø�O‰O˜AÕ #Ô%Ø�—‘ Õ#Ö+ t×+Ø×Ñ˜aÓ  AÔ%¨¯©¸Õ);¸qÔ)@àÐð 	ÒØ�?‰?˜2Õ¬Ò)A¡2ÀtÔLÜÒ  Y×%5Ñ%5°aÓ%8¸AÔ%=ÜÒ  _¸Ô%9àÐØ×&Ñ& qÓ)¨QÔ.‘°I·O±OÀAÕ4F€JØ×&Ñ& qÓ)¨QÔ.‘°I·O±OÀAÕ4F€JÜÒ I×$4Ñ$4°QÓ$7¸1Ô$<ÀÇÁÐQRÕASÐVWÔAWØ   * ¨k¨z¨kÐ!9Õ:ˆÜ×5Ò5°iÓ@ˆÜ�‰×/Ñ/Ü×Ò˜\Ó*Ü×Ò˜VÓ$Ü�J‰J×%Ñ% i×&6Ñ&6Ó7×CÑCô	
ð
 ˆØ˜[˜j˜[¨+¨:¨+°r¸1Ð<Õ=€Lä)5Ò)AŒ\×%Ò%Ä{€FÜ*Ò6�l¼M€GÜ&Ò2ˆ7�;Š;¸×8PÑ8P€DØ�_‰_˜RÕ  6Õ)¨AÕ-°&Õ8€Eà q�jÕ)¨AÕ-°!Õ3°qÕ8€LÜ�[Š[Ø	 Ð.Ü�m‰mØ×Ñô€Fð
 ‡J�J×$Ñ$Ùˆ\ÓÙˆV‹Ü�
‰
×!Ñ! )×"2Ñ"2Ó3×?Ñ?ôð
 €Mr#   c               ó    € V ^8„  d   QhRRRR/# )r&   r@   ztorch.device | Noner(   r2   r   )r)   s   "r$   r*   r*   ‡   s   € ÷ Uñ UÐ,ð U¸ñ Ur#   c                ó6  € V e   V P                   R8w  d   R# \        P                  P                  4       '       g   R# \        e   \        P
                  ! 4       # \        P                  P                  V 4      pV\        8¬  ;'       d    \        P                  # )zDReturn whether the compiled INT8 attention kernel supports this GPU.r   F)
ÚtyperJ   r   Úis_availablerB   Úint8_attention_is_availableÚget_device_capabilityÚ_NATIVE_MINIMUM_CAPABILITYrN   Ú_EXT_AVAILABLE)r@   Ú
capabilitys   & r$   r`   r`   ‡   sq   € àÒ˜fŸk™k¨VÔ3ÙÜ�:‰:×"Ñ"×$Ò$ÙÜÒô
 ×7Ò7Ó9Ð9Ü—‘×1Ñ1°&Ó9€JØÔ3Ñ3×TÐT¼×8TÑ8TÐTr#   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r&   r   r   r   r   r   r   r(   r   )r)   s   "r$   r*   r*   —   s:   € ÷ 6ñ 6Øð6àð6ð ð6ð #ð	6ð
 ñ6r#   c                óî  € V P                   ^8w  g#   VP                   ^8w  g   VP                   ^8w  d   \        R4      hV P                  \        9  d   \	        RV P                   24      hV P                  VP                  8w  g   V P                  VP                  8w  d3   \	        RV P                   RVP                   RVP                   24      hV P
                  '       d7   V P                  VP                  8w  g   V P                  VP                  8w  d   \        R4      h\        V P                  4      '       g   \        R4      hV P                  w  rErgVP                  w  r‰r«VP                  WIW§38w  d"   \        R\        VP                  4       24      hW„8w  g   W·8w  d8   \        R	\        V P                  4       R
\        VP                  4       24      hV^ 8X  g   V^ 8X  g   V	^ 8X  g   V^ 8X  g   V
^ 8X  d   \        R4      hWY,          ^ 8w  d   \        RV RV	 R24      hV^ 8:  g   VR8”  d   \        RV 24      hV P                  R4      ^8w  g-   VP                  R4      ^8w  g   VP                  R4      ^8w  d   \        R4      hVf   R# VP                  V P                  8w  d   \        R4      hVP                  \        P                  \        P                  \        P                  \        P                   39  d   \	        R4      h \        P"                  ! W4WVV
34      #   \         d4   p\        RT RT RT RT
 R\        TP                  4       2
4      ThRp?ii ; i)é   z>q, k, and v must have shape [batch, heads, sequence, head_dim]z7q, k, and v must be float32, float16, or bfloat16, got z*q, k, and v must have the same dtype, got z, z, and z+q, k, and v must be on the same CUDA deviceú“INT8 attention requires the comfy-kitchen CUDA extension on SM75 or newer, or the HIP extension on an AMD device with matrix cores (RDNA3 or newer)zBv must have shape [q.batch, k.heads, k.sequence, q.head_dim], got z.q and k batch/head dimensions must match, got z and z9batch, head counts, and sequence lengths must be positivezq head count (z') must be divisible by k/v head count (Ú)r=   z"head_dim must be in [1, 256], got z4the last dimension of q, k, and v must be contiguousNz8attn_mask must be on the same CUDA device as q, k, and vz5attn_mask must be bool, float16, bfloat16, or float32z$attn_mask must be broadcastable to [z], got rA   )ÚndimÚ
ValueErrorr?   Ú_SUPPORTED_DTYPESÚ	TypeErrorÚis_cudar@   r`   ÚRuntimeErrorrD   ÚtuplerE   rJ   r2   Úfloat16Úbfloat16rQ   Úbroadcast_to)r   r   r   r   ÚbatchÚq_headsÚq_lengthÚhead_dimÚk_batchÚkv_headsr0   Ú
k_head_dimÚerrors   &&&&         r$   Ú_validate_inputsr}   —   s  € ð 	‡v�v�„{�a—f‘f ”k Q§V¡V¨q¤[ÜÐYÓZÐZØ‡w�wÔ'Ô'ÜÐQÐRS×RYÑRYÐQZÐ[Ó\Ð\Ø‡w�w�!—'‘'Ô˜QŸW™W¨¯©Ô/ÜØ8¸¿¹¸	ÀÀAÇGÁGÀ9ÈFÐST×SZÑSZÐR[Ð\ó
ð 	
ð �9�9ˆ9˜Ÿ™ A§H¡HÔ,°·±¸A¿H¹HÔ0DÜÐFÓGÐGÜ˜Ÿ™×!Ò!ÜðWó
ð 	
ð
 *+¯©Ñ&€E�HØ/0¯w©wÑ,€G�yØ‡w�w�5 IÐ8Ô8ÜØPÔQVÐWX×W^ÑW^ÓQ_ÐP`Ðaó
ð 	
ð Ô˜:Ô1ÜØ<¼UÀ1Ç7Á7»^Ð<LÈEÔRWÐXY×X_ÑX_ÓR`ÐQaÐbó
ð 	
ð �„z�W ”\ X°¤]°hÀ!´mÀyÐTUÄ~ÜÐTÓUÐUØÕ˜QÔÜØ˜W˜IÐ%LÈXÈJÐVWÐXó
ð 	
ð �1„}˜ 3œÜÐ=¸h¸ZÐHÓIÐIØ‡x�x�ƒ|�qÔ˜AŸH™H R›L¨AÔ-°·±¸"³ÀÔ1BÜÐOÓPÐPØÒÙØ×Ñ˜1Ÿ8™8Ô#ÜÐSÓTÐTØ‡�œuŸz™z¬5¯=©=¼%¿.¹.Ì%Ï-É-ÐXÔXÜÐOÓPÐPðÜ×!Ò! )°WÈ	Ð-RÓSÐSøÜô ÜðØˆw�b˜˜	  H :¨R°	¨{¸'Ä%È	ÏÉÓBXÐAYð[ó
ð ð	ûðús   ÌL6 Ì6M4Í.M/Í/M4Úscaler   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# ©
r&   r   r   r   r   r~   úfloat | Noner   r   r(   r   )r)   s   "r$   r*   r*   Ð   sT   € ÷ mBñ mBØðmBàðmBð ðmBð
 ðmBð #ðmBð ñmBr#   c               óÐ
  € \        WW$4      pV P                  R	,          pV^@8:  d   ^@pMV^€8:  d   ^€pMRpWe8w  dM   ^ We,
          3p\        P                  ! W4      p \        P                  ! W4      p\        P                  ! W'4      pVf
   VR
,          M
\	        V4      p\
        P                  ! V4      '       g   \        RV 24      h\        VVV^€8*  V^€8*  R7      p\        eZ   \        P                  ! V VVVVR7      p	V	RRV13,          p	V P                  \        P                  8X  d   V	P	                  4       # T	# V P                  w  r«rÍVP                  w  rÞrý\        TTVRJ;'       d    VP                  ^8H  R7      p\!        VV4      p\        P"                  ! V P                  \        P$                  V P&                  R7      p\        P"                  ! VP                  \        P$                  VP&                  R7      pVR8X  d   ^@M^ p\        P"                  ! V
VV^,           ^€,          V,          \        P                  V P&                  R7      p\        P"                  ! V
VVV,           ^,
          V,          ^,          \        P                  V P&                  R7      p\        P"                  ! W®,          V,          V\        P$                  V P&                  R7      p\        P"                  ! W®,          V,          \        P                  V P&                  R7      pV P                  \        P                  8X  d   \        P(                  MV P                  p\        P"                  ! W«WÆVV P&                  R7      p	\        P"                  ! W®\        P*                  V P&                  R7      pVP-                  4       p\        P.                  P1                  V P&                  4      P2                  p\4        P6                  P9                  \4        P:                  ! V 4      \4        P:                  ! V4      \4        P:                  ! V4      \4        P:                  ! V	4      \4        P:                  ! V4      \4        P:                  ! V4      \4        P:                  ! V4      \4        P:                  ! V4      \4        P:                  ! V4      \4        P:                  ! V4      T\<        V P                  ,          \<        V,          TTVe   \4        P:                  ! V4      MRV4       V	RRV13,          p	V P                  \        P                  8X  d   V	P	                  4       # T	# )r,   r=   Núscale must be finite, got )r9   r:   )r   r   .©r1   r>   rA   ç      à¿)r}   rD   r   Úpadr   ÚmathÚisfiniterl   r\   rB   Úsage_int8_sdpar?   rJ   rQ   r8   rk   r-   rP   Úint8r@   rs   Úint32Údata_ptrr   rK   rL   rN   rG   Ú	sage_sdparO   r   )r   r   r   r~   r   r   r/   Úpaddingr   Úoutputru   rv   rw   Ú_rz   r0   r   Úpadded_k_lengthÚq_int8Úk_int8Úq_scales_per_blockr   r   Úv_int8r   Úoutput_dtypeÚanchor_indicesÚanchor_indices_ptrÚ
stream_ptrs   &&&$$                        r$   Ú_int8_attention_cudarš   Ð   s&  € ô !  qÓ4€IàŸ™ �ÐØ˜BÔØ‰Ø	˜cÔ	!Ø‰àˆØÔ+Ø�oÕ9Ð:ˆÜ�NŠN˜1Ó&ˆÜ�NŠN˜1Ó&ˆÜ�NŠN˜1Ó&ˆà16²Ð'¨Ö-ÄEÈ%ÃL€OÜ�=Š=˜×)Ò)ÜÐ5°oÐ5FÐGÓHÐHä"ØØØ+¨sÑ2Ø-°Ñ4ô	€Iô ÒÜ×,Ò,ØØØØ+Øô
ˆð ˜Ð/Ð/Ð/Ð/Õ0ˆØ!"§¡¬E¯M©MÔ!9ˆv�|‰|‹~ÐE¸vÐEà"#§'¡'Ñ€E�HØ !§¡Ñ€A�ÜØØØ $Ð&×>Ð>¨9¯>©>¸QÑ+>ô€Eô
 $ I¨uÓ5€OÜ�[Š[˜Ÿ™¬¯
©
¸1¿8¹8ÔD€FÜ�[Š[˜Ÿ™¬¯
©
¸1¿8¹8ÔD€FØ.°#Ô5™¸2ÐÜ�kŠkØØØ
�S�.˜SÕ	 Ð$6Õ6Ü�m‰mØ�x‰xô€Gô �kŠkØØØ
�eÕ
˜aÕ
 EÕ	)¨QÕ.Ü�m‰mØ�x‰xô€Gô �[Š[ØÕ˜?Õ*ØÜ�j‰jØ�x‰xô	€Fô �kŠk˜%Õ*¨_Õ<ÄEÇMÁMÐZ[×ZbÑZbÔc€Gà%&§W¡W´·±Ô%=”5—>’>À1Ç7Á7€LÜ�[Š[Ø˜¸ÈaÏhÉhô€Fô —[’[ØœuŸ{™{°1·8±8ô€Nð (×0Ñ0Ó2Ðä—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×Ñ×ÑÜ×&Ò& qÓ)Ü×&Ò& qÓ)Ü×&Ò& qÓ)Ü×&Ò& vÓ.Ü×&Ò& vÓ.Ü×&Ò& wÓ/Ü×&Ò& vÓ.Ü×&Ò& wÓ/Ü×&Ò& vÓ.Ü×&Ò& wÓ/ØÜ�a—g‘gÕÜ�lÕ#ØØØ5>Ò5JŒ×&Ò& yÔ1ÐPTØô#ð( �CÐ+Ð+Ð+Ð+Õ,€FØŸW™W¬¯©Ô5ˆ6�<‰<‹>ÐA¸6ÐAr#   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r&   r   r   r   r   r~   r�   r   r   r(   r   r   )r)   s   "r$   r*   r*   @  sL   € ÷ Hñ HØðHàðHð ðHð
 ðHð #ðHð ñHr#   c               ó”	  € \        WW$4      pV P                  R,          pV P                  pV^@8:  d   ^@pMV^€8:  d   ^€pMRpWu8w  dM   ^ Wu,
          3p\        P                  ! W4      p \        P                  ! W4      p\        P                  ! W(4      pVf
   VR,          M
\        V4      p	\        P                  ! V	4      '       g   \        RV	 24      h\        WI4      p\        eƒ   \        P                  ! WqP                  ^,          VRJ4      p
\        P                  ! WW*R7      p\        VR,          VR,          VR,          VR,          VR	,          VR
,          VVV	V
VR7      # V P                  w  rÍrïVP                  w  pppp\        TTVRJ;'       d    VP                  ^8H  R7      p\!        VV4      p\"        P$                  ! V P                  \"        P&                  V P(                  R7      p\"        P$                  ! VP                  \"        P&                  VP(                  R7      pVR8X  d   ^@M^ p\"        P$                  ! VVV^,           ^€,          V,          \"        P*                  V P(                  R7      p\"        P$                  ! VVVV,           ^,
          V,          ^,          \"        P*                  V P(                  R7      p\"        P$                  ! VV,          V,          V\"        P&                  V P(                  R7      p\"        P$                  ! VV,          V,          \"        P*                  V P(                  R7      p\"        P$                  ! VV\"        P,                  V P(                  R7      pVP/                  4       p\"        P0                  P3                  V P(                  4      P4                  p\6        P8                  P;                  \6        P<                  ! V 4      \6        P<                  ! V4      \6        P<                  ! V4      \6        P<                  ! V4      \6        P<                  ! V4      \6        P<                  ! V4      \6        P<                  ! V4      \6        P<                  ! V4      \6        P<                  ! V4      V\>        V,          VV4       \        VVVVVVVVV	VVR7      # )am  Quantize Q, K, and V without allocating the attention output.

The returned object does not retain the floating-point inputs, so model
code can delete those tensors before calling
:func:`int8_attention_from_prequantized`. Optimized key masks are prepared
as part of this snapshot; recreate it after changing the mask.
Quantization and consumption use
the current CUDA stream and preserve normal PyTorch stream ordering; no
host synchronization is introduced.

This is an inference and peak-memory API. The regular :func:`int8_attention`
remains the lower-overhead single-call path when early Q/K/V release is not
needed.
r=   Nrƒ   )r   r’   r“   r•   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r„   r>   rA   r…   ) r}   rD   r?   r   r†   r   r‡   rˆ   rl   r\   rB   Ú_sage_cta_kÚsage_int8_quantizer   r8   rk   r-   rJ   rP   rŠ   r@   rQ   r‹   rŒ   r   rK   rL   rN   rG   Úsage_sdpa_quantizerO   r   )r   r   r   r~   r   r   r   r/   rŽ   r   Ú	hip_cta_krV   ru   rv   rw   r�   rz   r0   r   r‘   r’   r“   r”   r   r   r•   r   r—   r˜   r™   s   &&&$$                         r$   Úprequantize_int8_attentionr¡   @  sÞ  € ô, !  qÓ4€IàŸ™ �ÐØ—'‘'€KØ˜BÔØ‰Ø	˜cÔ	!Ø‰àˆØÔ+Ø�oÕ9Ð:ˆÜ�NŠN˜1Ó&ˆÜ�NŠN˜1Ó&ˆÜ�NŠN˜1Ó&ˆà16²Ð'¨Ö-ÄEÈ%ÃL€OÜ�=Š=˜×)Ò)ÜÐ5°oÐ5FÐGÓHÐHä" 9Ó>€IÜÒô !×,Ò,ØŸW™W Q�Z¨¸$Ð)>ó
ˆ	ô ×0Ò0°°qÔJˆÜ(Ø�XÕØ�XÕØ�XÕØ˜9Õ%Ø˜9Õ%Ø˜9Õ%Ø/Ø#Ø+ØØô
ð 	
ð #$§'¡'Ñ€E�HØ !§¡Ñ€A€x�˜AÜØØØ $Ð&×>Ð>¨9¯>©>¸QÑ+>ô€Eô
 $ I¨uÓ5€OÜ�[Š[˜Ÿ™¬¯
©
¸1¿8¹8ÔD€FÜ�[Š[˜Ÿ™¬¯
©
¸1¿8¹8ÔD€FØ.°#Ô5™¸2ÐÜ�kŠkØØØ
�S�.˜SÕ	 Ð$6Õ6Ü�m‰mØ�x‰xô€Gô �kŠkØØØ
�eÕ
˜aÕ
 EÕ	)¨QÕ.Ü�m‰mØ�x‰xô€Gô �[Š[Ø�Õ˜?Õ*ØÜ�j‰jØ�x‰xô	€Fô �kŠkØ�Õ˜?Õ*Ü�m‰mØ�x‰xô€Gô —[’[ØˆxœuŸ{™{°1·8±8ô€Nð (×0Ñ0Ó2Ðä—‘×*Ñ*¨1¯8©8Ó4×@Ñ@€JÜ×Ñ×'Ñ'Ü×&Ò& qÓ)Ü×&Ò& qÓ)Ü×&Ò& qÓ)Ü×&Ò& vÓ.Ü×&Ò& wÓ/Ü×&Ò& vÓ.Ü×&Ò& wÓ/Ü×&Ò& vÓ.Ü×&Ò& wÓ/ØÜ�kÕ"ØØôô  %Ø
Ø
Ø
ØØØØ+ØØ'ØØôð r#   c               ó    € V ^8„  d   QhRRRR/# )r&   Ú	quantizedr   r(   r   r   )r)   s   "r$   r*   r*   Ë  s$   € ÷ QPñ QPØ(ðQPàñQPr#   c                ó	  a € \        S \        4      '       g"   \        R\        S 4      P                   24      hS P
                  S P                  S P                  S P                  S P                  S P                  3pS P
                  P                  '       g   \        R4      h\        ;QJ d&    V 3R lVR,           4       F  '       g   K   RM	  RM! V 3R lVR,           4       4      '       d   \        R4      hS P                  e;   S P                  P                  S P
                  P                  8w  d   \        R	4      h\!        S P
                  P                  4      '       g   \#        R
4      hS P
                  P$                  w  r#rES P&                  \(        P*                  8X  d   \(        P,                  MS P&                  p\.        eÃ   \.        P0                  ! S P
                  S P                  S P                  S P                  S P                  S P                  S P2                  S P                  VS P4                  R7
      pVRRS P6                  13,          pS P&                  \(        P*                  8X  d   VP9                  4       # T# \(        P:                  ! VVVVVS P
                  P                  R7      p\(        P<                  P?                  S P
                  P                  4      P@                  p\B        PD                  PG                  \B        PH                  ! S P
                  4      \B        PH                  ! S P                  4      \B        PH                  ! S P                  4      \B        PH                  ! V4      \B        PH                  ! S P                  4      \B        PH                  ! S P                  4      \B        PH                  ! S P                  4      S P4                  S P2                  \J        V,          TS P                  e!   \B        PH                  ! S P                  4      MR4       VRRS P6                  13,          pS P&                  \(        P*                  8X  d   VP9                  4       # T# )zFRun INT8 attention after the floating-point Q/K/V inputs are released.z>quantized must be returned by prequantize_int8_attention, got z<prequantized INT8 attention tensors must be on a CUDA devicec              3  óh   <"  € T F'  qP                   SP                  P                   8g  x € K)  	  R # 5i©N)r@   r   )Ú.0Útensorr£   s   & €r$   Ú	<genexpr>Ú3int8_attention_from_prequantized.<locals>.<genexpr>ß  s$   øé € Ð
PÑ=O°6�=‰=˜IŸK™K×.Ñ.Ö.Ó=Oùs   ƒ/2:r,   NNTFzCprequantized INT8 attention tensors must be on the same CUDA deviceNz?attn_mask must be on the same CUDA device as the packed tensorsri   )r   r   r–   r   .r>   )&Ú
isinstancer   rn   r_   r   r   r   r   r   r   r   ro   rl   Úanyr   r@   r`   rp   rD   r   rJ   rQ   rs   rB   Úsage_int8_attendr   r   r   r   rP   r   rK   rL   rN   rG   Úsage_sdpa_prequantizedrO   r   )	r£   Úpacked_tensorsru   rv   rw   r/   r–   r�   r™   s	   f        r$   Ú int8_attention_from_prequantizedr°   Ë  sf  ø€ ô �iÔ!:×;Ò;ÜØLÜ�I‹×'Ñ'Ð(ð*ó
ð 	
ð 	�‰Ø�‰Ø�‰Ø×ÑØ×ÑØ×Ñð€Nð �;‰;××ÐÜÐWÓXÐXß
ƒsÔ
P¸^ÈBÖ=OÓ
P‡s‡s‚sÔ
P¸^ÈBÖ=OÓ
P×PÒPÜÐ^Ó_Ð_Ø×ÑÒ&¨9×+>Ñ+>×+EÑ+EÈÏÉ×I[ÑI[Ô+[ÜÐZÓ[Ð[Ü˜	Ÿ™×*Ñ*×+Ò+ÜðWó
ð 	
ð
 1:·±×0AÑ0AÑ-€E�Hà#×/Ñ/´5·=±=Ô@Œ�ŠÀi×F[ÑF[ð ô ÒÜ×.Ò.Ø�K‰KØ�K‰KØ�K‰KØ×ÑØ×ÑØ×ÑØ%×5Ñ5Ø×)Ñ)Ø%Ø—/‘/ô
ˆð ˜Ð:˜y×:Ñ:Ð:Ð:Õ;ˆØ!*×!6Ñ!6¼%¿-¹-Ô!Gˆv�|‰|‹~ÐSÈVÐSä�[Š[ØØØØØØ�{‰{×!Ñ!ô€Fô —‘×*Ñ*¨9¯;©;×+=Ñ+=Ó>×JÑJ€JÜ×Ñ×+Ñ+Ü×&Ò& y§{¡{Ó3Ü×&Ò& y§{¡{Ó3Ü×&Ò& y§{¡{Ó3Ü×&Ò& vÓ.Ü×&Ò& y×'8Ñ'8Ó9Ü×&Ò& y×'8Ñ'8Ó9Ü×&Ò& y×'8Ñ'8Ó9Ø�‰Ø×!Ñ!Ü�lÕ#Øð ×"Ñ"Ò.ô ×*Ò*¨9×+>Ñ+>Ô?àôð& �CÐ6˜9×6Ñ6Ð6Ð6Õ7€FØ&×2Ñ2´e·m±mÔCˆ6�<‰<‹>ÐOÈÐOr#   zcomfy_kitchen::int8_attention)Úmutates_argsc          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r&   r   r   r   r   r~   r�   r(   r   )r)   s   "r$   r*   r*      s:   € ÷ ñ Øðàðð ðð ð	ð
 ñr#   c                ó"   € \        V VVVR R7      # )N©r~   r   ©rš   ©r   r   r   r~   s   &&&&r$   Ú_op_int8_attentionr·     s    € ô  Ø	Ø	Ø	ØØôð r#   c                ó8   € V P                  V P                  4      # r¦   ©Ú	new_emptyrD   r¶   s   &&&&r$   Ú_op_int8_attention_faker»   /  s   € ð �;‰;�q—w‘wÓÐr#   z$comfy_kitchen::int8_attention_maskedc               ó0   € V ^8„  d   QhRRRRRRRRRRRR/# )	r&   r   r   r   r   r   r~   r�   r(   r   )r)   s   "r$   r*   r*   :  sD   € ÷ ñ Øðàðð ðð ð	ð
 ðð ñr#   c                ó"   € \        V VVVVR 7      # )r´   rµ   ©r   r   r   r   r~   s   &&&&&r$   Ú_op_int8_attention_maskedr¿   9  s    € ô  Ø	Ø	Ø	ØØôð r#   c                ó8   € V P                  V P                  4      # r¦   r¹   r¾   s   &&&&&r$   Ú_op_int8_attention_masked_fakerÁ   J  s   € ð �;‰;�q—w‘wÓÐr#   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# r€   r   )r)   s   "r$   r*   r*   U  sD   € ÷ 'ñ 'Øð'àð'ð ð'ð
 ð'ð #ð'ð ñ'r#   c               ó¼   € Vf-   \         P                  P                  P                  V VVV4      # \         P                  P                  P	                  V VVVV4      # )aC  Compute inference SDPA with signed INT8 Q/K/V and unsigned INT8 P.

Inputs use ``[batch, heads, sequence, head_dim]`` layout. Grouped-query
attention and unequal non-causal Q/K sequence lengths are supported. Head
dimensions are padded to the kernel's 64-, 128-, or 256-wide tile and
sliced back on return; 64, 128, and 256 take the zero-copy dimension path.
Q and K receive the same fused block-Hadamard rotation before INT8
quantization, preserving their exact dot products while reducing
quantization outliers. K lengths up to 256 use low-overhead H4 blocks and
longer D64 attention uses H64. The common D128 path uses a fixed signed H128
transform, while padded D256 uses plain H128. The kernel samples K on the
GPU and subtracts a representative key only when doing so improves the
quantization range. This model-independent shift is exactly
softmax-invariant and uses one int32 of temporary storage per batch/KV-head.
Softmax score, maximum, exponential, denominator, reciprocal, and V-scale
arithmetic is FP32. This path does not allocate FP8 tensors or execute FP8
MMA instructions.
)rJ   ÚopsÚcomfy_kitchenÚint8_attentionÚint8_attention_masked)r   r   r   r~   r   s   &&&$$r$   rÆ   rÆ   U  sa   € ð4 ÒÜ�y‰y×&Ñ&×5Ñ5ØØØØó	
ð 	
ô �9‰9×"Ñ"×8Ñ8Ø	Ø	Ø	ØØóð r#   )é   é   r¦   r   )*r    Ú
__future__r   r‡   Údataclassesr   rJ   Útorch.nnr   Úbackendsr   rN   Úbackends.eager.quantizationr   ÚgetattrÚversionr   rB   r7   r6   rQ   rr   rs   rm   rc   r   r-   r8   r\   r`   r}   rš   r¡   r°   ÚlibraryÚ	custom_opr·   Úregister_faker»   r¿   rÁ   rÆ   r   r#   r$   Ú<module>rÔ      s§  ðñ Jå "ã Ý !ã Ý å +Ý 6á
ˆ5�=‰=˜% ×&Ò&Þ-à€Là
€Ø€Ø—]‘] E§M¡M°5·>±>ÐBÐ Ø#Ð ñ �$˜dÔ#÷#ð #ó $ð#ð2 -2÷ 3õð@ð !&ð	@ð
 #(÷@÷FUõ 6ðrmBð
 ðmBð &*÷mBð`Hð
 ðHð &*÷HõVQPðh ‡�×ÑÐ8ÀrÐÓJôó Kðð ×!Ñ!ñ ó "ð ð ‡�×ÑÐ?ÈbÐÓQôó Rðð  ×(Ñ(ñ ó )ð ð'ð
 ð'ð &*÷'ñ 'r#   