+
    �åûjZ¾  ã            
       óØ  a€ R’ tq0 t ^ RIt^ RIHt ^ RIHt ^ RIHtH	t	H
t
HtHtHtHtHtHtHtHt ^ RIHt ^ RIHtHtHt ^ RIHtHtHt ^ ]P:                  ^]P<                  ^]P>                  ^]P@                  ^]PB                  /t"]"PG                  4        U Uu/ uF  w  rWbK	  	  upp t$]P@                  3R R	 llt%]P>                  3R
 R llt&R t']P@                  3R R llt(R“R R llt)]PT                  ! . R”O4      PW                  ^4      t,/ t-]P>                  R3R R llt.R•R R llt/^ t0R–R R llt1]P>                  3R R llt2R—R R llt3]Ph                  Pk                  RR˜R7      R  R! l4       t6]6Pn                  R" 4       t8]Ph                  Pk                  R#R˜R7      R$ R% l4       t9]9Pn                  R& 4       t:]Ph                  Pk                  R'R˜R7      R( R) l4       t;];Pn                  R* 4       t<]Ph                  Pk                  R+R˜R7      R, R- l4       t=]=Pn                  R. 4       t>]Ph                  Pk                  R/R˜R7      R0 R1 l4       t?]?Pn                  R2 4       t@]Ph                  Pk                  R3R˜R7      R4 R5 l4       tA]APn                  R6 4       tB]Ph                  Pk                  R7R˜R7      R8 R9 l4       tC]CPn                  R: 4       tD]Ph                  Pk                  R;R˜R7      R< R= l4       tE]EPn                  R> 4       tF/ tG] ^ k R? R@ ltHRA RB ltIRC RD ltJRE RF ltKRG RH ltLRI RJ ltMRK RL ltNR™RM RN lltORšRO RP lltPR™RQ RR lltQR™RS RT lltRR™RU RV lltSRW RX ltTRY RZ ltUR[ R\ ltVR] R^ ltWR_ R` ltXRa Rb ltYR•Rc Rd lltZR]P>                  RReRRRRR3	Rf Rg llt[Rh Ri lt\R]P>                  RRe3Rj Rk llt]R]P>                  RRe3Rl Rm llt^]Ph                  Pk                  RnR˜R7      Ro Rp l4       t_]_Pn                  Rq 4       t`]Ph                  Pk                  RrR˜R7      Rs Rt l4       ta]aPn                  Ru 4       tb]Ph                  Pk                  RvR˜R7      Rw Rx l4       tc]cPn                  Ry 4       td]Ph                  Pk                  RzR˜R7      R{ R| l4       te]ePn                  R} 4       tf]Ph                  Pk                  R~R˜R7      R R€ l4       tg]gPn                  R� 4       th]Ph                  Pk                  R‚R˜R7      Rƒ R„ l4       ti]iPn                  R… 4       tj]Ph                  Pk                  R†R˜R7      R‡ Rˆ l4       tk]kPn                  R‰ 4       tl]Ph                  Pk                  RŠR˜R7      R‹ RŒ l4       tm]mPn                  R� 4       tn]Ph                  Pk                  RŽR˜R7      R›R� R� ll4       to]oPn                  R›R‘ l4       tpR# u upp i )œé    N)Úapply_input_act)Úapply_residual)Ú	E8M0_BIASÚF4_E2M1_MAXÚF8_E4M3_MAXÚF8_E5M2_MAXÚ_f32_to_floatx_unpackedÚ_float8_roundÚe8m0_to_f32Úfrom_blockedÚ
pack_uint4ÚroundupÚ
to_blocked)Úregistry)ÚScalingTypeÚSwizzleTypeÚscaled_mm_v2)Ú_build_hadamardÚ_rotate_activationÚ_rotate_weightc                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# ©é   ÚxÚscaleÚoutput_typeÚreturn©ÚtorchÚTensorÚdtype)Úformats   "ÚeC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\backends\eager\quantization.pyÚ__annotate__r$   -   s<   € ÷  ñ  Ü‡|�|ð Ü!ŸL™Lð Ü7<·{±{ð ä
‡\�\ñ ó    c                 ó6  € V\         P                  8X  d   \        pM+V\         P                  8X  d   \        pM\        R V R24      hV RV,          P                  V P                  4      ,          p\         P                  ! WC) W4R7      pVP                  V4      # )úUnsupported output_type: ú3. Expected torch.float8_e4m3fn or torch.float8_e5m2ç      ð?)Úout)	r   Úfloat8_e4m3fnr   Úfloat8_e5m2r   Ú
ValueErrorÚtor!   Úclamp)r   r   r   Úlp_maxÚtemps   &&&  r#   Úquantize_per_tensor_fp8r2   -   s„   € ð ”e×)Ñ)Ô)Ü‰Ø	œ×)Ñ)Ô	)Ü‰äØ'¨ }Ð4gÐhó
ð 	
ð ��e•×Ñ §¡Ó(Õ(€DÜ�;Š;�t˜W fÔ7€DØ�7‰7�;ÓÐr%   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# r   r   )r"   s   "r#   r$   r$   =   s<   € ÷ ñ Ü‡|�|ðÜ!ŸL™LðÜ7<·{±{ðä
‡\�\ñr%   c                 óX   € V P                  VR 7      VP                  VR 7      ,          pV# ©©r!   )r.   )r   r   r   Ú	dq_tensors   &&& r#   Údequantize_per_tensor_fp8r8   =   s*   € ð —‘˜;�Ó'¨%¯(©(¸¨(Ó*EÕE€IØÐr%   c                 óL  € \         P                  ! VV R W,
          ,          ,          R,
          ^V,          ,          V R V) ^,           V,
          ,          ,          4      pWeP                  VP                  R7      R,          ,          pVP	                  4       ^V,          ,          # )ç       @r)   r6   g      p?)r   Úwherer.   r!   Úfloor)Úabs_xÚexponentÚnormal_maskÚMANTISSA_BITSÚEXPONENT_BIASÚrngÚmantissa_scaleds   &&&&&& r#   Úcalc_mantissarD   D   s„   € Ü—k’kØØ	�#˜(Õ2Õ3Õ	4°sÕ	:¸qÀ-Õ?OÕPØ	�#˜=˜.¨1Õ,¨}Õ<Õ=Õ	>ó€Oð —v‘v O×$9Ñ$9�vÓ:¸kÕJÕJ€OØ× Ñ Ó" a¨Õ&6Õ7Ð7r%   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# )r   r   rB   r   r   r   )r"   s   "r#   r$   r$   O   s@   € ÷ % ñ % Ü‡|�|ð% ä	�‰ð% ô —‘ð% ô ‡\�\ñ	% r%   c                 óL  € V\         P                  8X  d   ^^^rTpM*V\         P                  8X  d   ^^^rTpM\        RV R24      hV P	                  4       p \         P
                  ! V 4      pV P                  4       p\         P                  ! V^ 8H  ^ V4      p\         P                  ! \         P                  ! \         P                  ! V4      4      V,           ^ ^V,          ^,
          4      pV^ 8H  ( p	\        WxW”WQ4      VR&   V\         P                  ! V	RW…,
          ,          RV,           ,          RV) ^,           ,          V,          4      ,          p\         P                  ! V4      p
\         P                  ! WjP                  V
P                  VR7       VP                  V4      # )é   r'   r(   :NNNr:   r)   )ÚminÚmaxr*   )r   r+   r,   r-   ÚhalfÚsignÚabsr;   r/   r<   Úlog2rD   ÚfinforH   rI   r.   )r   rB   r   ÚEXPONENT_BITSr@   rA   rK   r=   r>   r?   Úinfos   &&&        r#   Ústochastic_rounding_fp8rQ   O   sb  € ð
 ”e×)Ñ)Ô)Ø67¸¸A mˆ mØ	œ×)Ñ)Ô	)Ø67¸¸B mˆ mäØ'¨ }ð 5#ð #ó
ð 	
ð
 	
�‰‹€AÜ�:Š:�a‹=€DØ�E‰E‹G€EÜ�;Š;�u ‘z 1 dÓ+€Dä�{Š{Ü�Š”E—J’J˜uÓ%Ó&¨Õ6Ø	Ø	ˆ=Õ˜1Õó€Hð
  ‘MÐ"€Kä˜U¨kÈ-Ó]€Eˆ!�HàŒE�KŠKØØ	�Õ)Õ	*¨s°U­{Õ;Ø	�-� !Õ#Õ	$¨Õ-óõ €Dô �;Š;�{Ó#€DÜ	‡K‚K�Ÿ(™(¨¯©°dÕ;Ø�7‰7�;ÓÐr%   ç        FTc                óÒ   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\        R\        \         P                  \         P                  3,          /# ©r   r   Úper_tensor_scaleÚepsilonÚpad_16xÚhi_firstr   ©r   r    ÚfloatÚboolÚtuple)r"   s   "r#   r$   r$   w   s^   € ÷ -#ñ -#Ü‡|�|ð-#ä—l‘lð-#ô ð-#ô ð	-#ô
 ð-#ô Œ5�<‰<œŸ™Ð%Õ&ñ-#r%   c                 óˆ  € V P                   pV'       dy   V P                   w  rg\        V^4      p\        V^4      p	W†8w  g   W—8w  dG   \        P                  P                  P                  V ^ W—,
          ^ W†,
          34      p V P                   p^p
V P                  V^ ,          RV
4      p \        P                  ! \        P                  ! V 4      RR7      pVP                  \        P                  4      \        ,          pWÁ,          p\        P                  ! V\        R7      p\        V4      pW,          pV^ 8H  p\        P                  ! V\        P                   ! V4      V4      pV P#                  4       VP%                  R4      ,          p\        P                  ! VP%                  R4      \        P&                  ! V4      V4      pTp\        P                  ! V\        ) \        4      pVP)                  V4      p\+        V^^4      p\-        VVR7      p\/        VP                  \        P0                  4      RR7      pVV3# )é   ©Údim)rI   )rX   F©Úflattenéÿÿÿÿ)Úshaper   r   ÚnnÚ
functionalÚpadÚreshapeÚamaxrL   r.   Úfloat32r   r/   r   r
   r;   Ú	ones_likerZ   Ú	unsqueezeÚ
zeros_likeÚviewr	   r   r   r+   )r   rU   rV   rW   rX   Ú
orig_shapeÚrowsÚcolsÚpadded_rowsÚpadded_colsÚ
block_sizeÚmax_absÚblock_scaleÚscaled_block_scalesÚscaled_block_scales_fp8Úscaled_block_scales_fp32Útotal_scaleÚzero_scale_maskÚtotal_scale_safeÚdata_scaledÚ
out_scalesÚdata_lpÚblocked_scaless   &&&&&                  r#   Úquantize_nvfp4r�   w   sÆ  € ð —‘€J÷ Ø—W‘W‰
ˆÜ˜d BÓ'ˆÜ˜d BÓ'ˆØÔ +Ô"5Ü—‘×#Ñ#×'Ñ'¨¨A¨{Õ/AÀ1ÀkÕFXÐ+YÓZˆAð Ÿ™ˆJà€Jà	�	‰	�*˜Q•-  ZÓ0€AÜ�jŠjœŸš 1›¨2Ô.€GØ—*‘*œUŸ]™]Ó+¬kÕ9€KØ%Õ8ÐÜ#ŸkškÐ*=Ä;ÔOÐÜ,Ð-DÓEÐØ"Õ=€Kð # aÑ'€OÜ—{’{ ?´E·O²OÀKÓ4PÐR]Ó^Ðà—'‘'“)Ð.×8Ñ8¸Ó<Õ<€KÜ—+’+˜o×7Ñ7¸Ó;¼U×=MÒ=MÈkÓ=ZÐ\gÓh€Kà(€Jä—+’+˜k¬K¨<¼ÓE€KØ×"Ñ" :Ó.€Kä% k°1°aÓ8€GÜ˜¨8Ô4€GÜ 
§¡¬e×.AÑ.AÓ BÈEÔR€NØ�NÐ"Ð"r%   c                óÄ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\        R\         P                  /# )r   ÚqxrU   Úblock_scalesr   rX   r   )r   r    r!   r[   )r"   s   "r#   r$   r$   ¯   sX   € ÷ 0ñ 0Ü�‰ð0ä—l‘lð0ô —,‘,ð0ô —‘ð	0ô
 ð0ô ‡\�\ñ0r%   c                 ó¸  € \         P                  V P                  V34      pVf6   \        P	                  V P                  V4      pV\         V P                  V3&   V ^,          pV ^,	          pV'       d;   \
        P                  ! Wv.RR7      P                  ! . V P                  R R ORN5!  pM9\
        P                  ! Wg.RR7      P                  ! . V P                  R R ORN5!  p\
        P                  P                  P                  VP                  4       V4      P                  R4      pVP                  p	^p
VP                  V	^ ,          RV
4      pV	^,          V
,          p\        VV	^ ,          VR7      pVP	                  V4      VP	                  V4      ,          pW�P!                  R4      ,          pVP                  V	4      P	                  V4      pV# )Nr_   ©Únum_rowsÚnum_colsrc   )ÚE2M1_LUT_CACHEÚgetÚdeviceÚE2M1_LUTr.   r   Ústackrn   rd   re   rf   Ú	embeddingÚintÚsqueezerh   r   rl   )rƒ   rU   r„   r   rX   ÚlutÚloÚhir*   ro   rt   Únum_blocks_per_rowÚblock_scales_unswizzledrz   Údata_dequantizedÚresults   &&&&&           r#   Údequantize_nvfp4r˜   ¯   s˜  € ô ×
Ñ
˜bŸi™i¨Ð5Ó
6€CØ
‚{Ü�k‰k˜"Ÿ)™) [Ó1ˆØ36Œ˜Ÿ	™	 ;Ð/Ñ0à	ˆd�€BØ	ˆq�€BßÜ�kŠk˜2˜(¨Ô+×0Ò0ÐD°"·(±(¸3¸B°-ÐDÀÓD‰ä�kŠk˜2˜(¨Ô+×0Ò0ÐD°"·(±(¸3¸B°-ÐDÀÓDˆÜ
�(‰(×
Ñ
×
'Ñ
'¨¯©«	°3Ó
7×
?Ñ
?ÀÓ
C€Cð —‘€JØ€Jð �+‰+�j •m R¨Ó
4€Cð
 $ A�¨*Õ4Ðô +ØØ˜A•Ø#ôÐð #×%Ñ% kÓ2Ð5L×5OÑ5OÐP[Ó5\Õ\€Kð ×2Ñ2°2Ó6Õ6Ðð ×"Ñ" :Ó.×1Ñ1°+Ó>€Fà€Mr%   c                ó‚  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\         P                  R,          R
\         P                  R,          R\         P                  /
# )r   ÚaÚbÚtensor_scale_aÚtensor_scale_bÚblock_scale_aÚblock_scale_bÚbiasNÚ	out_dtypeÚalphar   r   )r"   s   "r#   r$   r$   â   s£   € ÷ ñ Ü‡|�|ðä‡|�|ðô —L‘Lðô —L‘Lð	ô
 —<‘<ðô —<‘<ðô �,‰,˜Õ
ðô �{‰{˜TÕ!ðô �<‰<˜$Õðô ‡\�\ñr%   c	                 óô  € \        V P                  \        P                  4      VP                  \        P                  4      P	                  4       VP                  R4      V.VP                  R4      V.VV\
        P                  \
        P                  .\
        P                  \
        P                  .\        P                  \        P                  .\        P                  \        P                  .R7
      p	V	# )é   ©Úscale_aÚscale_br    r¡   Úscale_recipe_aÚscale_recipe_bÚ	swizzle_aÚ	swizzle_brc   )r   rn   r   Úfloat4_e2m1fn_x2Útr   ÚBlockWise1x16Ú
TensorWiser   ÚSWIZZLE_32_4_4Ú
NO_SWIZZLE)
rš   r›   rœ   r�   rž   rŸ   r    r¡   r¢   r—   s
   &&&&&&&&& r#   Úscaled_mm_nvfp4r²   â   s¾   € ô Ø	�‰Œu×%Ñ%Ó&Ø	�‰Œu×%Ñ%Ó&×(Ñ(Ó*Ø×#Ñ# BÓ'¨Ð8Ø×#Ñ# BÓ'¨Ð8ØØÜ%×3Ñ3´[×5KÑ5KÐLÜ%×3Ñ3´[×5KÑ5KÐLÜ ×/Ñ/´×1GÑ1GÐHÜ ×/Ñ/´×1GÑ1GÐHô€Fð €Mr%   c                óš   € V ^8„  d   QhR\         P                  R\        R\        \         P                  \         P                  3,          /# ©r   r   Úpad_32xr   ©r   r    r[   r\   )r"   s   "r#   r$   r$     sA   € ÷ C?ñ C?Ü‡|�|ðC?äðC?ô Œ5�<‰<œŸ™Ð%Õ&ñC?r%   c                ód  € V P                   pV P                  4       ^8X  g   Q R4       hV'       dz   V P                   w  r4\        V^ 4      p\        V^ 4      pWS8w  g   Wd8w  dG   \        P                  P
                  P                  V ^ Wd,
          ^ WS,
          34      p V P                   pM1V P                   ^,          \        ,          ^ 8X  g   Q R\         24       hVw  r4V\        ,          pV P                  W7\        4      p\        P                  ! \        P                  ! V4      RR7      p	V	P                  4       \        ,          p
\        P                  ! V
RR7      p
\        P                  ! V
4      p\        P                  ! V4      P!                  \        P"                  4      \$        ,           p\        P                  ! V^ ^þ4      pVP!                  \        P&                  4      p\)        V4      pV	^ 8H  p\        P*                  ! V\        P,                  ! V4      V4      pVP                  4       VP/                  R4      ,          p\        P*                  ! VP/                  R4      \        P0                  ! V4      V4      p\        P                  ! V\        ) \        4      pVP                  V4      P!                  \        P2                  4      p\        P*                  ! V\        P0                  ! V4      V4      p\5        VRR7      pVVP7                  \        P8                  4      3# )	aé  Quantize tensor to MXFP8 format with block-wise E8M0 scaling.

MXFP8 uses block size 32 with power-of-2 (E8M0) block scales.

Args:
    x: Input tensor (2D, shape M x K)
    output_type: FP8 output dtype (float8_e4m3fn)
    pad_32x: If True, pad dimensions to be divisible by 32

Returns:
    Tuple of (quantized_fp8_tensor, block_scales_e8m0)
    - quantized_fp8_tensor: FP8 data of shape (M, K) or padded shape
    - block_scales_e8m0: E8M0 scales of shape (M, K//32) in swizzled layout
zInput must be 2Dz!K dimension must be divisible by r_   ©rH   Fra   rc   g       8)rd   r`   r   r   re   rf   rg   ÚMXFP8_BLOCK_SIZErh   ri   rL   rZ   r   r/   rM   Úceilr.   Úint32r   Úuint8r   r;   rk   rl   rm   r+   r   rn   Úfloat8_e8m0fnu)r   rµ   ro   rp   rq   rr   rs   Ú
num_blocksÚ	x_blockedru   Úscale_neededÚ
log2_scaleÚ
exp_biasedÚblock_scales_e8m0Úblock_scales_f32Ú	zero_maskr}   Údata_fp8r€   s   &&                 r#   Úquantize_mxfp8rÇ     sl  € ð$ —‘€JØ�5‰5‹7�aŒ<Ð+Ð+Ó+ˆ<÷ Ø—W‘W‰
ˆÜ˜d BÓ'ˆÜ˜d BÓ'ˆØÔ +Ô"5Ü—‘×#Ñ#×'Ñ'¨¨A¨{Õ/AÀ1ÀkÕFXÐ+YÓZˆAØŸ™ˆJøà�w‰w�q�zÔ,Õ,°Ô1ÐiÐ5VÔWgÐVhÐ3iÓiÐ1à�J€DØÔ)Õ)€JØ—	‘	˜$Ô,<Ó=€IÜ�jŠjœŸš 9Ó-°2Ô6€Gà—=‘=“?¤[Õ0€LÜ—;’;˜|°Ô;€Lô —’˜LÓ)€JÜ—’˜JÓ'×*Ñ*¬5¯;©;Ó7¼)ÕC€JÜ—’˜Z¨¨CÓ0€Jà"Ÿ™¤e§k¡kÓ2ÐÜ"Ð#4Ó5Ðð ˜A‘€IÜ—{’{ 9¬e¯oªoÐ>NÓ.OÐQaÓbÐð —/‘/Ó#Ð&6×&@Ñ&@ÀÓ&DÕD€KÜ—+’+˜i×1Ñ1°"Ó5´u×7GÒ7GÈÓ7TÐVaÓb€Kô —+’+˜k¬K¨<¼ÓE€KØ×"Ñ" :Ó.×1Ñ1´%×2EÑ2EÓF€Hô Ÿš I¬u×/?Ò/?Ð@QÓ/RÐTeÓfÐô  Ð 1¸5ÔA€Nà�^×(Ñ(¬×)=Ñ)=Ó>Ð>Ð>r%   c                ó˜   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  /# )r   rƒ   r„   r   r   r   )r"   s   "r#   r$   r$   K  sF   € ÷ %@ñ %@Ü�‰ð%@ä—,‘,ð%@ô —‘ð%@ô ‡\�\ñ	%@r%   c                ó€  € V P                   pVw  rEV\        ,          pVP                  \        P                  4      p\        VVVR7      p\        V4      p	V P                  \        P                  4      P                  WF\        4      p
W©P                  R4      ,          pVP                  V4      P                  V4      # )zöDequantize tensor from MXFP8 format.

Args:
    qx: Quantized FP8 tensor (float8_e4m3fn or float8_e5m2)
    block_scales: E8M0 block scales in swizzled layout (float8_e8m0fnu)
    output_type: Target output dtype

Returns:
    Dequantized tensor
r†   rc   )rd   r¹   rn   r   r¼   r   r   r.   rj   rh   rl   )rƒ   r„   r   ro   rp   rq   r¾   Úblock_scales_uint8r•   rÄ   Údata_f32r–   s   &&&         r#   Údequantize_mxfp8rÌ   K  s®   € ð —‘€JØ�J€DØÔ)Õ)€Jð &×*Ñ*¬5¯;©;Ó7ÐÜ*ØØØôÐô #Ð#:Ó;Ðð �u‰u”U—]‘]Ó#×+Ñ+¨DÔ>NÓO€Hð  ×"<Ñ"<¸RÓ"@Õ@Ðð ×#Ñ# JÓ/×2Ñ2°;Ó?Ð?r%   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  /# )	r   rš   r›   rž   rŸ   r    Nr¡   r   r   )r"   s   "r#   r$   r$   s  st   € ÷ #ñ #Ü‡|�|ð#ä‡|�|ð#ô —<‘<ð#ô —<‘<ð	#ô
 �,‰,˜Õ
ð#ô �{‰{˜TÕ!ð#ô ‡\�\ñ#r%   c                ó¼   € \        V VP                  4       VVVV\        P                  \        P                  \        P
                  \        P
                  R7
      pV# )aø  MXFP8 matrix multiplication using block-wise E8M0 scales.

Computes: output = a @ b.T + bias (linear semantics)

Args:
    a: FP8 matrix A of shape (M, K)
    b: FP8 matrix B of shape (N, K) - will be transposed internally
    block_scale_a: E8M0 block scales for A in swizzled format
    block_scale_b: E8M0 block scales for B in swizzled format
    bias: Optional bias vector of shape (N,)
    out_dtype: Output dtype

Scales are expected to be in swizzled (SWIZZLE_32_4_4) format from quantize_mxfp8.
r¥   )r   r­   r   ÚBlockWise1x32r   r°   )rš   r›   rž   rŸ   r    r¡   r—   s   &&&&&& r#   Úscaled_mm_mxfp8rÐ   s  sQ   € ô, Ø	Ø	�‰‹ØØØØÜ"×0Ñ0Ü"×0Ñ0Ü×,Ñ,Ü×,Ñ,ô€Fð €Mr%   zcomfy_kitchen::quantize_fp8)Úmutates_argsc                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# ©r   r   r   Úoutput_dtype_coder   ©r   r    r�   )r"   s   "r#   r$   r$      ó<   € ÷ ñ Ü‡|�|ðä�<‰<ðô ðô ‡\�\ñ	r%   c                ól   € \         V,          pRV RVRV/p\        P                  ! RVR7      pV! R/ VB # )zÝQuantize tensor to FP8 format with per-tensor scaling.

Args:
    x: Input tensor
    scale: Scale tensor (scalar)
    output_dtype_code: FP8 dtype code (5=float8_e4m3fn, 6=float8_e5m2)

Returns:
    Quantized FP8 tensor
r   r   r   r2   ©Úkwargs© ©ÚDTYPE_CODE_TO_DTYPEr   Úget_implementation©r   r   rÔ   Úoutput_dtyperÙ   Úimpls   &&&   r#   Ú_op_quantize_fp8rá   Ÿ  s@   € ô  'Ð'8Õ9€LØ�1�g˜u m°\ÐB€FÜ×&Ò&Ð'@ÈÔP€DÙ‰>�&‰>Ðr%   c                 óJ   € \         V,          p\        P                  ! WR 7      # r5   ©rÜ   r   Ú
empty_like©r   r   rÔ   rß   s   &&& r#   Ú_op_quantize_fp8_fakeræ   µ  ó   € ä&Ð'8Õ9€LÜ×Ò˜AÔ2Ð2r%   zcomfy_kitchen::dequantize_fp8c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# rÓ   rÕ   )r"   s   "r#   r$   r$   ¼  rÖ   r%   c                ól   € \         V,          pRV RVRV/p\        P                  ! RVR7      pV! R/ VB # )a"  Dequantize tensor from FP8 format with per-tensor scaling.

Args:
    x: Input FP8 tensor (float8_e4m3fn or float8_e5m2)
    scale: Scale tensor (scalar)
    output_dtype_code: Target dtype code (0=float32, 1=float16, 2=bfloat16)

Returns:
    Dequantized tensor in specified output format
r   r   r   r8   rØ   rÚ   rÛ   rÞ   s   &&&   r#   Ú_op_dequantize_fp8rê   »  s@   € ô  'Ð'8Õ9€LØ�1�g˜u m°\ÐB€FÜ×&Ò&Ð'BÈ6ÔR€DÙ‰>�&‰>Ðr%   c                 óJ   € \         V,          p\        P                  ! WR 7      # r5   rã   rå   s   &&& r#   Ú_op_dequantize_fp8_fakerì   Ñ  rç   r%   zcomfy_kitchen::quantize_nvfp4c                óÒ   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\        R\        \         P                  \         P                  3,          /# rT   rY   )r"   s   "r#   r$   r$   Ø  s^   € ÷ ñ Ü‡|�|ðä—l‘lðô ðô ð	ô
 ðô Œ5�<‰<œŸ™Ð%Õ&ñr%   c           
     óZ   € RV RVRVRVRV/p\         P                  ! RVR7      pV! R/ VB # )	aÈ  Quantize tensor to NVFP4 format with block-wise scaling.

Args:
    x: Input tensor (2D)
    per_tensor_scale: Global scale factor
    epsilon: Epsilon for numerical stability
    pad_16x: If True, pad dimensions to be divisible by 16
    hi_first: If True, the even-indexed element is packed into the high nibble (default).
              If False, the even-indexed element goes into the low nibble.

Returns:
    Tuple of (quantized_tensor, block_scales)
r   rU   rV   rW   rX   r�   rØ   rÚ   ©r   rÝ   )r   rU   rV   rW   rX   rÙ   rà   s   &&&&&  r#   Ú_op_quantize_nvfp4rð   ×  sC   € ð* �1Ð(Ð*:¸IÀwÐPYÐ[bÐdnÐpxÐy€FÜ×&Ò&Ð'7ÀÔG€DÙ‰>�&‰>Ðr%   c                 óx  € V P                   w  rVV'       d   \        V^4      p\        V^4      p\        P                  ! WV^,          3\        P                  V P
                  R7      p\        V^€4      p\        V^,          ^4      p	\        P                  ! W‰3\        P                  V P
                  R7      p
Wz3# )r^   ©r!   r‹   )rd   r   r   Úemptyr¼   r‹   r+   )r   rU   rV   rW   rX   rp   rq   ÚqdataÚ
scale_rowsÚ
scale_colsr„   s   &&&&&      r#   Ú_op_quantize_nvfp4_faker÷   ñ  s”   € à—‘�J€DçÜ�t˜RÓ ˆÜ�t˜RÓ ˆô �KŠK˜ q�yÐ)´·±ÀQÇXÁXÔN€Eô ˜˜sÓ#€JÜ˜ � QÓ'€JÜ—;’; 
Ð7¼u×?RÑ?RÐ[\×[cÑ[cÔd€LàÐÐr%   zcomfy_kitchen::dequantize_nvfp4c                ó°   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R\         P                  /# )r   rƒ   rU   r„   rÔ   rX   r   )r   r    r�   r[   )r"   s   "r#   r$   r$     sT   € ÷ ñ Ü�‰ðä—l‘lðô —,‘,ðô ð	ô
 ðô ‡\�\ñr%   c           
     ót   € \         V,          pRV RVRVRVRV/p\        P                  ! RVR7      pV! R/ VB # )	aù  Dequantize tensor from NVFP4 format with block-wise scaling.

Args:
    qx: Quantized FP4 tensor (packed as uint8)
    per_tensor_scale: Global scale factor
    block_scales: Block scales in swizzled layout (float8_e4m3fn)
    output_dtype_code: Target dtype code (0=float32, 1=float16, 2=bfloat16)
    hi_first: If True, the high nibble is the even-indexed element (default).
              If False, the low nibble is the even-indexed element.

Returns:
    Dequantized tensor in specified output format
rƒ   rU   r„   r   rX   r˜   rØ   rÚ   rÛ   )rƒ   rU   r„   rÔ   rX   rß   rÙ   rà   s   &&&&&   r#   Ú_op_dequantize_nvfp4rú     sY   € ô* 'Ð'8Õ9€LØ�BÐ*Ð,<¸nÈlÐ\iÐkwð  zDð  FNð  O€FÜ×&Ò&Ð'9À&ÔI€DÙ‰>�&‰>Ðr%   c                 óŒ   € \         V,          pV P                  w  rg\        P                  ! Wg^,          3WPP                  R7      # )r   rò   ©rÜ   rd   r   ró   r‹   )rƒ   rU   r„   rÔ   rX   rß   rp   Úcols_packeds   &&&&&   r#   Ú_op_dequantize_nvfp4_fakerþ     s5   € ä&Ð'8Õ9€LàŸ™Ñ€DÜ�;Š;˜¨A�oÐ.°lÏ9É9ÔUÐUr%   zcomfy_kitchen::scaled_mm_nvfp4c                ó`  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\        R
\         P                  R,          R\         P                  /
# )r   rš   r›   rœ   r�   rž   rŸ   r    NrÔ   r¢   r   rÕ   )r"   s   "r#   r$   r$   (  sš   € ÷ %ñ %Ü‡|�|ð%ä‡|�|ð%ô —L‘Lð%ô —L‘Lð	%ô
 —<‘<ð%ô —<‘<ð%ô �,‰,˜Õ
ð%ô ð%ô �<‰<˜$Õð%ô ‡\�\ñ%r%   c	                ó„   € \         V,          p	RV RVRVRVRVRVRVRV	R	V/	p
\        P                  ! R
V
R7      pV! R/ V
B # )a[  Matrix multiplication with NVFP4 quantized inputs.

Computes: y = (a @ b.T) * (tensor_scale_a * tensor_scale_b) + bias

Args:
    a: Quantized matrix A (M, K//2) in uint8 format
    b: Quantized matrix B (N, K//2) in uint8 format
    tensor_scale_a: Global scale for A
    tensor_scale_b: Global scale for B
    block_scale_a: Block-wise scales for A
    block_scale_b: Block-wise scales for B
    bias: Optional bias vector
    output_dtype_code: Output dtype code (0=float32, 1=float16, 2=bfloat16)
    alpha: Output scale (tensor_scale_a * tensor_scale_b)

Returns:
    Result tensor of shape (M, N)
rš   r›   rœ   r�   rž   rŸ   r    r¡   r¢   r²   rØ   rÚ   rÛ   )rš   r›   rœ   r�   rž   rŸ   r    rÔ   r¢   r¡   rÙ   rà   s   &&&&&&&&&   r#   Ú_op_scaled_mm_nvfp4r  '  s`   € ô< $Ð$5Õ6€IàˆQ��QØ˜.Ð*:¸NØ˜¨¸Ø��k 9¨g°uð	€Fô ×&Ò&Ð'8ÀÔH€DÙ‰>�&‰>Ðr%   c	                 ó®   € \         V,          p	V P                  ^ ,          p
VP                  ^ ,          p\        P                  ! W«3W�P                  R7      # ©r   rò   rü   )rš   r›   rœ   r�   rž   rŸ   r    rÔ   r¢   r¡   ÚmÚns   &&&&&&&&&   r#   Ú_op_scaled_mm_nvfp4_faker  P  s@   € ô
 $Ð$5Õ6€IØ	�‰��
€AØ	�‰��
€AÜ�;Š;˜�v Y·x±xÔ@Ð@r%   zcomfy_kitchen::quantize_mxfp8c                óš   € V ^8„  d   QhR\         P                  R\        R\        \         P                  \         P                  3,          /# r´   r¶   )r"   s   "r#   r$   r$   `  s<   € ÷ ñ Ü‡|�|ðäðô Œ5�<‰<œŸ™Ð%Õ&ñr%   c                óN   € RV RV/p\         P                  ! RVR7      pV! R/ VB # )a&  Quantize tensor to MXFP8 format with block-wise E8M0 scaling.

MXFP8 uses block size 32 with power-of-2 (E8M0) block scales.

Args:
    x: Input tensor (2D, shape M x K)
    pad_32x: If True, pad dimensions to be divisible by 32

Returns:
    Tuple of (quantized_fp8_tensor, block_scales_e8m0)
r   rµ   rÇ   rØ   rÚ   rï   )r   rµ   rÙ   rà   s   &&  r#   Ú_op_quantize_mxfp8r	  _  s1   € ð  �1�i Ð)€FÜ×&Ò&Ð'7ÀÔG€DÙ‰>�&‰>Ðr%   c                 ón  € V P                   w  r#V'       d   \        V^ 4      p\        V^ 4      p\        P                  ! W#3\        P                  V P
                  R7      pV^ ,          p\        V^€4      p\        V^4      p\        P                  ! Wg3\        P                  V P
                  R7      pWH3# )é    rò   )rd   r   r   ró   r+   r‹   r½   )	r   rµ   rp   rq   rô   r¾   rõ   rö   r„   s	   &&       r#   Ú_op_quantize_mxfp8_faker  t  s–   € à—‘�J€DçÜ�t˜RÓ ˆÜ�t˜RÓ ˆô �KŠK˜˜¬E×,?Ñ,?ÈÏÉÔQ€Eð ˜•€JÜ˜˜sÓ#€JÜ˜ QÓ'€JÜ—;’; 
Ð7¼u×?SÑ?SÐ\]×\dÑ\dÔe€LàÐÐr%   zcomfy_kitchen::dequantize_mxfp8c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# )r   rƒ   r„   rÔ   r   rÕ   )r"   s   "r#   r$   r$   Š  s<   € ÷ ñ Ü�‰ðä—,‘,ðô ðô ‡\�\ñ	r%   c                ól   € \         V,          pRV RVRV/p\        P                  ! RVR7      pV! R/ VB # )a)  Dequantize tensor from MXFP8 format.

Args:
    qx: Quantized FP8 tensor (float8_e4m3fn)
    block_scales: E8M0 block scales in swizzled layout (float8_e8m0fnu)
    output_dtype_code: Target dtype code (0=float32, 1=float16, 2=bfloat16)

Returns:
    Dequantized tensor in specified output format
rƒ   r„   r   rÌ   rØ   rÚ   rÛ   )rƒ   r„   rÔ   rß   rÙ   rà   s   &&&   r#   Ú_op_dequantize_mxfp8r  ‰  s@   € ô  'Ð'8Õ9€LØ�B˜¨°mÀ\ÐR€FÜ×&Ò&Ð'9À&ÔI€DÙ‰>�&‰>Ðr%   c                 óJ   € \         V,          p\        P                  ! WR 7      # r5   rã   )rƒ   r„   rÔ   rß   s   &&& r#   Ú_op_dequantize_mxfp8_faker  Ÿ  s   € ä&Ð'8Õ9€LÜ×Ò˜BÔ3Ð3r%   zcomfy_kitchen::scaled_mm_mxfp8c                óò   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\         P                  /# )	r   rš   r›   rž   rŸ   r    NrÔ   r   rÕ   )r"   s   "r#   r$   r$   ¦  sk   € ÷ ñ Ü‡|�|ðä‡|�|ðô —<‘<ðô —<‘<ð	ô
 �,‰,˜Õ
ðô ðô ‡\�\ñr%   c                óx   € \         V,          pRV RVRVRVRVRV/p\        P                  ! RVR7      pV! R	/ VB # )
aµ  Matrix multiplication with MXFP8 quantized inputs.

Computes: y = a @ b.T + bias

Args:
    a: Quantized FP8 matrix A (M, K)
    b: Quantized FP8 matrix B (N, K)
    block_scale_a: E8M0 block scales for A in swizzled layout
    block_scale_b: E8M0 block scales for B in swizzled layout
    bias: Optional bias vector
    output_dtype_code: Output dtype code (0=float32, 1=float16, 2=bfloat16)

Returns:
    Result tensor of shape (M, N)
rš   r›   rž   rŸ   r    r¡   rÐ   rØ   rÚ   rÛ   )	rš   r›   rž   rŸ   r    rÔ   r¡   rÙ   rà   s	   &&&&&&   r#   Ú_op_scaled_mm_mxfp8r  ¥  sR   € ô0 $Ð$5Õ6€IàˆQ��QØ˜¨¸Ø��k 9ð€Fô
 ×&Ò&Ð'8ÀÔH€DÙ‰>�&‰>Ðr%   c                 ó®   € \         V,          pV P                  ^ ,          pVP                  ^ ,          p\        P                  ! Wx3W`P                  R7      # r  rü   )	rš   r›   rž   rŸ   r    rÔ   r¡   r  r  s	   &&&&&&   r#   Ú_op_scaled_mm_mxfp8_faker  Ç  s@   € ô $Ð$5Õ6€IØ	�‰��
€AØ	�‰��
€AÜ�;Š;˜�v Y·x±xÔ@Ð@r%   c                ó0   € V ^8„  d   QhR\         R\        /# )r   Údevice_indexr   )r�   r[   )r"   s   "r#   r$   r$   Ù  s   € ÷ 	ñ 	¬ð 	´ñ 	r%   c                 óº   € \         P                  V 4      pVe   V#  \        P                  P	                  V 4      R8H  pV\         V &   V#   \
         d    Rp Li ; i)NF)é   é   )Ú_turing_device_cacherŠ   r   ÚcudaÚget_device_capabilityÚRuntimeError)r  ÚcachedÚ	is_turings   &  r#   Ú_cuda_device_is_turingr"  Ù  sd   € Ü!×%Ñ% lÓ3€FØÒØˆðÜ—J‘J×4Ñ4°\ÓBÀfÑLˆ	ð *3Ô˜Ñ&ØÐøô ô ØŠ	ðús   �"A
 Á
AÁAc                óD   € V ^8„  d   QhR\         P                  R\        /# )r   Útensorr   rÕ   )r"   s   "r#   r$   r$   å  s"   € ÷ Wñ W¤§¡ð W´#ñ Wr%   c                 ól   € V P                   '       d"   \        V P                  4       4      '       d   ^ # ^# )r  )Úis_cudar"  Ú
get_device)r$  s   &r#   Ú_int8_mm_n_alignmentr(  å  s*   € à——�Ô$:¸6×;LÑ;LÓ;N×$OÒ$Oˆ2ÐVÐUVÐVr%   c                ó<   € V ^8„  d   QhR\         R\         R\         /# )r   ÚvalueÚ	alignmentr   )r�   )r"   s   "r#   r$   r$   ê  s!   € ÷ >ñ >”Sð >¤Sð >¬Sñ >r%   c                 ó<   € W,           ^,
          V,          V,          # ©r¤   rÚ   )r*  r+  s   &&r#   Ú	_round_upr.  ê  s   € ØÕ Õ" yÕ0°IÕ=Ð=r%   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# ©r   rš   r›   r   ©r   r    )r"   s   "r#   r$   r$   î  s-   € ÷ !ñ !œuŸ|™|ð !´·±ð !ÄÇÁñ !r%   c                óÄ  € R R lpV P                  ^ 4      pVP                  ^4      pV P                  ^4      pV^ 8X  g   V^ 8X  d	   V! W4      # V P                  '       d   \        \        V^ 4      ^ 4      MTpWc8w  dO   \        P
                  ! Wc,
          V3V P                  V P                  R7      p\        P                  ! W3^ R7      p V^,           ^,          ^,          pW…8w  d»   \        P
                  ! V P                  ^ 4      W…,
          3V P                  V P                  R7      p	\        P
                  ! W…,
          VP                  ^4      3VP                  VP                  R7      p
\        P                  ! W	3^R7      p \        P                  ! W3^ R7      p\        V\        V 4      4      pW´8w  d^   \        P
                  ! VP                  ^ 4      W´,
          3VP                  VP                  R7      p
\        P                  ! W3^R7      pV! W4      pVP                  ^ 4      V8w  g   VP                  ^4      V8w  d   VRV1RV13,          pV# )z5Multiply INT8 matrices and return INT32 accumulators.c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r   ÚlhsÚrhsr   r1  )r"   s   "r#   r$   Ú-_int8_matmul_accumulate.<locals>.__annotate__ð  s-   € ÷ 'ñ 'œ%Ÿ,™,ð '¬U¯\©\ð '¼e¿l¹lñ 'r%   c                 ó†   € \        \        R 4      '       d   \        P                  ! W4      # \        P                  ! W4      # )Úint8_mm)Úhasattrr   r8  Ú_int_mm)r4  r5  s   &&r#   Úfast_int8_mmÚ-_int8_matmul_accumulate.<locals>.fast_int8_mmð  s-   € Ü”5˜)×$Ò$Ü—=’= Ó*Ð*Ü�}Š}˜SÓ&Ð&r%   ©r‹   r!   r_   N)
Úsizer&  r.  rI   r   Úzerosr‹   r!   Úcatr(  )rš   r›   r;  Úorig_mÚorig_nÚkÚpadded_mÚrow_paddingÚpadded_kÚ	a_paddingÚ	b_paddingÚpadded_nr—   s   &&           r#   Ú_int8_matmul_accumulaterJ  î  sÑ  € õ'ð
 �V‰V�A‹Y€FØ�V‰V�A‹Y€FØ	�‰ˆq‹	€AØ�„{�a˜1”fÙ˜AÓ!Ð!à12··°Œyœ˜V R›¨"Ô-À€HØÔÜ—k’k 8Õ#4°aÐ"8ÀÇÁÐQR×QXÑQXÔYˆÜ�IŠI�qÐ&¨AÔ.ˆà�Q•˜1• Õ!€HØ„}Ü—K’K §¡¨£¨H­LÐ 9À!Ç(Á(ÐRS×RYÑRYÔZˆ	Ü—K’K ¥¨q¯v©v°a«yÐ 9À!Ç(Á(ÐRS×RYÑRYÔZˆ	Ü�IŠI�q�n¨!Ô,ˆÜ�IŠI�q�n¨!Ô,ˆä˜Ô!5°aÓ!8Ó9€HØÔÜ—K’K §¡¨£¨HÕ,=Ð >ÀqÇxÁxÐWX×W^ÑW^Ô_ˆ	Ü�IŠI�q�n¨!Ô,ˆá˜!Ó€FØ‡{�{�1ƒ~˜Ô 6§;¡;¨q£>°VÔ#;Ø˜˜˜  & Ð(Õ)ˆØ€Mr%   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# r0  r1  )r"   s   "r#   r$   r$     s-   € ÷ )ñ )Œu�|‰|ð )¤§¡ð )´·±ñ )r%   c                ó�  € V P                   \        P                  8X  d    VP                   \        P                  8X  g   Q hV P                  4       ^8X  d   VP                  4       ^8X  g   Q hV P	                  ^4      VP	                  ^ 4      8X  g,   Q RV P	                  ^4       RVP	                  ^ 4       24       h\        W4      # )zíINT8 matrix multiplication: C[M,N] = A[M,K] @ B[K,N].

Uses torch._int_mm (cuBLASLt on CUDA). Output is int32.

Args:
    a: INT8 tensor [M, K].
    b: INT8 tensor [K, N].

Returns:
    INT32 tensor [M, N] with accumulated dot products.
zK mismatch: z vs )r!   r   Úint8r`   r>  rJ  )rš   r›   s   &&r#   Úmm_int8rN    s“   € ð �7‰7”e—j‘jÔ  Q§W¡W´·
±
Ô%:Ð:Ð:Ø�5‰5‹7�aŒ<˜AŸE™E›G qœLÐ(Ð(Ø�6‰6�!‹9˜Ÿ™˜q›	Ô!ÐL \°!·&±&¸³)°¸DÀÇÁÈÃÀÐ#LÓLÐ!Ü" 1Ó(Ð(r%   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r   r   Úseedr   rÕ   )r"   s   "r#   r$   r$   $  s)   € ÷ 	ñ 	œEŸL™Lð 	´ð 	¼¿¹ñ 	r%   c                 óì   € \         P                  ! V P                  R 7      pVP                  V4       \         P                  ! V P
                  V P                  V P                  V P                  VR7      # )©r‹   )r!   Úlayoutr‹   Ú	generator)r   Ú	Generatorr‹   Úmanual_seedÚrandrd   r!   rS  )r   rP  rT  s   && r#   Ú_int8_stochastic_rngrX  $  sR   € Ü—’ q§x¡xÔ0€IØ×Ñ˜$ÔÜ�:Š:Ø	�‰Ø�g‰gØ�x‰xØ�x‰xØôð r%   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r   r   r   r   r1  )r"   s   "r#   r$   r$   0  s2   € ÷ Mñ M¤§¡ð M´·±ð MÄ%Ç,Á,ñ Mr%   c                 ó  € V P                  VP                  VP                  R 7      p \        P                  ! VP                  4      P
                  p\        P                  ! V ^ 8H  \        P                  ! W4      V 4      # )r=  )r.   r‹   r!   r   rN   Útinyr;   Ú	full_like)r   r   Ú	scale_mins   && r#   Ú_int8_scale_for_mathr^  0  sT   € Ø�H‰H˜AŸH™H¨A¯G©GˆHÓ4€EÜ—’˜AŸG™GÓ$×)Ñ)€IÜ�;Š;�u ‘z¤5§?¢?°5Ó#DÀeÓLÐLr%   c                ór   € V ^8„  d   QhR\         P                  R\        R,          R\         P                  /# )r   ÚscaledÚstochastic_roundingNr   rÕ   )r"   s   "r#   r$   r$   6  s2   € ÷ @ñ @œŸ™ð @¼3À½:ð @ÌeÏlÉlñ @r%   c                 óD  € Vea   V^ 8”  dZ   \        W4      pV P                  V4       V P                  4       P                  RR4      P	                  \
        P                  4      # V P                  4       P                  RR4      P	                  \
        P                  4      # )Nç     À_@g      `À)rX  Úadd_Úfloor_Úclamp_r.   r   rM  Úround_)r`  ra  rB   s   && r#   Ú_round_int8rh  6  sv   € ØÒ&Ð+>ÀÔ+BÜ" 6Ó?ˆØ�‰�CÔØ�}‰}‹×%Ñ% f¨eÓ4×7Ñ7¼¿
¹
ÓCÐCØ�=‰=‹?×!Ñ! &¨%Ó0×3Ñ3´E·J±JÓ?Ð?r%   c          
      ó  € V ^8„  d   QhR\         P                  R\         P                  \        ,          \        ,          R,          R\        R,          R\
        \         P                  \         P                  3,          /# )r   r   r   Nra  r   )r   r    rZ   Ústrr�   r\   )r"   s   "r#   r$   r$   >  s]   € ÷ ñ Ü‡|�|ðä�<‰<œ%Õ¤#Õ%¨Õ,ðô ˜t�ðô Œ5�<‰<œŸ™Ð%Õ&ñ	r%   c                ó  € Ve   \        V\        4      '       dN   VR8X  dG   V P                  4       P                  4       pVP	                  4       R,          P                  RR7      pM|\        V\        P                  4      '       g2   \        P                  ! WP                  \        P                  R7      pM+VP                  V P                  \        P                  R7      p\        V \        W4      ,          VR7      pWA3# )a|  Quantize tensor to INT8 with single tensorwise scale.

Args:
    x: Input tensor of any shape.
    scale: Optional scale. "recalculate" computes scale from absmax.
    stochastic_rounding: Seed for stochastic rounding. Disabled when <= 0.

Returns:
    Tuple of (quantized_int8, scale):
        - quantized_int8: INT8 tensor with same shape
        - scale: Scalar float32 tensor
Úrecalculaterc  ç ÂëþKH´9r¸   r=  ©ra  )Ú
isinstancerj  rL   rI   rZ   r/   r   r    r$  r‹   rj   r.   rh  r^  )r   r   ra  Úabs_maxÚqs   &&&  r#   Úquantize_int8_tensorwiserr  >  s­   € ð" ‚}œ E¬3×/Ò/°E¸]Ô4JØ—%‘%“'—+‘+“-ˆØ—‘“ 5Õ(×/Ñ/°EÐ/Ó:‰Ü˜œuŸ|™|×,Ò,Ü—’˜U¯8©8¼5¿=¹=ÔI‰à—‘ §¡´·±�Ó>ˆÜ�AÔ,¨UÓ6Õ6ÐL_Ô`€AØˆ8€Or%   c                ó¨   € V ^8„  d   QhR\         P                  R\        R,          R\        \         P                  \         P                  3,          /# )r   r   ra  Nr   ©r   r    r�   r\   )r"   s   "r#   r$   r$   Z  s@   € ÷ ñ Ü‡|�|ðä˜t�ðô Œ5�<‰<œŸ™Ð%Õ&ñr%   c                óÒ   € V P                  4       P                  RRR7      pVP                  4       R,          P                  RR7      p\	        V \        W04      ,          VR7      pWC3# )ar  Quantize tensor to INT8 with per-row scales (for activations).

Args:
    x: Input tensor [..., K] where quantization is per-row.
    stochastic_rounding: Seed for stochastic rounding. Disabled when <= 0.

Returns:
    Tuple of (quantized_int8, scales):
        - quantized_int8: INT8 tensor with same shape
        - scales: Float32 tensor [..., 1] with per-row scales
T)r`   Úkeepdimrc  rm  r¸   rn  rc   )rL   ri   rZ   r/   rh  r^  )r   ra  rp  r   rq  s   &&   r#   Úquantize_int8_rowwiserw  Z  sY   € ð �e‰e‹g�l‰l˜r¨4ˆlÓ0€GØ�]‰]‹_˜uÕ$×+Ñ+°Ð+Ó6€EÜ�AÔ,¨UÓ6Õ6ÐL_Ô`€AØˆ8€Or%   c                óÔ   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R,          R\        \         P                  \         P                  3,          /# )r   r   ÚhÚ
group_sizera  Nr   rt  )r"   s   "r#   r$   r$   o  s_   € ÷ Qñ QÜ‡|�|ðQä‡|�|ðQô ðQô ˜t�ð	Qô
 Œ5�<‰<œŸ™Ð%Õ&ñQr%   c                ó2   € \        WV4      p\        WCR7      # )a,  Fused online activation rotation + row-wise quantization (Eager fallback).

Args:
    x: Input unrotated activation tensor [M, K].
    h: Pre-built normalized Hadamard matrix [group_size, group_size].
    group_size: ConvRot group size.

Returns:
    Tuple of (rotated_quantized_x_int8, row_scales).
rn  )r   rw  )r   ry  rz  ra  Úx_rots   &&&& r#   Úquantize_and_rotate_rowwiser}  o  s   € ô  ˜q ZÓ0€EÜ  ÔPÐPr%   c          
      ó´   € V ^8„  d   QhR\         P                  R\        R\        R,          R\        \         P                  \         P                  3,          /# )r   Úweightrz  ra  Nr   rt  )r"   s   "r#   r$   r$   ƒ  sP   € ÷ Vñ VÜ�L‰LðVäðVô ˜t�ðVô Œ5�<‰<œŸ™Ð%Õ&ñ	Vr%   c                ót   € \        WP                  V P                  R7      p\        WV4      p\	        WBR7      # )zGOffline ConvRot weight rotation followed by row-wise INT8 quantization.r=  rn  )r   r‹   r!   r   rw  )r  rz  ra  ry  Ú
weight_rots   &&&  r#   Úquantize_int8_convrot_weightr‚  ƒ  s/   € ô 	˜
¯=©=ÀÇÁÔM€AÜ ¨:Ó6€JÜ  ÔUÐUr%   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# ©r   r  rz  r   rÕ   )r"   s   "r#   r$   r$   Ž  s)   € ÷ 1ñ 1¤u§|¡|ð 1Äð 1ÌÏÉñ 1r%   c                ó\   € \        WP                  V P                  R7      p\        WV4      # )zOPortable ConvRot weight rotation used when no accelerated backend is available.r=  )r   r‹   r!   r   )r  rz  ry  s   && r#   Úrotate_int8_convrot_weightr†  Ž  s#   € ä˜
¯=©=ÀÇÁÔM€AÜ˜& ZÓ0Ð0r%   c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# ©r   rq  r   rz  r   rÕ   )r"   s   "r#   r$   r$   ”  s>   € ÷ Kñ K¤e§l¡lð K¼5¿<¹<ð KÔUXð KÔ]b×]iÑ]iñ Kr%   c                óv   € \        W P                  \        P                  R7      p\	        \        W4      W24      # )zKDequantize INT8 ConvRot weights and rotate them back to the original basis.r=  )r   r‹   r   rj   r   Údequantize_int8_simple)rq  r   rz  ry  s   &&& r#   Údequantize_int8_convrot_weightr‹  ”  s)   € ä˜
¯8©8¼5¿=¹=ÔI€AÜÔ0°Ó:¸AÓJÐJr%   c          
      ó�   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\         P                  /# ©r   rq  r   rz  rÔ   r   rÕ   )r"   s   "r#   r$   r$   š  sG   € ÷ kñ kÜ‡|�|ðkÜ!ŸL™LðkÜ69ðkÜNQðkä
‡\�\ñkr%   c                óN   € \        WV4      P                  \        V,          4      # )z@Dequantize INT8 ConvRot weights into a requested floating dtype.)r‹  r.   rÜ   ©rq  r   rz  rÔ   s   &&&&r#   Ú$dequantize_int8_convrot_weight_dtyper�  š  s#   € ô *¨!°JÓ?×BÑBÔCVÐWhÕCiÓjÐjr%   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# ©r   rq  r   r   r1  )r"   s   "r#   r$   r$   ¡  s-   € ÷ 
ñ 
œeŸl™lð 
´5·<±<ð 
ÄEÇLÁLñ 
r%   c                ó0   € V P                  4       V,          # )z£Dequantize INT8 tensor with scale.

Args:
    q: Quantized INT8 tensor.
    scale: Scale tensor (scalar or broadcastable).

Returns:
    Dequantized float tensor.
)rZ   ©rq  r   s   &&r#   rŠ  rŠ  ¡  s   € ð �7‰7‹9�uÕÐr%   c                ó°   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R\         P                  /# ©r   rq  r   Úindicesrz  rÔ   r   rÕ   )r"   s   "r#   r$   r$   ®  sT   € ÷ 8ñ 8Ü‡|�|ð8ä�<‰<ð8ô �\‰\ð8ô ð	8ô
 ð8ô ‡\�\ñ8r%   c                óº  € \         P                  P                  P                  W 4      P	                  \         P
                  4      pVP                  4       ^8¼  dO   V\         P                  P                  P                  W!4      P	                  \         P
                  4      ,          pM%WQP	                  \         P
                  4      ,          pV^ 8”  dl   \        W0P                  \         P
                  R7      p\        VP                  RVP                  R,          4      Wc4      P                  VP                  4      pVP	                  \        V,          4      # )aµ  Gather rows from an INT8 embedding table ``[vocab, dim]`` and dequantize only those rows.

Indexes first, unlike ``dequantize_int8_convrot_weight*`` which materializes the whole table.
``scale`` is per-row ``[vocab, 1]`` or scalar. ``group_size <= 0`` means the table is not
ConvRot-rotated; if it is, the rows are un-rotated after the lookup (a Linear folds that into
its GEMM, a lookup has no GEMM). Returns ``[*indices.shape, dim]``.
r=  rc   )r   re   rf   rŽ   r.   rj   r`   r   r‹   r   rh   rd   rÜ   )rq  r   r—  rz  rÔ   r   ry  s   &&&&&  r#   Údequantize_int8_embeddingr™  ®  sÝ   € ô 	�‰×Ñ×%Ñ% gÓ1×4Ñ4´U·]±]ÓC€AØ‡y�yƒ{�aÔØ”—‘×#Ñ#×-Ñ-¨gÓ=×@Ñ@ÄÇÁÓOÕO‰à—‘œŸ™Ó'Õ'ˆØ�A„~Ü˜J¯x©x¼u¿}¹}ÔMˆÜ˜1Ÿ9™9 R¨¯©°­Ó5°qÓE×MÑMÈaÏgÉgÓVˆØ�4‰4Ô#Ð$5Õ6Ó7Ð7r%   c                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# ©r   rq  r   rÔ   r   rÕ   )r"   s   "r#   r$   r$   Ç  s>   € ÷ Wñ W¤E§L¡Lð W¼¿¹ð WÔZ]ð WÔbg×bnÑbnñ Wr%   c                óL   € \        W4      P                  \        V,          4      # )zBDequantize INT8 tensor with scale into a requested floating dtype.)rŠ  r.   rÜ   ©rq  r   rÔ   s   &&&r#   Údequantize_int8_simple_dtyperž  Ç  s   € ä! !Ó+×.Ñ.Ô/BÐCTÕ/UÓVÐVr%   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  R,          R\         P                  /# )r   r   r  r    NÚresidualÚresidual_scaler   r1  )r"   s   "r#   r$   r$   Ì  sk   € ÷ 
:ñ 
:Ü‡|�|ð
:ä�L‰Lð
:ô �,‰,˜Õ
ð
:ô �l‰l˜TÕ!ð	
:ô
 —L‘L 4Õ'ð
:ô ‡\�\ñ
:r%   c                ón   € \         P                  P                  P                  WV4      p\	        WSV4      # )zd``linear(x)``, or ``residual + residual_scale * linear(x)``; accumulates
in torch's configured mode.)r   re   rf   ÚlinearÚ_apply_residual)r   r  r    r   r¡  r*   s   &&&&& r#   Úfp16_linearr¥  Ì  s-   € ô �(‰(×
Ñ
×
$Ñ
$ Q°Ó
5€CÜ˜3¨.Ó9Ð9r%   é   c                ó®  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R\        R\        R	\
        R,          R
\         P                  R,          R\        R\         P                  R,          R\         P                  R,          R\         P                  /# )r   r   r  Úweight_scaler    Nr¡   ÚconvrotÚconvrot_groupsizeÚ	input_actÚinput_act_weightÚinput_act_epsr   r¡  r   )r   r    r!   r[   r�   rj  rZ   )r"   s   "r#   r$   r$   Ù  sÕ   € ÷ `=ñ `=Ü‡|�|ð`=ä�L‰Lð`=ô —,‘,ð`=ô �,‰,˜Õ
ð	`=ô
 �{‰{ð`=ô ð`=ô ð`=ô �T�zð`=ô —l‘l TÕ)ð`=ô ð`=ô �l‰l˜TÕ!ð`=ô —L‘L 4Õ'ð`=ô ‡\�\ñ`=r%   c           	     óä  € \        WW‰4      p V P                  R,          VP                  R,          8w  d4   \        RV P                  R,           RVP                  R,           24      hVP                  V P                  R7      P                  4       pVP                  V P                  \        P                  R7      P                  R4      pVP                  4       ^VP                  ^ ,          39  d8   \        R\        VP                  4       R\        VP                  4       24      hV'       do   V P                  R,          V,          ^ 8w  d#   \        RV RV P                  R,           24      h\        W`P                  V P                  R7      p\        WV4      p V P                  pV P                  RV P                  R,          4      p\        V4      w  pp\        WñP                   P                  4       4      pVP                  w  pp\#        ^\%        VRV^,          ,          4      4      pVP                  ^R4      p. p\'        ^ VV4       F‰  p\%        VV,           V4      pVVV P)                  4       pVVV P                  VP                  \        P                  R7      V,          pVV,          pVP                  V4      pVP+                  V4       K‹  	  \        P,                  ! V^ R	7      pVe?   VVP                  VP                  VP                  R7      P                  ^R4      ,           pVP                  ! . VR
R OVP                  ^ ,          N5!  p\/        VW«4      # )av  INT8 linear layer using torch.int8_mm with memory-efficient scaling.

Quantizes x dynamically per-row, uses tensorwise weight scale.
Processes scaling in chunks to avoid materializing large float32 tensors.

Args:
    x: Input tensor [..., K].
    weight: INT8 weight tensor [N, K].
    weight_scale: Scalar weight scale.
    bias: Optional bias [N].
    out_dtype: Output dtype.
    convrot: If True, apply online activation rotation.
    convrot_groupsize: Group size for Hadamard rotation.
    input_act: Optional activation folded in before quantization.
    input_act_weight: K-element weight for input_act "rms_norm".
    input_act_eps: Eps for input_act "rms_norm".
    residual: Optional [..., N] tensor; the result becomes
        ``residual + residual_scale * linear(x)``.
    residual_scale: Per-channel [N] scale for the residual form.

Returns:
    Result tensor [..., N].
z2Input and weight inner dimensions must match, got z and rR  r=  z<INT8 weight scale must be scalar or per-output-channel, got z for weight shape zConvRot group size z  does not divide input features r_   Nrc   i   )Ú_apply_input_actrd   r-   r.   r‹   Ú
contiguousr   rj   rh   Únumelr\   r   r!   r   rw  rJ  ÚTrI   rH   ÚrangerZ   Úappendr@  r¤  )r   r  r¨  r    r¡   r©  rª  r«  r¬  r­  r   r¡  ry  ro   Úx_2dÚx_8Úx_scaler—   r  r  Ú
chunk_sizeÚscaled_partsÚiÚend_iÚchunkÚchunk_scalesÚchunk_scaleds   &&&&&&&&&&&&               r#   Úint8_linearr¿  Ù  sø  € ôJ 	˜Ð'7ÓG€AØ‡w�wˆr…{�f—l‘l 2Õ&Ô&ÜØ@ÀÇÁÈÅÀÈUÐSY×S_ÑS_Ð`bÕScÐRdÐeó
ð 	
ð �Y‰Y˜aŸh™hˆYÓ'×2Ñ2Ó4€FØ—?‘?¨!¯(©(¼%¿-¹-�?ÓH×PÑPÐQSÓT€LØ×ÑÓ A v§|¡|°A¥Ð#7Ô7ÜØJÌ5ÐQ]×QcÑQcÓKdÐJeð f Ü % f§l¡lÓ 3Ð4ð6ó
ð 	
÷
 Ø�7‰7�2�;Ð*Õ*¨aÔ/ÜØ%Ð&7Ð%8Ð8XÐYZ×Y`ÑY`ÐacÕYdÐXeÐfóð ô Ð-·h±hÀaÇgÁgÔNˆÜ˜qÐ%6Ó7ˆà—‘€JØ�9‰9�R˜Ÿ™ �Ó%€Dô )¨Ó.�L€Cˆô % S¯(©(×*=Ñ*=Ó*?Ó@€Fð �<‰<�D€A€qÜ�Qœ˜AÐ0°Q¸µUÕ;Ó<Ó=€Jà×'Ñ'¨¨2Ó.€LØ€LÜ�1�a˜Ö$ˆÜ�A˜
•N AÓ&ˆØ�q˜�×%Ñ%Ó'ˆð ˜q Ð'×*Ñ*°%·,±,ÄeÇmÁmÐ*ÓTÐWcÕcˆØ˜|Õ+ˆð $—‘ yÓ1ˆØ×Ñ˜LÖ)ñ %ô �YŠY�|¨Ô+€FàÒØ˜$Ÿ'™'¨¯©¸f¿l¹l˜'ÓK×SÑSÐTUÐWYÓZÕZˆà�^Š^Ð>˜Z¨¨˜_Ð>¨f¯l©l¸1­oÓ>€FÜ˜6 8Ó<Ð<r%   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r   r   r   r   r1  )r"   s   "r#   r$   r$   <  s-   € ÷ 	,ñ 	,œEŸL™Lð 	,´·±ð 	,Ä%Ç,Á,ñ 	,r%   c                ó  € V P                   R,          pVP                  4       V8w  d    \        RV RVP                  4        24      hRV P                  ^,
          ,          V3,           pV RVP	                  V4      ,           ,          # )z<Materialize the portable batch-one modulation contract once.z3modulation_scale must contain one batch-one row of z values, got r)   rc   r-  )rd   r±  r-   Úndimrh   )r   r   rC  rd   s   &&  r#   Ú_modulate_int8_inputrÃ  <  s{   € à	�‰��€AØ‡{�{ƒ}˜ÔÜØAÀ!Àð EØ—;‘;“=�/ð#ó
ð 	
ð �A—F‘F˜Q•JÕ 1 $Õ&€EØ��e—m‘m EÓ*Õ*Õ+Ð+r%   c                ó  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R\        R	\        R
\         P                  /	# )r   r   Úmodulation_scaler  r¨  r    Nr¡   r©  rª  r   )r   r    r!   r[   r�   )r"   s   "r#   r$   r$   H  sƒ   € ÷ ñ Ü‡|�|ðä—l‘lðô �L‰Lðô —,‘,ð	ô
 �,‰,˜Õ
ðô �{‰{ðô ðô ðô ‡\�\ñr%   c           	     ó0   € \        \        W4      W#WEWg4      # )z?Portable batch-one modulation reference for an INT8 projection.)r¿  rÃ  )r   rÅ  r  r¨  r    r¡   r©  rª  s   &&&&&&&&r#   Úint8_linear_modulatedrÇ  H  s!   € ô Ü˜QÓ1Ø˜dØóð r%   c                óJ  € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\         P                  R
\        R\
        R\         P                  /# )r   r   Únorm_weightÚnorm_epsrÅ  r  r¨  r    Nr¡   r©  rª  r   )r   r    rZ   r!   r[   r�   )r"   s   "r#   r$   r$   Z  s›   € ÷ ñ Ü‡|�|ðä—‘ðô ðô —l‘lð	ô
 �L‰Lðô —,‘,ðô �,‰,˜Õ
ðô �{‰{ðô ðô ðô ‡\�\ñr%   c
           
     ó–   € \         P                  P                  P                  W P                  R,          3W4      p
\        W£WEWgW‰4      # )z5Portable RMSNorm plus batch-one modulation reference.rc   )r   re   rf   Úrms_normrd   rÇ  )r   rÉ  rÊ  rÅ  r  r¨  r    r¡   r©  rª  Ú
normalizeds   &&&&&&&&&& r#   Úint8_linear_rms_modulatedrÎ  Z  sF   € ô —‘×$Ñ$×-Ñ-Ø	�G‰G�B�Kˆ>˜;ó€Jô !Ø f¸DØóð r%   z'comfy_kitchen::quantize_int8_tensorwisec                óŽ   € V ^8„  d   QhR\         P                  R\        \         P                  \         P                  3,          /# ©r   r   r   ©r   r    r\   )r"   s   "r#   r$   r$   v  ó2   € ÷ ñ Ü‡|�|ðä
Œ5�<‰<œŸ™Ð%Õ&ñr%   c                 óJ   € R V /p\         P                  ! RVR7      pV! R/ VB # )r   rr  rØ   rÚ   rï   ©r   rÙ   rà   s   &  r#   Ú_op_quantize_int8_tensorwiserÕ  u  s,   € ð �1ˆX€FÜ×&Ò&Ð'AÈ&ÔQ€DÙ‰>�&‰>Ðr%   c                 ó¶   € \         P                  ! V \         P                  R 7      p\         P                  ! R\         P                  V P
                  R7      pW3# )r6   rò   rÚ   )r   rä   rM  ró   rj   r‹   ©r   rq  r   s   &  r#   Ú!_op_quantize_int8_tensorwise_fakerØ  ~  s9   € ä×Ò˜¤%§*¡*Ô-€AÜ�KŠK˜¤%§-¡-¸¿¹ÔA€EØˆ8€Or%   z$comfy_kitchen::quantize_int8_rowwisec                óŽ   € V ^8„  d   QhR\         P                  R\        \         P                  \         P                  3,          /# rÐ  rÑ  )r"   s   "r#   r$   r$   †  rÒ  r%   c                 óJ   € R V /p\         P                  ! RVR7      pV! R/ VB # )r   rw  rØ   rÚ   rï   rÔ  s   &  r#   Ú_op_quantize_int8_rowwiserÛ  …  s,   € ð �1ˆX€FÜ×&Ò&Ð'>ÀvÔN€DÙ‰>�&‰>Ðr%   c                 óØ   € \         P                  ! V \         P                  R 7      p\         P                  ! . V P                  RR O^N5R\         P
                  RV P                  / pW3# ©r6   Nr!   r‹   rc   ©r   rä   rM  ró   rd   rj   r‹   r×  s   &  r#   Ú_op_quantize_int8_rowwise_fakerß  Ž  sQ   € ä×Ò˜¤%§*¡*Ô-€AÜ�KŠKÐO˜Ÿ™  "˜ÐO qÒO´·±ÐOÀaÇhÁhÑO€EØˆ8€Or%   z+comfy_kitchen::quantize_int8_convrot_weightc                óš   € V ^8„  d   QhR\         P                  R\        R\        \         P                  \         P                  3,          /# r„  rt  )r"   s   "r#   r$   r$   –  s<   € ÷ ñ Ü�L‰Lðäðô Œ5�<‰<œŸ™Ð%Õ&ñr%   c                 óN   € R V RV/p\         P                  ! RVR7      pV! R/ VB # )r  rz  r‚  rØ   rÚ   rï   )r  rz  rÙ   rà   s   &&  r#   Ú _op_quantize_int8_convrot_weightrâ  •  s1   € ð
 ˜ ¨jÐ9€FÜ×&Ò&Ð'EÈfÔU€DÙ‰>�&‰>Ðr%   c                 óØ   € \         P                  ! V \         P                  R 7      p\         P                  ! . V P                  RR O^N5R\         P
                  RV P                  / pW#3# rÝ  rÞ  )r  rz  rq  r   s   &&  r#   Ú%_op_quantize_int8_convrot_weight_fakerä  Ÿ  sR   € ä×Ò˜¤u§z¡zÔ2€AÜ�KŠKÐY˜Ÿ™ c rÐ*ÐY¨AÒY´U·]±]ÐYÈ6Ï=É=ÑY€EØˆ8€Or%   z-comfy_kitchen::dequantize_int8_convrot_weightc                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# rˆ  rÕ   )r"   s   "r#   r$   r$   §  s<   € ÷ ñ Ü‡|�|ðä�<‰<ðô ðô ‡\�\ñ	r%   c                 óR   € R V RVRV/p\         P                  ! RVR7      pV! R/ VB # )rq  r   rz  r‹  rØ   rÚ   rï   )rq  r   rz  rÙ   rà   s   &&&  r#   Ú"_op_dequantize_int8_convrot_weightrç  ¦  s6   € ð �1�g˜u l°JÐ?€FÜ×&Ò&Ð'GÐPVÔW€DÙ‰>�&‰>Ðr%   c                 óN   € \         P                  ! V \         P                  R 7      # r5   ©r   rä   rj   )rq  r   rz  s   &&&r#   Ú'_op_dequantize_int8_convrot_weight_fakerê  ±  ó   € ä×Ò˜A¤U§]¡]Ô3Ð3r%   z3comfy_kitchen::dequantize_int8_convrot_weight_dtypec          
      ó�   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\         P                  /# r�  rÕ   )r"   s   "r#   r$   r$   ·  sF   € ÷ ñ Ü‡|�|ðä�<‰<ðô ðô ð	ô
 ‡\�\ñr%   c                 óV   € R V RVRVRV/p\         P                  ! RVR7      pV! R/ VB # )rq  r   rz  rÔ   r�  rØ   rÚ   rï   )rq  r   rz  rÔ   rÙ   rà   s   &&&&  r#   Ú(_op_dequantize_int8_convrot_weight_dtyperî  ¶  s<   € ð �1�g˜u l°JÐ@SÐUfÐg€FÜ×&Ò&Ð'MÐV\Ô]€DÙ‰>�&‰>Ðr%   c                 óH   € \         P                  ! V \        V,          R 7      # r5   ©r   rä   rÜ   r�  s   &&&&r#   Ú-_op_dequantize_int8_convrot_weight_dtype_fakerñ  Â  ó   € ä×Ò˜AÔ%8Ð9JÕ%KÔLÐLr%   z(comfy_kitchen::dequantize_int8_embeddingc                ó°   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R\         P                  /# r–  rÕ   )r"   s   "r#   r$   r$   È  sT   € ÷ ñ Ü‡|�|ðä�<‰<ðô �\‰\ðô ð	ô
 ðô ‡\�\ñr%   c           
      óZ   € R V RVRVRVRV/p\         P                  ! RVR7      pV! R/ VB # )rq  r   r—  rz  rÔ   r™  rØ   rÚ   rï   )rq  r   r—  rz  rÔ   rÙ   rà   s   &&&&&  r#   Ú_op_dequantize_int8_embeddingrõ  Ç  sF   € ð 	ˆQØ�Ø�7Ø�jØÐ.ð€Fô ×&Ò&Ð'BÈ6ÔR€DÙ‰>�&‰>Ðr%   c                 óž   € \         P                  ! . VP                  OV P                  R,          N5\        V,          V P                  R7      # )r¤   rò   rc   )r   ró   rd   rÜ   r‹   )rq  r   r—  rz  rÔ   s   &&&&&r#   Ú"_op_dequantize_int8_embedding_faker÷  Ú  s<   € ä�;Š;Ø%ˆ'�-‰-Ð%˜Ÿ™ �Ñ%Ü!Ð"3Õ4Ø�x‰xôð r%   z%comfy_kitchen::dequantize_int8_simplec                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# r’  r1  )r"   s   "r#   r$   r$   ä  s2   € ÷ ñ Ü‡|�|ðä�<‰<ðô ‡\�\ñr%   c                 óN   € R V RV/p\         P                  ! RVR7      pV! R/ VB # )rq  r   rŠ  rØ   rÚ   rï   )rq  r   rÙ   rà   s   &&  r#   Ú_op_dequantize_int8_simplerú  ã  s1   € ð
 �1�g˜uÐ%€FÜ×&Ò&Ð'?ÈÔO€DÙ‰>�&‰>Ðr%   c                 óN   € \         P                  ! V \         P                  R 7      # r5   ré  r”  s   &&r#   Ú_op_dequantize_int8_simple_fakerü  í  rë  r%   z+comfy_kitchen::dequantize_int8_simple_dtypec                ó„   € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  /# r›  rÕ   )r"   s   "r#   r$   r$   ó  s<   € ÷ ñ Ü‡|�|ðä�<‰<ðô ðô ‡\�\ñ	r%   c                 óR   € R V RVRV/p\         P                  ! RVR7      pV! R/ VB # )rq  r   rÔ   rž  rØ   rÚ   rï   )rq  r   rÔ   rÙ   rà   s   &&&  r#   Ú _op_dequantize_int8_simple_dtyperÿ  ò  s7   € ð �1�g˜uÐ&9Ð;LÐM€FÜ×&Ò&Ð'EÈfÔU€DÙ‰>�&‰>Ðr%   c                 óH   € \         P                  ! V \        V,          R 7      # r5   rð  r�  s   &&&r#   Ú%_op_dequantize_int8_simple_dtype_faker  ý  rò  r%   zcomfy_kitchen::int8_linearc                óš  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R\        R	\        R,          R
\         P                  R,          R\
        R\         P                  R,          R\         P                  R,          R\         P                  /# )r   r   r  r¨  r    NrÔ   r©  rª  r«  r¬  r­  r   r¡  r   )r   r    r�   r[   rj  rZ   )r"   s   "r#   r$   r$     sÂ   € ÷ ñ Ü‡|�|ðä�L‰Lðô —,‘,ðô �,‰,˜Õ
ð	ô
 ðô ðô ðô �T�zðô —l‘l TÕ)ðô ðô �l‰l˜TÕ!ðô —L‘L 4Õ'ðô ‡\�\ñr%   c                 ó�   € \         V,          pR V RVRVRVRVRVRVRVRVR	V	R
V
RV/p\        P                  ! RVR7      pV! R/ VB # )r   r  r¨  r    r¡   r©  rª  r«  r¬  r­  r   r¡  r¿  rØ   rÚ   rÛ   )r   r  r¨  r    rÔ   r©  rª  r«  r¬  r­  r   r¡  r¡   rÙ   rà   s   &&&&&&&&&&&&   r#   Ú_op_int8_linearr    st   € ô $Ð$5Õ6€IàˆQØ�&Ø˜Ø�Ø�YØ�7ØÐ.Ø�YØÐ,Ø˜Ø�HØ˜.ð€Fô ×&Ò& }¸VÔD€DÙ‰>�&‰>Ðr%   c                 ó¦   € \         V,          p\        P                  ! . V P                  R R OVP                  ^ ,          N5RVRV P                  / # )Nr!   r‹   rc   )rÜ   r   ró   rd   r‹   )r   r  r¨  r    rÔ   r©  rª  r«  r¬  r­  r   r¡  r¡   s   &&&&&&&&&&&& r#   Ú_op_int8_linear_faker  $  sF   € ô
 $Ð$5Õ6€IÜ�;Š;ÐX˜Ÿ™  ˜ÐX f§l¡l°1¥oÒX¸YÐXÈqÏxÉxÑXÐXr%   c                ób   € V ^8„  d   Qh/ ^ \         9   d   \        \        \        3,          ;R&   # )r   r  )Ú__conditional_annotations__Údictr�   r[   )r"   s   "r#   r$   r$   	   s"   € ‡�÷Z +Ò *”dœ3¤˜9•oÑ *ò[ r%   )rR   FT)rR   g      à?r)   g      ø?r:   g      @g      @g      @g       €g      à¿g      ð¿g      ø¿g       Àg      Àg      Àg      À)NNN)F)NNrÚ   )r   )Nr   )Fr¦  NNrR   NN)rr  r   Ú#comfy_kitchen.backends._activationsr   r¯  r   r¤  Úcomfy_kitchen.float_utilsr   r   r   r   r	   r
   r   r   r   r   r   Úcomfy_kitchen.registryr   Úcomfy_kitchen.scaled_mm_v2r   r   r   Úcomfy_kitchen.tensor.int8_utilsr   r   r   rj   Úfloat16Úbfloat16r+   r,   rÜ   ÚitemsÚDTYPE_TO_CODEr2   r8   rD   rQ   r�   r$  rl   rŒ   r‰   r˜   r²   r¹   rÇ   rÌ   rÐ   ÚlibraryÚ	custom_oprá   Úregister_fakeræ   rê   rì   rð   r÷   rú   rþ   r  r  r	  r  r  r  r  r  r  r"  r(  r.  rJ  rN  rX  r^  rh  rr  rw  r}  r‚  r†  r‹  r�  rŠ  r™  rž  r¥  r¿  rÃ  rÇ  rÎ  rÕ  rØ  rÛ  rß  râ  rä  rç  rê  rî  rñ  rõ  r÷  rú  rü  rÿ  r  r  r  r$   )rC  Úvr  s   00@r#   Ú<module>r     sª  øð÷ €å SÝ Q÷÷ ÷ ñ õ ,ß MÑ Mß _Ñ _ð €u‡}�}Ø€u‡}�}Ø€u‡~�~Ø€u×ÑØ€u×ÑðÐ ð #6×";Ñ";Ô"=Ô>Ñ"=™$˜!�’Ñ"=Ò>€ð FK×EXÑEX÷ ð" FKÇ^Á^÷ò8ð  %×2Ñ2÷% ÷P-#ð` �<Š<ò ó ÷ �9ˆQƒ<ð 	ð
 €ð  %Ÿ~™~Ø÷0÷fð@ Ð ÷C?ðR  %Ÿ~™~÷%@÷P#ðX ‡�×ÑÐ6ÀRÐÓHôó Iðð* ×Ññ3ó  ð3ð
 ‡�×ÑÐ8ÀrÐÓJôó Kðð* ×!Ñ!ñ3ó "ð3ð
 ‡�×ÑÐ8ÀrÐÓJôó Kðð2 ×!Ñ!ñó "ðð$ ‡�×ÑÐ:ÈÐÓLôó Mðð4 ×#Ñ#ñVó $ðVð ‡�×ÑÐ9ÈÐÓKô%ó Lð%ðP ×"Ñ"ñAó #ðAð ‡�×ÑÐ8ÀrÐÓJôó Kðð( ×!Ñ!ñó "ðð( ‡�×ÑÐ:ÈÐÓLôó Mðð* ×#Ñ#ñ4ó $ð4ð
 ‡�×ÑÐ9ÈÐÓKôó LððB ×"Ñ"ñAó #ðAð )+Ð Ó *õ	õWõ
>õ!õH)õ$	õM÷@÷÷8÷*Q÷(Võ1õKõkõ
õ8õ2W÷

:ð" !%Ø"Ÿ^™^ØØ Ø Ø,0ØØ$(Ø*.÷`=õF	,ð" !%Ø"Ÿ^™^ØØ ÷ð2 !%Ø"Ÿ^™^ØØ ÷ð6 ‡�×ÑÐBÐQSÐÓTôó Uðð ×+Ñ+ñó ,ðð ‡�×ÑÐ?ÈbÐÓQôó Rðð ×(Ñ(ñó )ðð ‡�×ÑÐFÐUWÐÓXôó Yðð "×/Ñ/ñó 0ðð ‡�×ÑÐHÐWYÐÓZôó [ðð $×1Ñ1ñ4ó 2ð4ð ‡�×ÑÐNÐ]_ÐÓ`ôó aðð *×7Ñ7ñMó 8ðMð ‡�×ÑÐCÐRTÐÓUôó Vðð$ ×,Ñ,ñó -ðð ‡�×ÑÐ@ÈrÐÓRôó Sðð ×)Ñ)ñ4ó *ð4ð ‡�×ÑÐFÐUWÐÓXôó Yðð "×/Ñ/ñMó 0ðMð ‡�×ÑÐ5ÀBÐÓGöó HððB ×ÑóYó òYùóu' ?s   ÂW&