+
    �åûj�u  ã                  óR  € R t ^ RIHt ^ RIt^ RIHtHt ^ RIHtH	t	H
t
Ht ^RIHtHt R4tR5t^t^tRt^~tR R ltR	 R
 ltR R ltR R ltR R ltR R ltR6R R lltR R ltR7R R lltR8R R llt^R]P>                  RR^ ^R3R R llt R R lt!R9t"R9t#R  R! lt$R:R" R# llt%^R$R]P>                  RR^ ^R3	R% R& llt&R' R( lt'R) R* lt(R+ R, lt)RR^R$]PT                  3R- R. llt+RRR^R$]PT                  RRR/RR3R0 R1 llt,RRR^R$]PT                  RRR/RR3R2 R3 llt-R# );a¾  Eager W4A8 / W6A8 weight preparation, dequantization, and linear operations.

Both bit widths share one storage contract: ``qdata`` is an int8 [N, K*bits/8] tensor. At
4 bits a row is K/2 bytes of packed nibbles (even col = low nibble). At 6 bits the row is the
same K/2-byte nibble plane followed by a K/4-byte plane holding each code's top 2 bits (col c
in byte c//4 at bit 2*(c%4)), so every consumer reads the width and infers the bit depth.
)ÚannotationsN)Úapply_input_actÚapply_residual)Ú
_FIXED_LUTÚ_pack_codesÚ_unpack_codesÚunpack_w4a8_mma_weight)Úint8_linearÚrotate_int8_convrot_weightg¸…ëQ¸ž?c               ó$   € V ^8„  d   QhRRRRRR/# )é   ÚkÚintÚ
group_sizeÚreturnÚNone© )Úformats   "ÚbC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\backends\eager\w4a8_int8.pyÚ__annotate__r   (   s!   € ÷ 
ñ 
˜Sð 
¨cð 
°dñ 
ó    c                ón   € V ^ ,          ^ 8w  g   V^8  g   V^,          ^ 8w  d   \        RV  RV 24      hR# )zû6-bit rows are 3K/4 bytes: K % 32 keeps every row and its high plane 8-byte aligned,
and a 16-col decode vector must not span groups. Finer groups buy nothing at 6 bits
(g16 already sits on the int8 rounding floor), so they are not a supported layout.zM6-bit storage needs K divisible by 32 and group_size a multiple of 16, got K=z, group_size=N)Ú
ValueError)r   r   s   &&r   Ú_check_six_bit_layoutr   (   sG   € ð 	ˆ2…v�„{�j 2”o¨°b­¸AÔ)=ÜðØ�C�} Z Lð2ó
ð 	
ñ *>r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Úqdataútorch.TensorÚs_relr   r   r   útuple[int, int, int]r   )r   s   "r   r   r   3   s,   € ÷ ñ ØðØ ,ðØ:=ðàñr   c                ó  € V P                   w  r4VP                   ^,          V,          pV^ 8”  d   V^,          V,          M^ pV^ 8:  g   VR9  g   WV,          V^,          8w  d   \        RV RV R24      hV^8X  d   \        WR4       W5V3# )zœ(n, k, bits) of a packed weight: K from the scale grid, bits from the packed width.
The one place the Python side infers and validates the storage contract.zpacked row width z does not match K=z at 4 or 6 bits©é   é   )Úshaper   r   )r   r   r   ÚnÚcolsr   Úbitss   &&&    r   Ú_w4a8_geometryr'   3   s†   € ð
 �k‰k�G€AØ�‰�A�˜Õ#€AØ !œeˆD�1�H˜Ž?¨€DØˆA„v�˜VÔ# q¥x°4¸!µ8Ô';ÜÐ,¨T¨FÐ2DÀQÀCÀÐWÓXÐXØˆq„yÜ˜aÔ,Ø�ˆ:Ðr   c               ó    € V ^8„  d   QhRRRR/# )r   Ú
normalizedr   r   r   )r   s   "r   r   r   B   s   € ÷ 
%ñ 
%˜lð 
%¨|ñ 
%r   c           	     óÌ  € V P                  4       P                  4       pVP                  4       \        8”  dp   \        P
                  ! VP                  R7      P                  ^ 4      pV\        P                  ! ^ VP                  4       \        3VP                  VR7      ,          pVP                  4       pWP                  4       ,
          VP                  4       R,           ,          P                  ^4      P                  4       R,
          pVP                  4       \        8:  d6   \        P                  ! \         V P                  \        P"                  R7      # \%        V 4      # )zPFrozen LUT unless the group-normalized rotated weight is heavy-tailed, then fit.©Údevice©r,   Ú	generatorg•Ö&è.>g      @©r,   Údtype)ÚdetachÚflattenÚnumelÚ_KURTOSIS_SAMPLEÚtorchÚ	Generatorr,   Úmanual_seedÚrandintÚfloatÚmeanÚstdÚpowÚitemÚ_W4A8_GATE_KURTOSISÚtensorr   Úfloat32Ú_fit_codebook)r)   ÚxÚgenÚexcess_kurtosiss   &   r   Ú_codebook_forrE   B   sé   € à×ÑÓ×#Ñ#Ó%€AØ‡w�wƒyÔ#Ô#Ü�oŠo Q§X¡XÔ.×:Ñ:¸1Ó=ˆØŒe�mŠm˜A˜qŸw™w›yÔ+;Ð*=ÀaÇhÁhÐZ]Ô^Õ_ˆØ	�‰‹	€AØŸF™F›H�¨¯©«°4­Õ8×=Ñ=¸aÓ@×EÑEÓGÈ#ÕM€OØ×ÑÓÔ!4Ô4Ü�|Š|œJ¨z×/@Ñ/@ÌÏÉÔVÐVÜ˜Ó$Ð$r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r)   r   Úcodebookr   r   )r   s   "r   r   r   O   s!   € ÷ :ñ :˜lð :°lð :À|ñ :r   c                ó®  € VP                  4       ^,
          p\        P                  ! WP                  4       4      pV^,
          P	                  ^ V4      pVP	                  ^ V4      pWV,          ,
          P                  4       pWV,          ,
          P                  4       p\        P                  ! Wv8  WT4      P                  \        P                  4      # )a�  Nearest codebook index via binary search on the sorted codebook.

The codebook is monotonic, so the nearest level is one of the two neighbors of the
insertion point -- one searchsorted + a neighbor compare, vs a 16-pass linear scan.
Bit-identical to the scan (ties resolve to the lower index in both) and ~4x faster,
which matters because the ALS refinement calls this several times per requantize.
)	r3   r5   ÚsearchsortedÚ
contiguousÚclampÚabs_ÚwhereÚtoÚint32)r)   rG   ÚlastÚposÚloÚhiÚdloÚdhis   &&      r   Ú_assign_codesrV   O   sž   € ð �>‰>Ó˜aÕ€DÜ
×
Ò
˜X×'<Ñ'<Ó'>Ó
?€CØ
��'�‰˜˜DÓ	!€BØ	�‰�1�dÓ	€BØ �Õ$×
*Ñ
*Ó
,€CØ �Õ$×
*Ñ
*Ó
,€CÜ�;Š;�s‘y "Ó)×,Ñ,¬U¯[©[Ó9Ð9r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   rB   r   Úseedr   r   r   )r   s   "r   r   r   `   s!   € ÷ "ñ "˜ð "¨Sð "°\ñ "r   c                ó  € \         P                  ! V P                  R7      P                  \	        V4      4      p\         P
                  ! V P                  W P                  V P                  R7      p\         P                  ! W,           4      # )zþRound to nearest integer with probability 1-frac, up with probability frac.

floor(x + U[0,1)) is stochastic rounding: unbiased in expectation, so sub-quantum deltas
(e.g. a small LoRA merged into a low-bit weight) survive instead of being rounded away.
r+   ©r.   r,   r0   )	r5   r6   r,   r7   r   Úrandr#   r0   Úfloor)rB   rX   rC   Únoises   &&  r   Ú_stochastic_roundr^   `   sT   € ô �/Š/ §¡Ô
*×
6Ñ
6´s¸4³yÓ
A€CÜ�JŠJ�q—w‘w¨#·h±hÀaÇgÁgÔN€EÜ�;Š;�q•yÓ!Ð!r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   rB   r   Ústochastic_roundingr   r   r   )r   s   "r   r   r   k   s&   € ÷ _ñ _ˆlð _°ð _¸ñ _r   c                óF   € V^ 8”  d   \        W4      # V P                  4       # )é    )r^   Úround)rB   r`   s   &&r   Ú_roundrd   k   s$   € Ø8KÈaÔ8OÔ˜QÓ4Ð^ÐUV×U\ÑU\ÓU^Ð^r   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# )
r   Úweightr   ÚlevelsÚ	s_channelr`   r   Útargetútorch.Tensor | Noner   r   )r   s   "r   r   r   o   sL   € ÷ #Tñ #TØð#Tàð#Tð ð#Tð ð	#Tð
  ð#Tð ñ#Tr   c                óª  € V P                   w  rVpVP                   R,          ^,
          pVP                  WV,          V^,           4      P                  4       p	Vf   WP                  R^^4      ,          pVP                  WV,          V4      P                  4       p
\        P
                  ! Wš4      pV^ 8”  EdA   V^,
          P                  ^ V^,
          4      p\        P                  ! V	^V4      p\        P                  ! V	^V^,           4      P                  V4      p\        P                  ! V
P                  R7      P                  \        V4      4      p\        P                  ! V
P                   WúP                  V
P                  R7      pVP                  V4      P!                  V4       WÊV8„  ,           P                  ^ V4      pVP#                  \        P$                  4      P                  WVV4      # V^,
          P'                  ^ V4      pVP'                  ^ V4      pV
P)                  \        P                  ! V	^V4      4      P+                  4       pV
P)                  \        P                  ! V	^V4      4      P+                  4       p\        P,                  ! VV8  VV4      P#                  \        P$                  4      P                  WVV4      # )a8  Pick a decoded INT8 codebook level for each grouped weight.

Nearest by default; with ``stochastic_rounding`` > 0, round stochastically between the two
bracketing (sorted) codebook levels so a merged LoRA delta below the int4 step is preserved.
``target`` is ``weight / s_channel`` if the caller already has it.
r+   rZ   éÿÿÿÿ)r#   ÚreshaperJ   Úviewr5   rI   Úclamp_ÚgatherÚsub_r6   r,   r7   r   r[   r0   Úmul_Úadd_rN   rO   rK   ÚsubrL   rM   )rf   rg   rh   r`   ri   r$   ÚgroupsÚgsizerP   ÚlvÚtgrQ   rR   Úlevel_loÚwidthrC   ÚthrÚidxrS   rT   rU   s   &&&&&                r   Ú_assign_gridr}   o   s  € ð —|‘|Ñ€AˆuØ�<‰<˜Õ˜aÕ€DØ	�‰˜�
 D¨1¥HÓ	-×	8Ñ	8Ó	:€BØ‚~ØŸ.™.¨¨Q°Ó2Õ2ˆØ	�‰˜�
 EÓ	*×	5Ñ	5Ó	7€BÜ
×
Ò
˜RÓ
$€CØ˜QÕð �A�g×Ñ˜a ¨¥Ó*ˆÜ—<’<  A rÓ*ˆÜ—’˜R  B¨¥FÓ+×0Ñ0°Ó:ˆÜ�oŠo R§Y¡YÔ/×;Ñ;¼CÐ@SÓ<TÓUˆÜ�jŠj˜Ÿ™¨S¿¹È"Ï(É(ÔSˆØ�‰�‹×Ñ˜XÔ&Ø˜#‘X�×&Ñ& q¨$Ó/ˆØ�v‰v”e—k‘kÓ"×*Ñ*¨1°eÓ<Ð<Ø
��'�‰˜˜DÓ	!€BØ	�‰�1�dÓ	€BØ
�&‰&”—’˜b ! RÓ(Ó
)×
.Ñ
.Ó
0€CØ
�&‰&”—’˜b ! RÓ(Ó
)×
.Ñ
.Ó
0€CÜ�;Š;�s˜S‘y " bÓ)×,Ñ,¬U¯[©[Ó9×AÑAÀ!ÈUÓSÐSr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úlevel_tabler   r   r   r   )r   s   "r   r   r   •   s&   € ÷ añ a˜lð a°<ð aÀLñ ar   c                ó¬   € V P                  ^^R4      VP                  4       P                  R4      ,          P                  4       P	                  R^4      # )zRThe int8 values each level of every group decodes to: round(clamp(level * s_rel)).rl   é�ÿÿÿ)rn   r9   Ú	unsqueezeÚround_ro   )r   r   s   &&r   Ú_grid_levelsr„   •   sE   € à×Ñ˜Q  2Ó&¨¯©«×)@Ñ)@ÀÓ)DÕD×LÑLÓN×UÑUÐVZÐ\_Ó`Ð`r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r   r)   r   rg   r   Ú
iterationsÚsample_sizer   r   )r   s   "r   r   r   š   s:   € ÷ !ñ !Øð!àð!ð ð!ð ð	!ð
 ñ!r   c           
     ó  € V P                  4       pVP                  4       V8”  dm   \        P                  ! VP                  R7      P                  ^ 4      p\        P                  ! ^ VP                  4       V3VP                  VR7      pWF,          pVP                  4       p\        P                  ! V\        P                  ! ^ ^WP                  R7      4      p\        V4       F�  pVP                  R4      V,
          P                  4       P                  R4      p	VP                  4       p
\        V4       F7  pW›8H  pVP                  4       '       g   K  WL,          P!                  4       W«&   K9  	  T
pK’  	  VP#                  4       # )zAFit a data-free Lloyd-Max codebook to normalized rotated weights.r+   r-   rl   )r2   r3   r5   r6   r,   r7   r8   r9   ÚquantileÚlinspaceÚranger‚   ÚabsÚargminÚcloneÚanyr:   rJ   )r)   rg   r†   r‡   Úsamplesr.   ÚindicesrG   Ú_ÚassignmentsÚupdatedÚindexÚselecteds   &&&&         r   rA   rA   š   s2  € ð × Ñ Ó"€GØ‡}�}ƒ˜Ô$Ü—O’O¨7¯>©>Ô:×FÑFÀqÓIˆ	Ü—-’-ØØ�M‰M‹OØˆNØ—>‘>Øô
ˆð Õ"ˆØ�m‰m‹o€GÜ�~Š~ØÜ�Š�q˜!˜V¯N©NÔ;ó€Hô �:ÖˆØ×(Ñ(¨Ó,¨xÕ7×<Ñ<Ó>×EÑEÀbÓIˆØ—.‘.Ó"ˆÜ˜6–]ˆEØ"Ñ+ˆHØ�|‰|�~Œ~Ø!(Õ!2×!7Ñ!7Ó!9�“ñ #ð Šñ ð ×ÑÓ Ð r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )	r   rf   r   r   r   Úconvrot_groupsizer&   r   r   r   )r   s   "r   r   r   ¼   s:   € ÷ 
ñ 
Øð
àð
ð ð
ð ð	
ð
 
ñ
r   c           	     óª  € V P                  4       ^8w  d"   \        R\        V P                  4       24      hVR9  d   \        RV 24      hV P                  ^,          pV^8X  d   \	        WA4       V^,          ^ 8w  g>   WA,          ^ 8w  g1   WB,          ^ 8w  g$   V^8  g   ^V,          ^ 8w  d&   V^,          ^ 8w  d   \        RV RV RV R24      hR# R# )	zSValidate a floating weight (and the requested layout) before W4A8/W6A8 preparation.z"W4A8 weight must be 2D, got shape zbits must be 4 or 6, got úK=z% must be divisible by 16, group_size=z, and convrot_groupsize=z=; group_size must be >=4 and divide 16 or be a multiple of 16Nr    )Údimr   Útupler#   r   )rf   r   r˜   r&   r   s   &&&& r   Úvalidate_w4a8_weight_shaper�   ¼   sÝ   € ð ‡z�zƒ|�qÔÜÐ=¼eÀFÇLÁLÓ>QÐ=RÐSÓTÐTØ�6ÔÜÐ4°T°FÐ;Ó<Ð<Ø�‰�Q�€AØˆq„yÜ˜aÔ,à	ˆB��!ŒØ�>˜QÔØÕ  AÔ%Ø˜Œ>Ø��O˜qÔ  Z°"¥_¸Ô%9äØ��Ð8¸¸ð E!Ø!2Ð 3ð 43ð4ó
ð 	
ñ &:Ñ r   Tc               ó@   € V ^8„  d   QhRRRRRRRRR	RR
RRRRRRRRR/
# )r   rf   r   r   r   Ú	symmetricÚboolÚscale_dtypeútorch.dtyperG   Úcodebook_overriderj   r`   r&   Úscale_searchr   úYtuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]r   )r   s   "r   r   r   Ø   sx   € ÷ oñ oØðoàðoð ðoð ð	oð
 ðoð +ðoð ðoð ðoð ðoðñor   c	                ó
  a € V\         P                  \         P                  39  d   \        RV 24      hV P                  p	V P
                  w  r«W±,          pV P                  4       P                  W¬V4      pRpRo V^8X  g   V'       Ed{   V'       Edr   VP                  4       P                  RRR7      pV^8X  ds   \         P                  ! \        ) \        ^,           V P                  \         P                  R7      o V\        ,          P                  RR7      pR pV! VV,          4      pM<VP                  RR7      pVV,          pVe   TM
\        V4      pVo V 3R	 lpV! V4      p\        \         4       Ft  pS V,          pVV,          P#                  RRR
7      VV,          P#                  RRR
7      P                  RR7      ,          P                  RR7      pV! VV,          4      pKv  	  S V,          V,          pRpEM©V'       dž   VP                  4       P                  RRR7      R,          P                  RR7      p\%        VV,          V4      P                  R^4      P'                  \         P(                  4      pV^,           P                  W«4      pVV,          pRpEMVP+                  RRR7      pVP                  RRR7      V,
          R,          P                  RR7      p\%        VV,
          V,          V4      P                  ^ ^4      P'                  \         P(                  4      P                  W«4      pVP                  W¬V4      ^,
          V,          pRV,          V,           P-                  R4      P/                  4       P1                  4       P'                  V	4      pVP                  4       P                  RR7      R,          P                  RR7      pVP-                  R4      VP3                  ^4      ,          P                  4       P1                  4       pV\         P                  8w  d    VP'                  V4      P1                  4       pS e{   V^8H  ;'       d    TpV'       d   \5        VVVS 4      w  ppV'       d   V^ 8”  d   \7        S V4      p\9        VVVV4      pV^8X  d   XP;                  ^4       XP                  W«4      p\=        XV4      pVVVP                  4       P1                  4       VV3# )aá  Quantize an already ConvRot-rotated weight into W4A8 (or W6A8) storage.

``codebook_override`` lets a chunked caller pin the same table across chunks; when None
it is chosen from this (sub)tensor via _codebook_for. ``stochastic_rounding`` > 0 seeds
stochastic rounding of the final level assignment (the LoRA-merge path). ``bits=6`` is
symmetric, uniform and codebook-free; ``scale_search`` (6-bit only) runs the grid-aware
group-scale search, which the requantize path turns off.
z2scale_dtype must be float32 or float8_e4m3fn, got NT©r›   Úkeepdimr/   ç:Œ0âŽyE>©Úminc                ó¤   € V P                  4       P                  \        ) \        4      \        ,           P                  \        P
                  4      # ©N)rƒ   ro   Ú
_W6_LEVELSrN   r5   rO   )r)   s   &r   ÚassignÚ2_quantize_rotated_w4a8_int8_weight.<locals>.assign  s7   € Ø"×)Ñ)Ó+×2Ñ2´J°;Ä
ÓKÌjÕX×\Ñ\Ô]b×]hÑ]hÓiÐir   c                ó   <€ \        V S4      # r­   )rV   )r)   r   s   &€r   r¯   r°     s   ø€ Ü$ Z°Ó=Ð=r   )r¨   g      @g      .@g       @©r›   g     À_@rl   iøÿÿÿ)é   r   )r5   r@   Úfloat8_e4m3fnr   r0   r#   r9   rn   rŒ   ÚamaxÚaranger®   r,   rK   rE   r‹   Ú
_ALS_ITERSÚsumrd   rN   rO   ÚaminÚsqueezeÚtrJ   r‚   Ú_search_group_scaler„   r}   rs   r   )!rf   r   rŸ   r¡   rG   r£   r`   r&   r¤   Úoriginal_dtyper$   r   ru   Úgrouped_weightÚcodebook_tensorrµ   Úgroup_scaler¯   Ú	quantizedr)   r’   Úquantized_levelsÚshifted_weightÚ
correctionÚsignedÚunsignedÚminimumrh   r   Úsearchedrg   Úpackedr   s!   &&&&&&&&&                       @r   Ú"_quantize_rotated_w4a8_int8_weightrÊ   Ø   sf  ø€ ð2 œ5Ÿ=™=¬%×*=Ñ*=Ð>Ô>ÜÐMÈkÈ]Ð[Ó\Ð\à—\‘\€NØ�<‰<�D€AØ�_€FØ—\‘\“^×(Ñ(¨°JÓ?€Nà€OØ€KØˆq„y—Y�Y§8 8ð ×!Ñ!Ó#×(Ñ(¨R¸Ð(Ó>ˆØ�1Œ9ÜŸ,š,Ü�œZ¨!�^°F·M±MÌÏÉôˆKð  ¤*Õ,×3Ñ3¸Ð3Ó=ˆKòjñ ˜~°Õ;Ó<‰IàŸ*™*¨˜*Ó.ˆKØ'¨+Õ5ˆJà%6Ò%BÑ!ÌÐV`ÓHað ð *ˆKõ>ñ ˜zÓ*ˆIÜ”zÖ"ˆAØ*¨9Õ5ÐàÐ"2Õ2×7Ñ7¸ÀDÐ7ÓIØ#Ð&6Õ6×;Ñ;¸BÈÐ;ÓM×SÑSÐX\ÐSÓ]õ^ç‰e˜ˆe‹oð ñ ˜~°Õ;Ó<ŠIñ #ð % YÕ/°+Õ=ˆØŠ
ß	Ø%×)Ñ)Ó+×0Ñ0°RÀÐ0ÓFÈÕL×SÑSÐX\ÐSÓ]ˆÜ˜¨Õ4Ð6IÓJ×PÑPÐQSÐUVÓW×ZÑZÔ[`×[fÑ[fÓgˆØ˜Q•J×$Ñ$ QÓ*ˆØ +Õ-ˆØŠ
à ×%Ñ%¨"°dÐ%Ó;ˆØ&×+Ñ+°¸DÐ+ÓAÀGÕKÈtÕS×ZÑZÐ_cÐZÓdˆä�N WÕ,°Õ;Ð=PÓQß‰U�1�b‹\ß‰R”—‘‹_ß‰T�!‹Zð	 	ð #Ÿ-™-¨°:Ó>ÀÕBÀkÕQˆØ˜KÕ'¨'Õ1×:Ñ:¸2Ó>×@Ñ@ÓB×MÑMÓO×RÑRÐSaÓbˆ
à×#Ñ#Ó%×*Ñ*¨vÐ*Ó6¸Õ>×EÑEÈ$ÐEÓO€IØ× Ñ  Ó$ y×':Ñ':¸1Ó'=Õ=×DÑDÓF×QÑQÓS€EØ”e—m‘mÔ#Ø—‘˜Ó%×0Ñ0Ó2ˆØÒØ˜1‘9×-Ð- ˆßÜ1°.À%ÈÐT_Ó`‰OˆE�8ßÐ.°Ô2Ü! +¨uÓ5ˆFÜ# N°F¸IÐGZÓ[ˆHØ�1Œ9Ø�M‰M˜!ÔØ—=‘= Ó&ˆä˜ 4Ó(€FàØØ�‰Ó×$Ñ$Ó&ØØðð r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )r   r¾   r   r   rh   r   r   z!tuple[torch.Tensor, torch.Tensor]r   )r   s   "r   r   r   J  s:   € ÷ #=ñ #=Ø ð#=àð#=ð ð#=ð ð	#=ð
 'ñ#=r   c                óÜ  a aaa€ S SP                  R^^4      ,          oV VVV3R lpVP                  \        P                  8X  d¼   VP                  \        P                  4      pVP                  4       V^,
          P                  ^R7      P                  \        P                  4      P                  4       V^,           P                  \        R7      P                  \        P                  4      P                  4       .pM8VP                  4       pWw^\        ,
          ,          V^\        ,           ,          .pV! V^ ,          4      w  r‰p
VR,           Fj  pV! V4      w  r¼pWÉ8  p\        P                  ! WìW™R7       \        P                  ! WëWˆR7       \        P                  ! VP                  R4      WÚV
R7       Kl  	  VP                  VP                  4      P                  4       V
3# )zøPer group, the stored-dtype neighbour of the ALS scale whose decoded int8 levels fit
the group best (through-grid squared error). Returns the chosen scales in ``s_rel``'s
dtype and the matching level assignment, so the caller need not assign again.c                óè   <€ \        SV 4      p\        SVSSR 7      p\        P                  ! V^VP	                  4       4      P                  S4      P                  ^4      P                  R4      pWV3# ))ri   rl   )r„   r}   r5   rp   Úlongrq   Úpow_r¸   )Úcandrg   r|   Úerrr¾   r   rh   ri   s   &   €€€€r   ÚscoreÚ"_search_group_scale.<locals>.scoreU  sb   ø€ Ü˜k¨4Ó0ˆÜ˜>¨6°9ÀVÔLˆÜ�lŠl˜6 1 c§h¡h£jÓ1×6Ñ6°vÓ>×CÑCÀAÓF×JÑJÈ2ÓNˆØ˜#ˆ~Ðr   rª   )Úmax:r³   NN)Úoutrl   )rn   r0   r5   r´   Úuint8r9   ro   Ú_E4M3_MAX_FINITE_BITSÚ_SCALE_SEARCH_FP32_STEPrM   r‚   rN   rJ   )r¾   r   rh   r   rÒ   ÚrawÚ
candidatesÚbaseÚ
best_scaleÚbest_errÚbest_idxrÐ   rÑ   r|   Úbetterri   s   f&ff           @r   r¼   r¼   J  sy  û€ ð ˜iŸn™n¨R°°AÓ6Õ6€F÷ð ð ‡{�{”e×)Ñ)Ô)à�j‰jœŸ™Ó%ˆà�K‰K‹MØ�1�W×Ñ ÐÓ#×(Ñ(¬×)<Ñ)<Ó=×CÑCÓEØ�1�W×ÑÔ!6ÐÓ7×<Ñ<¼U×=PÑ=PÓQ×WÑWÓYð
‰
ð �{‰{‹}ˆØ AÔ(?Õ$?Õ@À$È!ÔNeÕJeÕBfÐgˆ
Ù%*¨:°a­=Ó%9Ñ"€J˜(Ø˜2—�ˆÙ˜t›‰ˆ�3Ø‘ˆÜ�Š�F Õ8Ü�Š�F *Õ=Ü�Š�F×$Ñ$ RÓ(¨#¸X×Fñ ð �=‰=˜Ÿ™Ó%×0Ñ0Ó2°HÐ<Ð<r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   rf   r   r   r   r˜   r   r   )r   s   "r   r   r   y  s2   € ÷ %ñ %Øð%ð ð%ð ð	%ð
 ñ%r   c                ó(  € V P                   w  rE\        ^\        \        V^4      ,          4      pWF8”  dL   \        P                  ! ^ V^,
          W`P
                  R7      P                  4       pV P                  ^ V4      pMT pV! VP                  4       V4      p	V	P                  4       P                  VP                   ^ ,          WR,          V4      p
WªP                  4       P                  RRR7      P                  RR7      ,          p\        V4      # )z®Choose one codebook for the whole tensor from a bounded, deterministic row sample that
spans every chunk (evenly strided rows), so the choice never depends on the chunk size.r+   Tr§   r©   rª   rl   )r#   rÔ   Ú_CODEBOOK_SAMPLE_ELEMSr5   rŠ   r,   rÎ   Úindex_selectrJ   r9   rn   rŒ   rµ   rK   rE   )rf   Ú	rotate_fnr   r˜   r$   r   Úmax_rowsr|   ÚsampleÚrotatedÚgroupedr)   s   &&&&        r   Ú_decide_codebookré   y  s×   € ð �<‰<�D€AÜ�1Ô,´°A°q³	Õ9Ó:€HØ„|Ü�nŠn˜Q  A¥ x¿¹ÔF×KÑKÓMˆØ×$Ñ$ Q¨Ó,‰àˆÙ˜×)Ñ)Ó+Ð->Ó?€GØ�m‰m‹o×"Ñ" 6§<¡<°¥?°AµOÀZÓP€GØŸ;™;›=×-Ñ-°"¸dÐ-ÓC×IÑIÈdÐIÓSÕS€JÜ˜Ó$Ð$r   c               óD   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRRRRRR/# )r   rf   r   r   r   r˜   rŸ   r    r¡   r¢   rG   r£   rj   r`   r&   r¤   r   r¥   r   )r   s   "r   r   r   Ž  s…   € ÷ Vñ VØðVð ðVð ð	Vð
 ðVð ðVð ðVð +ðVð ðVð ðVð ðVðñVr   c                óÒ  € V P                   w  r¼\        ^\        \        V^4      ,          4      pRpV	^8X  d   V'       g   \        R4      hRpV'       dD   V'       d<   Ve,   VP	                  V P
                  \        P                  R7      M\        WW#4      pW½8:  d*   \        V! V P                  4       V4      VVVVVVV	V
R7	      # . . . . 3w  ppppRp\        ^ W½4       F™  pV! V VVV,            P                  4       V4      pV^ 8”  d
   VV,           M^ p\        VVVVVVVV	V
R7	      w  pppppVP                  V4       VP                  V4       VP                  V4       Ve   VP                  V4       ?K›  	  \        P                  ! V^ R7      \        P                  ! V^ R7      \        P                  ! V^ R7      V'       d   \        P                  ! V^R7      V3# RV3# )a$  Rotate and pack a weight in row chunks to cap peak memory.

One codebook is chosen for the whole tensor -- from a caller ``codebook_override`` (e.g. a
LoRA requantize reusing the prior table), else from a whole-tensor-spanning sample -- and
shared across every chunk, so the packed result does not depend on the chunk size. Row math
is otherwise independent. ``stochastic_rounding`` > 0 seeds SR of the final assignment
(the LoRA-merge path); the seed is offset by the row start so chunks decorrelate yet stay
deterministic for a fixed chunk size.
Nz6-bit storage is symmetric onlyFr/   )r   rŸ   r¡   rG   r£   r`   r&   r¤   r²   )r#   rÔ   Ú_QUANT_ROW_ELEM_BUDGETr   rN   r,   r5   r@   ré   rÊ   rJ   r‹   ÚappendÚcat)rf   rä   r   r˜   rŸ   r¡   rG   r£   r`   r&   r¤   r$   r   ÚblockÚoverrideÚpacked_partsÚ
srel_partsÚ	sch_partsÚ
corr_partsr¿   Úr0ÚrotÚchunk_srrÉ   r   rh   rÄ   s   &&&&&&&&&&&                r   Ú_quantize_w4a8_chunkedrø   Ž  só  € ð8 �<‰<�D€AÜ�Ô)¬S°°A«YÕ6Ó7€EØ€HØˆq„yßÜÐ>Ó?Ð?Øˆß—Xð !Ò,ð × Ñ ¨¯©¼U¿]¹]Ð ÔKä! &°ZÓSð 	ð 	„zÜ1Ù�f×'Ñ'Ó)Ð+<Ó=Ø!ØØ#ØØ&Ø 3ØØ%ô

ð 
	
ð 79¸"¸bÀ"°nÑ3€L�*˜i¨Ø€OÜ�A�qÖ ˆÙ˜˜r B¨¥JÐ/×:Ñ:Ó<Ð>OÓPˆØ/BÀQÔ/FÐ&¨Ö+ÈAˆä.ØØ%Ø#Ø'Ø!Ø"*Ø$,ØØ)ô
ñ 	>ˆ��y *¨oð 	×Ñ˜FÔ#Ø×Ñ˜%Ô Ø×Ñ˜Ô#ØÒ!Ø×Ñ˜jÔ)Úñ+ !ô0 	�	Š	�, AÔ&Ü�	Š	�* !Ô$Ü�	Š	�) Ô#ß(2Œ�	Š	�* !Ô$Øðð ð 9=Øðð r   é   c               óD   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRRRRRR/# )r   rf   r   r   r   r˜   rŸ   r    r¡   r¢   rG   r¿   rj   r`   r&   r¤   r   r¥   r   )r   s   "r   r   r   ç  sv   € ÷  ñ  Øð àð ð ð ð ð	 ð
 ð ð ð ð )ð ð ð ð ð ð ð ðñ r   c
                óN   € \        WW(4       \        V \        VVVVVVVVV	R7      # )zSRotate and prepare a floating weight for grouped W4A8 (``bits=4``) or W6A8 storage.)	r   r˜   rŸ   r¡   rG   r£   r`   r&   r¤   )r�   rø   r
   )
rf   r   r˜   rŸ   r¡   rG   r¿   r`   r&   r¤   s
   &&&&&&&&&&r   Úquantize_w4a8_int8_weightrü   ç  s=   € ô& ˜vÐ3DÔKÜ!ØÜ"ØØ+ØØØØ)Ø/ØØ!ôð r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )	r   r   r   r   rG   rj   r   r   r   r   )r   s   "r   r   r   
  sA   € ÷ Fñ FØðFàðFð "ðFð ð	Fð
 ñFr   c                ó  € \        WV4      w  rEpWS,          p\        WV4      pVe4   VP                  V P                  \        P
                  R7      V,          p	M.VP                  4       \        ^V^,
          ,          4      ,
          p	V	P                  WGV4      VP                  4       P                  R4      ,          p	V	P                  WE4      P                  4       P                  R^4      P                  \        P                  4      # )zGDecode grouped INT4/INT6 storage to the INT8 grid consumed by the GEMM.r/   rl   r�   )r'   r   rN   r,   r5   r@   r9   rn   r‚   rc   ro   Úint8)
r   r   rG   r   r$   r   r&   ru   rÁ   Úvaluess
   &&&&      r   Ú_dequant_int4_grouped_to_int8r  
  sÄ   € ô   ¨jÓ9�J€Aˆ$Ø�_€FÜ˜e¨Ó-€IØÒØ—‘ E§L¡L¼¿¹�ÓFÀyÕQ‰à—‘Ó"¤U¨1°¸µ­?Ó%;Õ;ˆØ�[‰[˜ JÓ/°%·+±+³-×2IÑ2IÈ"Ó2MÕM€FØ�;‰;�qÓ×"Ñ"Ó$×+Ñ+¨D°#Ó6×9Ñ9¼%¿*¹*ÓEÐEr   c               ó8   € V ^8„  d   QhRRRRRRRRRRRR	R
R	RR/# )r   r   r   r   rh   rG   rj   rÄ   r   r   r˜   r   r   r   )r   s   "r   r   r     sX   € ÷ !ñ !Øð!àð!ð ð!ð "ð	!ð
 $ð!ð ð!ð ð!ð ñ!r   c                óî  € V P                  4       ^8w  g    V P                  \        P                  8w  d   \	        R4      hVP                  4       ^8w  d   \	        R4      hV^8  g   ^V,          ^ 8w  d   V^,          ^ 8w  d   \	        RV R24      h\        WV4      w  rxp	V	^8X  d   Vf   Ve   \	        R4      hV^,          ^ 8w  g   W†,          ^ 8w  d   \	        RV RV 24      hW…,          p
Wz3p\        VP                  4      V8w  d%   \	        RV R	\        VP                  4       24      h\        VP                  4      V38w  d&   \	        R
V3 R	\        VP                  4       24      hVeA   \        VP                  4      W§38w  d&   \	        RW§3 R	\        VP                  4       24      hVe<   \        VP                  4      R8w  d"   \	        R\        VP                  4       24      hWxV	3# )zXValidate packed W4A8/W6A8 tensors shared by dequantize and linear; returns (n, k, bits).z&packed weight must be a 2D int8 tensorz"s_rel must be 2D [N, K/group_size]zgroup_size=z1 must be >=4 and divide 16 or be a multiple of 16zA6-bit storage is uniform and symmetric: no codebook or correctionrš   z/ must be divisible by 16 and convrot_groupsize=zs_rel must have shape z, got zs_channel must have shape zcorrection must have shape z$codebook must have shape (16,), got )é   )r›   r0   r5   rÿ   r   r'   rœ   r#   )r   r   rh   rG   rÄ   r   r˜   r$   r   r&   ru   Úexpected_scale_shapes   &&&&&&&     r   Úvalidate_w4a8_operandsr    sÝ  € ð ‡y�yƒ{�aÔ˜5Ÿ;™;¬%¯*©*Ô4ÜÐAÓBÐBØ‡y�yƒ{�aÔÜÐ=Ó>Ð>Ø�A„~˜"˜z�/¨QÔ.°:Àµ?ÀaÔ3GÜ˜; z lÐ2cÐdÓeÐeÜ ¨jÓ9�J€Aˆ$Øˆq„y�hÒ*¨jÒ.DÜÐ\Ó]Ð]Øˆ2…v�„{�aÕ+¨qÔ0ÜØ��ÐBÐCTÐBUÐVó
ð 	
ð �_€FØ˜;ÐÜˆU�[‰[ÓÐ1Ô1ÜÐ1Ð2FÐ1GÀvÌeÐTY×T_ÑT_ÓN`ÐMaÐbÓcÐcÜˆY�_‰_Ó ! Ô%ÜÐ5°q°d°V¸6Ä%È	ÏÉÓBXÐAYÐZÓ[Ð[ØÒ¤%¨
×(8Ñ(8Ó"9¸f¸[Ô"HÜÐ6¸°{°mÀ6Ì%ÐPZ×P`ÑP`ÓJaÐIbÐcÓdÐdØÒ¤ h§n¡nÓ 5¸Ô >ÜÐ?ÄÀhÇnÁnÓ@UÐ?VÐWÓXÐXØ�ˆ:Ðr   c               ó0   € V ^8„  d   QhRRRRRRRRRR	R
R/# )r   Úint8_weightr   rh   rÄ   rj   r   r   Úoutput_dtyper¢   r   r   )r   s   "r   r   r   @  sD   € ÷ 6ñ 6Øð6àð6ð $ð6ð ð	6ð
 ð6ð ñ6r   c                ój  € V P                   w  rVWc,          pV P                  4       P                  WWV4      pW�P                  4       P                  V^^4      ,          pVe4   W‚P                  4       P	                  R4      P                  4       ,           pVP                  WV4      P                  V4      # )zEApply channel scaling and correction to decoded grouped INT8 weights.rl   )r#   r9   rn   r»   r‚   rN   )	r  rh   rÄ   r   r	  r$   r   ru   Úweight_rotateds	   &&&&&    r   Ú&_dequantize_w4a8_int8_weight_from_int8r  @  s•   € ð ×Ñ�D€AØ�_€FØ ×&Ñ&Ó(×-Ñ-¨a¸ÓD€NØ#§o¡oÓ&7×&<Ñ&<¸QÀÀ1Ó&EÕE€NØÒØ'¯,©,«.×*BÑ*BÀ2Ó*F×*LÑ*LÓ*NÕNˆØ×Ñ˜qÓ$×'Ñ'¨Ó5Ð5r   c               ó<   € V ^8„  d   QhRRRRRRRRRRRR	R
R	RRRR/	# )r   r   r   r   rh   rG   rj   rÄ   r   r   r˜   r	  r¢   r   r   )r   s   "r   r   r   Q  sm   € ÷ !Zñ !ZØð!Zàð!Zð ð!Zð "ð	!Zð
 $ð!Zð ð!Zð ð!Zð ð!Zð ñ!Zr   c           	     ó’   € \        V VVVVVV4       \        V VVV4      p\        VVVVV4      p	\        W–4      P	                  V4      # )z=Decode W4A8 storage into its physical [N, K] floating weight.)r  r  r  r
   rN   )
r   r   rh   rG   rÄ   r   r˜   r	  r  r  s
   &&&&&&&&  r   Údequantize_w4a8_int8_weightr  Q  sl   € ô ØØØØØØØôô 0ØØØØó	€Kô <ØØØØØó€Nô & nÓH×KÑKÈLÓYÐYr   g        c                óX   € V ^8„  d   QhRRRRRRRRRRRRR	RR
RRRRRRRRRRRRRRRRR/# ©r   rB   r   r   r   rh   rG   rj   rÄ   Úbiasr   r   r˜   Ú	out_dtyper¢   Ú	input_actz
str | NoneÚinput_act_weightÚinput_act_epsr9   ÚresidualÚresidual_scaler   r   )r   s   "r   r   r   u  sº   € ÷ Cñ CØðCàðCð ðCð ð	Cð
 "ðCð $ðCð ðCð ðCð ðCð ðCð ðCð *ðCð ðCð "ðCð (ðCð  ñ!Cr   c                óî  € Ve   \        \        WW#WEWgW‰W«V4      VV4      # \        W
W¼4      p \        VVVVVVV4      w  pppV P                  R,          V8w  d#   \        RV P                  R,           RV 24      hVeX   \        VVVVVVVV P                  R7      p\        P                  P                  P                  V VV4      P                  V	4      # \        VVVV4      p\        V VVVV	RVR7      # )z^Compute x @ W.T + bias (then residual + residual_scale * that) using portable W4A8 operations.zInput K=z does not match qdata K=)rG   rÄ   r   r˜   r	  T)r  r  Úconvrotr˜   rl   )r   Úw4a8_int8_linearr   r  r#   r   r  r0   r5   ÚnnÚ
functionalÚlinearrN   r  r	   )rB   r   r   rh   rG   rÄ   r  r   r˜   r  r  r  r  r  r  Ú_nr   Ú_bitsrf   r  s   &&&&&&&&&&&&&&&     r   r  r  u  s1  € ð$ ÒÜÜØ˜%¨HÀ$Ø!¨iØóð
 Øó
ð 	
ô 	˜Ð&6ÓF€AÜ)ØØØØØØØó�L€Bˆˆ5ð 	‡w�wˆr…{�aÔÜ˜8 A§G¡G¨B¥K =Ð0HÈÈÐLÓMÐMØÒÜ,ØØØØØ!Ø!Ø/ØŸ™ô	
ˆô �x‰x×"Ñ"×)Ñ)¨!¨V°TÓ:×=Ñ=¸iÓHÐHä/ØØØØó	€Kô Ø	ØØØØØØ+ôð r   c                óX   € V ^8„  d   QhRRRRRRRRRRRRR	RR
RRRRRRRRRRRRRRRRR/# r  r   )r   s   "r   r   r   »  s¨   € ÷ ñ Øðàðð ðð ð	ð
 "ðð $ðð ðð ðð ðð ðð ðð *ðð ðð "ðð (ðð  ñ!r   c                ó–   € \        W
W¼4      p \        WP                  4       V P                  R,          4      w  r\	        WW#WEWgW‰WÞR7      # )z.Portable fallback for resident packed weights.)r  r  rl   )r   r   r3   r#   r  )rB   r   r   rh   rG   rÄ   r  r   r˜   r  r  r  r  r  r  s   &&&&&&&&&&&&&&&r   Úw4a8_packed_linearr#  »  sI   € ô$ 	˜Ð&6ÓF€AÜ)¨%·±Ó1BÀAÇGÁGÈBÅKÓP�L€EÜØ	�% H¸$Ø¨xôð r   gš™™™™™¹¿i   )rb   N)r  é   ià“ )r!   i  @ )Nrb   r!   T).Ú__doc__Ú
__future__r   r5   Ú#comfy_kitchen.backends._activationsr   r   Ú comfy_kitchen.tensor.w4a8_packedr   r   r   r   Úquantizationr	   r
   r>   r4   r·   r®   rØ   r×   r   r'   rE   rV   r^   rd   r}   r„   rA   r�   r´   rÊ   r¼   rì   râ   ré   rø   rü   r  r  r  Úbfloat16r  r  r#  r   r   r   Ú<module>r+     s}  ðñõ #ã ç Oß kÓ kç Að Ð ØÐ ð €
ð €
ð
 Ð ØÐ õ
õõ
%õ:õ""õ_÷#TõLa÷
!÷D
ð< ØØ$×2Ñ2ØØ-1Ø ØØ÷oõd#=ðR !Ð ð !Ð õ%÷*Vðv Ø ØØ$×2Ñ2ØØ+/Ø ØØ÷ õFFõ$!õH6ð* %)Ø&*ØØ Ø %§¡÷!ZðR %)Ø&*Ø $ØØ Ø"Ÿ^™^Ø Ø,0ØØ$(Ø*.÷CðV %)Ø&*Ø $ØØ Ø"Ÿ^™^Ø Ø,0ØØ$(Ø*.÷ñ r   