+
    �åûj³  ã                  ó:  € R t ^ RIHt ^ RIt^ RIt^RIHtHtHtH	t	 / t
R]P                  3R R lltR R ltR	 R
 ltR R ltR R ltR!R R lltR!R R lltR!R R lltR R ltR]^ 3R R lltR]]P                  3R R lltR R ltRR]R3R R  lltR# )"zEager ConvRot W4A4 kernels.)ÚannotationsN)Ú_INT4_GROUP_SIZEÚ	_INT4_MAXÚ_pack_int4_row_majorÚ_unpack_int4_row_majorÚcpuc               ó(   € V ^8„  d   QhRRRRRRRR/# )	é   ÚsizeÚintÚdevicezstr | torch.deviceÚdtypeútorch.dtypeÚreturnútorch.Tensor© )Úformats   "ÚeC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\backends\eager\convrot_w4a4.pyÚ__annotate__r      s0   € ÷ ñ Ø
ðàðð ðð ñ	ó    c                óº  € V \        V4      V3pV\        9   d   \        V,          # V ^8  g8   W ^,
          ,          ^ 8w  g$   \        P                  ! V ^4      ^,          ^ 8w  d   \	        RV  24      h\
        P                  ! . RO. RO. RO. RO.VVR7      pTp^pW`8  d"   \
        P                  ! WT4      pV^,          pK'  WPR,          ,          pV\        V&   V# )é   z0Regular Hadamard size must be a power of 4, got ©r   r   g      à?)é   r   r   éÿÿÿÿ)r   r   r   r   )r   r   r   r   )r   r   r   r   )ÚstrÚ_HADAMARD_CACHEÚmathÚlogÚ
ValueErrorÚtorchÚtensorÚkron)r
   r   r   Ú	cache_keyÚh4ÚhÚcurrent_sizeÚh_normalizeds   &&&     r   Ú_build_hadamardr(      sË   € ð
 ”s˜6“{ EÐ*€IØ”OÔ#Ü˜yÕ)Ð)àˆa„x�D 1�HÕ%¨!Ô+¬t¯xªx¸¸aÓ/@À1Õ/DÈÔ/IÜÐKÈDÈ6ÐRÓSÐSä	�ŠÚ	š¢}²mÐDØØô
€Bð 	€AØ€LØ
Ô
Ü�JŠJ�qÓˆØ˜ÕŠà˜c�	•?€LØ!-„O�IÑØÐr   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r	   Úweightr   r%   Ú
group_sizer   r   r   )r   s   "r   r   r   ,   s0   € ÷ /ñ /Øð/àð/ð ð/ð ñ	/r   c                ó>  € V P                   w  r4WB,          ^ 8w  d   \        RV RV 24      hWB,          pV P                  W5V4      pVP                  P	                  V P
                  V P                  R7      p\        P                  ! Wg4      pVP                  W44      # )é    úin_features ú not divisible by group_size r   )	Úshaper   ÚreshapeÚTÚtor   r   r    Úmatmul)	r*   r%   r+   Úout_fÚin_fÚn_groupsÚweight_groupedÚh_tÚweight_rotateds	   &&&      r   Ú_rotate_weightr;   ,   sˆ   € ð
 —,‘,�K€EØÕ˜AÔÜ˜<¨ vÐ-JÈ:È,ÐWÓXÐXØÕ!€Hà—^‘^ E°ZÓ@€NØ
�#‰#�&‰&�v—|‘|¨F¯M©Mˆ&Ó
:€CÜ—\’\ .Ó6€NØ×!Ñ! %Ó.Ð.r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r	   Úxr   r%   r+   r   r   r   )r   s   "r   r   r   <   s0   € ÷ )ñ )Øð)àð)ð ð)ð ñ	)r   c                ó8  € V P                   pVR,          pWB,          ^ 8w  d   \        RV RV 24      hWB,          pV P                  RWR4      pVP                  V P                  V P
                  R7      p\        P                  ! Wa4      pVP                  V4      # )r   z	features r/   r   r   )r0   r   r1   r3   r   r   r    r4   )r=   r%   r+   Ú
orig_shapeÚfeaturesr7   Ú	x_groupedÚ	x_rotateds   &&&     r   Ú_rotate_activationrC   <   s‹   € ð
 —‘€JØ˜"�~€HØÕ Ô!Ü˜9 X JÐ.KÈJÈ<ÐXÓYÐYØÕ%€Hà—	‘	˜"˜hÓ3€IØ	�‰�1—7‘7 1§8¡8ˆÓ,€AÜ—’˜YÓ*€IØ×Ñ˜ZÓ(Ð(r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r	   r!   r   Úconvrot_groupsizer   Úquant_group_sizer   ÚNoner   )r   s   "r   r   r   M   s0   € ÷ bñ b ð bÀð bÐX[ð bÐ`dñ br   c                ó  € V P                  4       ^8w  d"   \        R\        V P                  4       24      hV P                  R,          pW1,          ^ 8w  d   \        RV RV 24      hW2,          ^ 8w  d   \        RV RV 24      hR# )r	   z,ConvRot W4A4 expects a 2D tensor, got shape r.   ú$ not divisible by convrot_groupsize z# not divisible by quant_group_size Nr   )Údimr   Útupler0   )r!   rE   rF   Úks   &&& r   Úvalidate_w4a4_shaperM   M   sŒ   € Ø‡z�zƒ|�qÔÜÐGÌÈfÏlÉlÓH[ÐG\Ð]Ó^Ð^Ø�‰�RÕ€AØÕ Ô!Ü˜<¨ sÐ*NÐO`ÐNaÐbÓcÐcØÕ˜qÔ Ü˜<¨ sÐ*MÐN^ÐM_Ð`ÓaÐañ !r   c               ó$   € V ^8„  d   QhRRRRRR/# )r	   r=   r   Úseedr   r   r   )r   s   "r   r   r   W   s!   € ÷ 	ñ 	˜Lð 	°ð 	¸ñ 	r   c                óì   € \         P                  ! V P                  R 7      pVP                  V4       \         P                  ! V P
                  V P                  V P                  V P                  VR7      # ))r   )r   Úlayoutr   Ú	generator)r    Ú	Generatorr   Úmanual_seedÚrandr0   r   rQ   )r=   rO   rR   s   && r   Ú_int4_stochastic_rngrV   W   sR   € Ü—’ q§x¡xÔ0€IØ×Ñ˜$ÔÜ�:Š:Ø	�‰Ø�g‰gØ�x‰xØ�x‰xØôð r   c               ó$   € V ^8„  d   QhRRRRRR/# )r	   Úscaledr   Ústochastic_roundingú
int | Noner   r   )r   s   "r   r   r   c   s&   € ÷ Hñ H˜ð H¸:ð HÈlñ Hr   c                óh  € Vej   V^ 8”  dc   \        W4      pV P                  V4       V P                  4       P                  \        ) \        4      P                  \        P                  4      # V P                  4       P                  \        ) \        4      P                  \        P                  4      # ©N)	rV   Úadd_Úfloor_Úclamp_r   r3   r    Úint8Úround_)rX   rY   Úrngs   && r   Ú_round_int4rc   c   sz   € ØÒ&Ð+>ÀÔ+BÜ" 6Ó?ˆØ�‰�CÔØ�}‰}‹×%Ñ%¤y j´)Ó<×?Ñ?ÄÇ
Á
ÓKÐKØ�=‰=‹?×!Ñ!¤9 *¬iÓ8×;Ñ;¼E¿J¹JÓGÐGr   c               ó$   € V ^8„  d   QhRRRRRR/# ©r	   r=   r   rY   rZ   r   ú!tuple[torch.Tensor, torch.Tensor]r   )r   s   "r   r   r   k   s&   € ÷ 	5ñ 	5Øð	5à#ð	5ð 'ñ	5r   c                ó  € V P                   w  r#V P                  4       P                  RRR7      P                  RR7      pV\        ,          p\        W,          VR7      pWeP                  V4      P                  \        P                  4      3# )zBReturn signed INT4 codes in INT8 storage, without packing nibbles.T)rJ   Úkeepdimg»½×Ùß|Û=)Úmin©rY   r   )
r0   ÚabsÚamaxÚclampr   rc   r1   r3   r    Úfloat32)r=   rY   ÚrowsÚ_ÚabsmaxÚscalesÚqs   &&     r   Ú&_quantize_signed_int4_rowwise_unpackedrt   k   so   € ð
 �g‰g�G€DØ�U‰U‹W�\‰\˜b¨$ˆ\Ó/×5Ñ5¸%Ð5Ó@€FØ”iÕ€FÜ�A•JÐ4GÔH€AØ�n‰n˜TÓ"×%Ñ%¤e§m¡mÓ4Ð4Ð4r   c               ó$   € V ^8„  d   QhRRRRRR/# re   r   )r   s   "r   r   r   w   s&   € ÷ +ñ +Øð+à#ð+ð 'ñ+r   c                ó6   € \        W4      w  r#\        V4      V3# r\   )rt   r   )r=   rY   rs   rr   s   &&  r   Úquantize_signed_int4_rowwiserw   w   s    € ô 7°qÓN�I€AÜ Ó" FÐ*Ð*r   c               ó    € V ^8„  d   QhRRRR/# )r	   r*   r   r   r   )r   s   "r   r   r      s   € ÷ 9ñ 9°ð 9Àñ 9r   c                ó’   € V P                  4       ^8w  d   \        R4      h\        V 4      P                  \        P
                  4      # )z;Unpack a packed signed INT4 weight matrix into INT8 values.z1prepared INT4 fallback weight expects a 2D tensor)rJ   r   r   r3   r    r`   )r*   s   &r   Ú#prepare_int4_weight_for_int8_linearrz      s6   € à‡z�zƒ|�qÔÜÐLÓMÐMÜ! &Ó)×,Ñ,¬U¯Z©ZÓ8Ð8r   é   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
r	   r*   r   rE   r   rF   rY   rZ   r   rf   r   )r   s   "r   r   r   †   sA   € ÷ ]ñ ]Øð]àð]ð ð]ð $ð	]ð
 'ñ]r   c                óÆ   € V\         8w  d   \        R\          24      h\        WV4       \        WP                  V P
                  R7      p\        WV4      p\        WSR7      # )z=Rotate a weight matrix with ConvRot and pack it as signed W4.ú*int4 MMA kernel requires quant_group_size ©r   r   rj   )r   r   rM   r(   r   r   r;   rw   )r*   rE   rF   rY   r%   Ú
weight_rots   &&&&  r   Úquantize_convrot_w4a4_weightr�   †   sZ   € ð Ô+Ô+ÜÐEÔFVÐEWÐXÓYÐYÜ˜Ð3CÔDÜÐ)·-±-ÀvÇ|Á|ÔT€AÜ Ð+<Ó=€JÜ'¨
Ô\Ð\r   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R/# )
r	   Úqdatar   rr   rE   r   rF   Úoutput_dtyper   r   r   )r   s   "r   r   r   •   sL   € ÷ Pñ PØðPàðPð ðPð ð	Pð
 ðPð ñPr   c                óª  € V\         8w  d   \        R\          24      h\        V 4      P                  \        P
                  4      pWQP                  V P                  \        P
                  R7      P                  R^4      ,          p\        W P                  \        P
                  R7      p\        VP                  4       Wr4      P                  V4      # )zIDequantize packed ConvRot W4A4 weights and rotate back to original basis.r~   r   r   )r   r   r   r3   r    rn   r   r1   r(   r;   Úfloat)rƒ   rr   rE   rF   r„   Úw_intÚw_rotr%   s   &&&&&   r   Údequantize_convrot_w4a4_weightr‰   •   sš   € ð Ô+Ô+ÜÐEÔFVÐEWÐXÓYÐYÜ" 5Ó)×,Ñ,¬U¯]©]Ó;€EØ—I‘I U§\¡\¼¿¹�IÓG×OÑOÐPRÐTUÓVÕV€EÜÐ)·,±,ÄeÇmÁmÔT€AÜ˜%Ÿ+™+›-¨Ó>×AÑAÀ,ÓOÐOr   c               ó4   € V ^8„  d   QhRRRRRRRRRRRR	R
R/# )r	   Úx_qdatar   r*   Úx_scaleÚweight_scaleÚbiasútorch.Tensor | NoneÚ	out_dtyper   r   r   )r   s   "r   r   r   ¥   sN   € ÷ ñ Øðàðð ðð ð	ð
 ðð ðð ñr   c                óØ  € \        V 4      P                  VR 7      p\        V4      P                  VR 7      pWgP                  4       ,          pW‚P                  R^4      P                  VP                  VR7      ,          pWƒP                  ^R4      P                  VP                  VR7      ,          pVe4   W„P                  VP                  VR7      P                  ^R4      ,           pV# ))r   r   r   )r   r3   Útr1   r   )	r‹   r*   rŒ   r�   rŽ   r�   Úx_intr‡   Úouts	   &&&&&&   r   Úint4_linearr•   ¥   sÃ   € ô # 7Ó+×.Ñ.°YÐ.Ó?€EÜ" 6Ó*×-Ñ-°IÐ-Ó>€EØ
—'‘'“)Õ
€CØ
—‘  AÓ&×)Ñ)°·±À9Ð)ÓMÕ
M€CØ
×$Ñ$ Q¨Ó+×.Ñ.°c·j±jÈ	Ð.ÓRÕ
R€CØÒØ—G‘G 3§:¡:°Y�GÓ?×GÑGÈÈ2ÓNÕNˆØ€Jr   Úint4c               ó8   € V ^8„  d   QhRRRRRRRRRRR	RR
RRR/# )r	   r=   r   ÚqweightÚwscalesrŽ   r�   rE   r   rF   Úlinear_dtyper   r   r   )r   s   "r   r   r   ·   sb   € ÷ Kñ KØðKàðKð ðKð ð	Kð
 ðKð ðKð ðKð ñKr   c                ó\  € VR9  d   \        RV: 24      hV\        8w  d   \        R\         24      hV P                  R	,          VP                  R	,          ^,          8w  d;   \        RV P                  R	,           RVP                  R	,          ^,           24      hV P                  R	,          V,          ^ 8w  d#   \        RV P                  R	,           RV 24      hV P                  pV P                  R	VR	,          4      P	                  4       p\        WHP                  VP                  R7      p	\        W‰V4      P	                  4       p
\        V
4      w  r¼\        W±WÂW0P                  4      pVRVP                  ^ ,           P                  ! . VRR	 OVP                  ^ ,          N5!  # )
z=Compute ``x @ W.T + bias`` using the eager ConvRot W4A4 path.z8ConvRot W4A4 linear_dtype must be 'int4' or 'int8', got r~   zInput K=z does not match qweight K=rI   r   N>   r–   r`   r   )r   r   r0   r1   Ú
contiguousr(   r   r   rC   rw   r•   )r=   r˜   r™   rŽ   rE   rF   rš   r?   Úx2dr%   Úx_rotÚqactrŒ   r”   s   &&&&&&&       r   Úconvrot_w4a4_linearr    ·   so  € ð Ð+Ô+ÜÐSÐT`ÑScÐdÓeÐeØÔ+Ô+ÜÐEÔFVÐEWÐXÓYÐYØ‡w�wˆr…{�g—m‘m BÕ'¨!Õ+Ô+Ü˜8 A§G¡G¨B¥K =Ð0JÈ7Ï=É=ÐY[ÕK\Ð_`ÕK`ÐJaÐbÓcÐcØ‡w�wˆr…{Ð&Õ&¨!Ô+Ü˜8 A§G¡G¨B¥K =Ð0TÐUfÐTgÐhÓiÐià—‘€JØ
�)‰)�B˜
 2�Ó
'×
2Ñ
2Ó
4€CÜÐ)·*±*ÀCÇIÁIÔN€AÜ˜sÐ'8Ó9×DÑDÓF€EÜ0°Ó7�M€DÜ
�d W°t¿W¹WÓ
E€CØˆ~�—‘˜1•Ð×&Ò&ÐJ¨
°3°B¨ÐJ¸¿¹ÀqÕ9IÓJÐJr   )r-   )Ú__doc__Ú
__future__r   r   r    Úsvdquantr   r   r   r   r   rn   r(   r;   rC   rM   rV   rc   rt   rw   rz   r�   r‰   r•   r    r   r   r   Ú<module>r¤      sž   ðñ "Ý "ã ã ç _Ó _à€ð
 "'ØŸ™÷õ:/õ )õ"bõ	÷H÷	5÷+õ9ð !Ø,Ø&'÷	]ð$ !Ø,Ø %§¡÷Põ ð, !%Ø Ø,Ø÷Kñ Kr   