+
    ªåûjãK  ã                  ó   € R t ^ RIHt ^ RIt^ RIHt ^ RIt^ RIHt ^RIH	t	H
t
HtHtHt ^RIHtHtHt ^RR]P$                  RR^ ^R3	R	 R
 lltRR^R]P(                  3R R lltRRR^R]P(                  RRRRRR3R R llt ! R R]
4      tR R ltR R ltR R lt]! ]P6                  P8                  P:                  P<                  ]4      R 4       t]! ]P6                  P8                  P@                  P<                  ]4      R 4       t!]! ]P6                  P8                  PD                  P<                  ]4      R 4       t#]! ]P6                  P8                  PH                  P<                  ]4      R 4       t%R# )aB  Grouped W4A8 / W6A8 weights executed through the shared INT8 linear operation.

Weights are ConvRot-rotated and quantized per ``group_size``. The default (``bits=4``)
uses a symmetric Lloyd-Max codebook and FP8 group scales; ``bits=6`` stores uniform
6-bit codes (no codebook) in a 3K/4-byte row, at ~3x lower weight error for 1.5x the
bytes. The bit depth is implied by the packed row width, so one layout class and one
checkpoint format cover both. The tensor layout owns only metadata and operator
routing; eager, CUDA, and Triton backends own the operation implementations.
)ÚannotationsN)Ú	dataclass)Úregistry)ÚBaseLayoutParamsÚQuantizedLayoutÚQuantizedTensorÚdequantize_argsÚregister_layout_op)Ú	_mma_bitsÚdefault_w4a8_codebookÚunpack_w4a8_mma_weighté   Tc               óD   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRRRRRR/# )é   Úweightútorch.TensorÚ
group_sizeÚintÚconvrot_groupsizeÚ	symmetricÚboolÚscale_dtypeútorch.dtypeÚcodebookÚcodebook_tensorútorch.Tensor | NoneÚstochastic_roundingÚbitsÚscale_searchÚreturnúYtuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]© )Úformats   "ÚZC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\tensor\w4a8_int8.pyÚ__annotate__r$   $   sv   € ÷ *ñ *Øð*àð*ð ð*ð ð	*ð
 ð*ð ð*ð )ð*ð ð*ð ð*ð ð*ðñ*ó    c
                óÔ   € V\         P                  \         P                  39  d   \        RV 24      hRV RVRVRVRVRVRVR	VR
VRV	/
p
\        P
                  ! RV
R7      pV! R/ V
B # )a.  Rotate and prepare a floating weight for grouped W4A8 (or W6A8) storage.

``codebook_tensor`` reuses a previously decided table (e.g. on LoRA requantize) so the
codebook decision -- kurtosis probe and any k-means -- is skipped. ``stochastic_rounding``
> 0 seeds stochastic rounding of the level assignment so a merged LoRA below the int4 step
is preserved instead of rounded away. ``bits=6`` selects uniform, codebook-free 6-bit
storage; ``scale_search`` (6-bit only) runs the grid-aware group-scale search at quantize
time, which the requantize path skips.
z2scale_dtype must be float32 or float8_e4m3fn, got r   r   r   r   r   r   r   r   r   r   Úquantize_w4a8_int8_weight©Úkwargsr!   )ÚtorchÚfloat32Úfloat8_e4m3fnÚ
ValueErrorr   Úget_implementation)r   r   r   r   r   r   r   r   r   r   r)   Úimpls   &&&&&&&&&&  r#   r'   r'   $   sŠ   € ð6 œ5Ÿ=™=¬%×*=Ñ*=Ð>Ô>ÜÐMÈkÈ]Ð[Ó\Ð\à�&Ø�jØÐ.Ø�YØ�{Ø�HØ˜?ØÐ2Ø�Ø˜ð€Fô ×&Ò&Ð'BÈ6ÔR€DÙ‰>�&‰>Ðr%   c               ó<   € V ^8„  d   QhRRRRRRRRRRRR	R
R	RRRR/	# )r   Úqdatar   Ús_relÚ	s_channelr   r   Ú
correctionr   r   r   Úoutput_dtyper   r   r!   )r"   s   "r#   r$   r$   Q   sb   € ÷ ñ Øðàðð ðð "ð	ð
 $ðð ðð ðð ðð ñr%   c                óf   € RV RVRVRVRVRVRVRV/p\         P                  ! R	VR
7      p	V	! R/ VB # )z8Dequantize a packed W4A8 weight into its original basis.r1   r2   r3   r   r4   r   r   r5   Údequantize_w4a8_int8_weightr(   r!   ©r   r.   )
r1   r2   r3   r   r4   r   r   r5   r)   r/   s
   &&&&&&&&  r#   r7   r7   Q   sU   € ð 	�Ø�Ø�YØ�HØ�jØ�jØÐ.Ø˜ð	€Fô ×&Ò&Ð'DÈVÔT€DÙ‰>�&‰>Ðr%   Fç        c          "     ó\   € V ^8„  d   QhRRRRRRRRRRRRR	RR
RRRRRRRRRRRRRRRRRRR/# )r   Úxr   r1   r2   r3   r   r   r4   Úbiasr   r   r   Ú	out_dtyper   Ú
mma_packedr   Ú	input_actz
str | NoneÚinput_act_weightÚinput_act_epsÚfloatÚresidualÚresidual_scaler   r!   )r"   s   "r#   r$   r$   j   s²   € ÷ /ñ /Øð/àð/ð ð/ð ð	/ð
 "ð/ð $ð/ð ð/ð ð/ð ð/ð ð/ð ð/ð ð/ð *ð/ð ð/ð "ð/ð  (ð!/ð" ñ#/r%   c                óš   € RV RVRVRVRVRVRVRVR	VR
V	RVRVRVRVRV/pV
'       d   RMRp\         P                  ! VVR7      pV! R/ VB # )a3  Compute ``x @ W.T + bias`` with the selected W4A8 backend.

input_act ("swiglu" or "rms_norm", with input_act_weight/input_act_eps for the
norm) is applied to x on the way into the activation quantizer; the CUDA
decode path folds it into that kernel, every other path applies it eagerly
with identical results. residual/residual_scale make the result
``residual + residual_scale * (x @ W.T + bias)`` (a pre-norm block's addcmul,
with the linear rounded to out_dtype first), fused into the CUDA decode
kernel's epilogue and applied after the GEMM everywhere else.
r;   r1   r2   r3   r   r4   r<   r   r   r=   r?   r@   rA   rC   rD   Úw4a8_packed_linearÚw4a8_int8_linearr(   r!   r8   )r;   r1   r2   r3   r   r4   r<   r   r   r=   r>   r?   r@   rA   rC   rD   r)   Ú	operationr/   s   &&&&&&&&&&&&&&&&   r#   rG   rG   j   sƒ   € ð: 	ˆQØ�Ø�Ø�YØ�HØ�jØ�Ø�jØÐ.Ø�YØ�YØÐ,Ø˜Ø�HØ˜.ð€F÷" )3Ñ$Ð8J€IÜ×&Ò& y¸Ô@€DÙ‰>�&‰>Ðr%   c            
      ó.  € ] tR t^œtRtRtRt]! RR7       ! R R]4      4       t	]
^RR]P                  RR^ ^R3	R	 R
 ll4       t]
R R l4       t]
R R l4       t]
R R l4       t]
RR R ll4       t]
R R l4       t]
R R l4       t]
R R l4       tRtR# )ÚAsymW4A8Int8LayoutzMGrouped W4A8 (or uniform W6A8) weights run through the selected INT8 backend.FT)Úfrozenc                  óŒ   € ] tR t^¢t$ RtR]R&   RtR]R&   RtR]R&   ^tR]R&   Rt	R]R	&   R
t
R]R&   R
tR]R&   R R ltR tRtR# )ÚAsymW4A8Int8Layout.ParamsNr   r3   r4   r   r   r   r   r   Fr   r>   Ú
transposedc               ó   € V ^8„  d   QhRR/# )r   r   z	list[str]r!   )r"   s   "r#   r$   Ú&AsymW4A8Int8Layout.Params.__annotate__®   s   € ÷ 	ñ 	 Iñ 	r%   c                	óŠ   € R R.pV P                   e   VP                  R4       V P                  e   VP                  R4       V# )Úscaler3   r4   r   )r4   Úappendr   )ÚselfÚfieldss   & r#   Ú_tensor_fieldsÚ(AsymW4A8Int8Layout.Params._tensor_fields®   s=   € Ø˜{Ð+ˆFØ�‰Ò*Ø—‘˜lÔ+Ø�}‰}Ò(Ø—‘˜jÔ)ØˆMr%   c           	     	óZ  € \        V P                  4      ^8w  d   \        RV P                   24      hV P                  w  rV P                  '       d   Y!r!V P                  ^8  gm   V^,          ^ 8w  g_   W P                  ,          ^ 8w  gH   W P
                  ,          ^ 8w  g1   ^V P                  ,          ^ 8w  dB   V P                  ^,          ^ 8w  d*   \        RV RV P                   RV P
                   R24      hW P                  ,          pV P                  '       d   RMW3p\        V P                  P                  4      V8w  d/   \        RV R\        V P                  P                  4       24      hV P                  e&   \        V P                  P                  4      V38w  dB   V P                  f   RM\        V P                  P                  4      p\        R	V3 RV 24      hV P                  eU   \        V P                  P                  4      W138w  d0   \        R
W13 R\        V P                  P                  4       24      hV P                  eR   \        V P                  P                  4      R8w  d,   \        R\        V P                  P                  4       24      hR# R# )r   zorig_shape must be 2D, got zK=z% must be divisible by 16, group_size=z, and convrot_groupsize=z=; group_size must be >=4 and divide 16 or be a multiple of 16zscale must have shape z, got Nzs_channel must have shape zcorrection must have shape z$codebook must have shape (16,), got )é    )é   )ÚlenÚ
orig_shaper-   rN   r   r   r>   ÚtuplerR   Úshaper3   r4   r   )rT   ÚnÚkÚgroupsÚexpected_scale_shapeÚactual_shapes   &     r#   Ú_validate_tensor_fieldsÚ1AsymW4A8Int8Layout.Params._validate_tensor_fields¶   sB  € Ü�4—?‘?Ó# qÔ(Ü Ð#>¸t¿¹Ð>OÐ!PÓQÐQØ—?‘?‰DˆAØ��ˆØ�1à—‘ !Ô#Ø�r•6˜Q”;Ø—‘Õ&¨!Ô+Ø×-Ñ-Õ-°Ô2Ø˜Ÿ™Õ(¨AÔ-°$·/±/ÀBÕ2FÈ!Ô2Kä Ø˜˜Ð@ÀÇÁÐ@Qð R)Ø)-×)?Ñ)?Ð(@ð A;ð<óð ð
 Ÿ/™/Õ)ˆFØ+/¯?¯?¨?¡4ÀÀÐ Ü�T—Z‘Z×%Ñ%Ó&Ð*>Ô>Ü Ø,Ð-AÐ,BÀ&ÌÈtÏzÉz×O_ÑO_ÓI`ÐHaÐbóð ð �~‰~Ò%¬¨t¯~©~×/CÑ/CÓ)DÈÈÔ)LØ'+§~¡~Ò'=™tÄ5ÈÏÉ×I]ÑI]ÓC^�Ü Ð#=¸q¸d¸VÀ6È,ÈÐ!XÓYÐYØ�‰Ò*¬u°T·_±_×5JÑ5JÓ/KÐPVÈ{Ô/ZÜ Ø1°6°+°¸fÄUÈ4Ï?É?×K`ÑK`ÓEaÐDbÐcóð ð �}‰}Ò(¬U°4·=±=×3FÑ3FÓ-GÈ5Ô-PÜ Ø:¼5ÀÇÁ×ATÑATÓ;UÐ:VÐWóð ñ .QÑ(r%   r!   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r3   Ú__annotations__r4   r   r   r   r>   rN   rV   rd   Ú__static_attributes__r!   r%   r#   ÚParamsrM   ¢   sX   ‡ ð *.ˆ	Ð&Ó-Ø*.ˆ
Ð'Ó.Ø(,ˆÐ%Ó,Øˆ
�CÓØ!$Ð˜3Ó$à ˆ
�DÓ Ø ˆ
�DÓ õ	ö"	r%   rl   r   Nc               óD   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRRRRRR/# )r   Útensorr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ztuple[torch.Tensor, Params]r!   )r"   s   "r#   r$   ÚAsymW4A8Int8Layout.__annotate__Û   sx   € ÷ $ñ $àð$ð ð$ð ð	$ð
 ð$ð !ð$ð ð$ð -ð$ð !ð$ð ð$ð ð$ð 
%ñ$r%   c                	ó¨   € \        VVVVVVVVV	V
R 7
      w  rÍrïpV P                  VVVVVP                  \        VP                  4      VVR7      pVV3# ))	r   r   r   r   r   r   r   r   r   )rR   r3   r4   r   Ú
orig_dtyper\   r   r   )r'   rl   Údtyper]   r^   )Úclsrn   r   r   r   r   r   r   r   r   r   r)   r1   r2   r3   r4   Úparamss   &&&&&&&&&&&,     r#   ÚquantizeÚAsymW4A8Int8Layout.quantizeÚ   s|   € ô @YØØ!Ø/ØØ#ØØ+Ø 3ØØ%ô@
Ñ<ˆ�i¨_ð —‘ØØØ!Ø$Ø—|‘|Ü˜VŸ\™\Ó*Ø!Ø/ð ó 	
ˆð �fˆ}Ðr%   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r   r   Ústorager   r   r!   )r"   s   "r#   r$   ro     s'   € ÷ *Añ *A ð *AÀð *AÐRañ *Ar%   c                óº  € VP                   VP                  rCVP                  '       g‹   VP                  '       gy   VP                  '       dg   \
        P                  P                  fK   \        P                  ! R4      '       d/   \
        P                  P                  VP                  4      R8  d   V# VP                  w  rVVP                  f&   VP                  ^,          ^,          V^,          8H  pME\
        P                   ! VP                  P#                  4       P%                  4       \'        4       4      pV^,          ^ 8w  g_   VR,          ^ 8w  gQ   VP(                  ^8w  g@   VP*                  f2   VP,                  P.                  \
        P0                  8w  g	   V'       g   V# VP.                  \
        P2                  8w  gC   VP4                  ^8w  g2   VP7                  4       '       d   VP                  VP                  8w  d   \9        R4      hVP;                  4       VP;                  4       ,
          pVP,                  P;                  4       VP;                  4       ,
          p	W”P,                  P<                  ,           p
V^ 8  g(   W˜VP<                  ,           8  g   W¢P<                  8”  d   V# ^ RIH p W(WƒP<                  ,           VP,                  P<                  ,            PC                  VP.                  4      pV! WÄP,                  4       \D        PF                  ! WDP,                  PI                  ^ 4      RR7      p\K        WÁPL                  V4      # )z‘Pack resident codes/scales for small-M MMA. Prefill dequantizes the same
records directly into INT8 chunks; neither path needs canonical weights.Úcudar   zHresident storage must be a contiguous byte buffer on the weight's device)Úpack_w4a8_mma_weight_T)rR   r>   ©é   rY   )'Ú_qdataÚ_paramsr>   rN   Úis_cudar*   ÚversionÚhipr   Úis_availablerz   Úget_device_capabilityÚdevicer\   r   r^   ÚequalrB   Úcpur   r   r4   rR   rr   r,   Úuint8ÚndimÚis_contiguousr-   Údata_ptrÚnbytesÚcomfy_kitchen.backends.cudar{   ÚviewÚdataclassesÚreplaceÚ	new_emptyr   Ú_layout_cls)rs   r   rx   r1   rt   r_   r`   Úcodebook_okÚstartÚscale_startÚendr{   Údests   &&&          r#   Úoptimize_for_gemvÚ$AsymW4A8Int8Layout.optimize_for_gemv  sP  € ð Ÿ™ v§~¡~ˆvà××ÐØ× × Ð Ø—=—=�=Ü�}‰}× Ñ Ò,Ü×(Ò(¨×0Ò0Ü�z‰z×/Ñ/°·±Ó=ÀÔFàˆMØ× Ñ ‰ˆØ�?‰?Ò"àŸ+™+ a�.¨1Õ,°°AµÑ5‰KäŸ+š+ f§o¡o×&;Ñ&;Ó&=×&AÑ&AÓ&CÔEZÓE\Ó]ˆKà��F�aŒK˜1˜s�7 aœ<Ø× Ñ  BÔ&Ø× Ñ Ò,Ø�|‰|×!Ñ!¤U×%8Ñ%8Ô8ßàˆMØ�=‰=œEŸK™KÔ'¨7¯<©<¸1Ô+<ÀG×DYÑDY×D[ÒD[Ð_f×_mÑ_mÐqv×q}Ñq}Ô_}ÜÐgÓhÐhØ—‘Ó  7×#3Ñ#3Ó#5Õ5ˆØ—l‘l×+Ñ+Ó-°×0@Ñ0@Ó0BÕBˆØŸL™L×/Ñ/Õ/ˆØ�1Œ9˜¨e¯l©lÕ&:Ô:¸cÇNÁNÔ>RØˆMåEà˜U§\¡\Õ1°F·L±L×4GÑ4GÕGÐH×MÑMÈeÏkÉkÓZˆÙ˜d§L¡LÔ1Ü×$Ò$ØŸ,™,×0Ñ0°Ó3Àô
ˆô ˜t×%7Ñ%7¸Ó@Ð@r%   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r1   r   rt   rl   r   r!   )r"   s   "r#   r$   ro   /  s!   € ÷ 
ñ 
˜|ð 
°Vð 
Àñ 
r%   c                	ód  € VP                   pVP                  '       dE   VP                  '       d   VP                  R R R1,          MVP                  w  rE\	        WV4      w  r\        VVVP                  VP                  VP                  VP                  VP                  VP                  R7      # )N)r   r4   r   r   r5   éÿÿÿÿ)rR   r>   rN   r\   r   r7   r3   r   r4   r   r   rq   )rs   r1   rt   r2   r_   r`   s   &&&   r#   Ú
dequantizeÚAsymW4A8Int8Layout.dequantize.  s•   € à—‘ˆØ××ÐØ.4×.?×.?Ð.?�6×$Ñ$¡T r TÖ*ÀV×EVÑEV‰DˆAÜ1°%¸AÓ>‰LˆEÜ*ØØØ×ÑØ—_‘_Ø×(Ñ(Ø×(Ñ(Ø$×6Ñ6Ø×*Ñ*ô	
ð 		
r%   c               ó    € V ^8„  d   QhRRRR/# )r   Úqtensorr   r   r    r!   )r"   s   "r#   r$   ro   @  s   € ÷ 
ñ 
à ð
ð
ñ
r%   c                	óŒ   € VP                   pVP                  VP                  VP                  VP                  VP
                  3# )N)r   r~   rR   r3   r4   r   ©rs   r    rt   s   && r#   Úget_plain_tensorsÚ$AsymW4A8Int8Layout.get_plain_tensors?  s>   € ð —‘ˆà�N‰NØ�L‰LØ×ÑØ×ÑØ�O‰Oð
ð 	
r%   c               ó    € V ^8„  d   QhRRRR/# )r   r   r   r;   r   r!   )r"   s   "r#   r$   ro   T  s   € ÷ 
ñ 
 oð 
¸,ñ 
r%   c	                	óð   € VP                   p	V	P                  '       d   \        # V P                  V4      w  r«rÍp\	        W*W¼WíVV	P
                  V	P                  V	P                  V	P                  WEVWxR 7      # ))r   r4   r<   r   r   r=   r>   r?   r@   rA   rC   rD   )	r   rN   ÚNotImplementedr£   rG   r   r   rq   r>   )rs   r   r;   r<   r?   r@   rA   rC   rD   rt   r1   r2   r3   r4   r   s   &&&&&&&&&      r#   Úlinear_input_actÚ#AsymW4A8Int8Layout.linear_input_actS  sx   € ð —‘ˆØ××ÐÜ!Ð!Ø8;×8MÑ8MÈfÓ8UÑ5ˆ�i¨XÜØ�e°ÐW[Ø×(Ñ(¸F×<TÑ<TØ×'Ñ'°F×4EÑ4EØÐR_Øô
ð 	
r%   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r1   r   rt   rl   r   zdict[str, torch.Tensor]r!   )r"   s   "r#   r$   ro   d  s"   € ÷ ñ  |ð ¸Vð ÐH_ñ r%   c                	óì   € VP                   '       d   \        R 4      hRVRVP                  RVP                  /pVP                  e   VP                  VR&   VP
                  e   VP
                  VR&   V# )zCGEMV-optimized weights are resident-only; save the canonical layoutÚ Ú_s_relÚ
_s_channelÚ_correctionÚ	_codebook)r>   r-   rR   r3   r4   r   )rs   r1   rt   Úouts   &&& r#   Ústate_dict_tensorsÚ%AsymW4A8Int8Layout.state_dict_tensorsc  su   € à××ÐÜÐbÓcÐcà�Ø�f—l‘lØ˜&×*Ñ*ð
ˆð
 ×ÑÒ(Ø!'×!2Ñ!2ˆC�ÑØ�?‰?Ò&Ø%Ÿ™ˆC�ÑØˆ
r%   c               ó    € V ^8„  d   QhRRRR/# )r   r    r   r   r   r!   )r"   s   "r#   r$   ro   s  s   € ÷ Rñ R˜?ð R¨sñ Rr%   c                ó  € ^ RI Hp VP                  pVP                  '       d3   VP                  w  rE\        VP                  P                  4       WE4      # V! VP                  VP                  VP                  4      ^,          # )zECode width implied by the packed row: K/2 bytes at 4 bits, 3K/4 at 6.)Ú_w4a8_geometry)
Ú&comfy_kitchen.backends.eager.w4a8_int8r¶   r   r>   r\   r
   r~   ÚnumelrR   r   )rs   r    r¶   rt   r_   r`   s   &&    r#   r   ÚAsymW4A8Int8Layout.bitsr  sf   € õ 	Jà—‘ˆØ××ÐØ×$Ñ$‰DˆAÜ˜WŸ^™^×1Ñ1Ó3°QÓ:Ð:Ù˜gŸn™n¨f¯l©l¸F×<MÑ<MÓNÈqÕQÐQr%   c               ó    € V ^8„  d   QhRRRR/# )r   r    r   r   zdict[str, object]r!   )r"   s   "r#   r$   ro   ~  s   € ÷ 
ñ 
¨ð 
Ð<Mñ 
r%   c                	óð   € VP                   pR VP                  RVP                  RVP                  RJ RVP                  RJRVP                  RVP
                  P                  RV P                  V4      RR	/# )
r   r   r   Nr   r   r   r   r   F)r   r   r   r4   r   rR   rr   r   r¢   s   && r#   Úrequantize_kwargsÚ$AsymW4A8Int8Layout.requantize_kwargs}  st   € à—‘ˆà˜&×+Ñ+Ø ×!9Ñ!9Ø˜×*Ñ*¨dÐ2Ø˜Ÿ™¨tÐ3Ø˜vŸ™Ø˜6Ÿ<™<×-Ñ-Ø�C—H‘H˜WÓ%à˜Eð

ð 
	
r%   r!   r|   )NNNr9   NN)rf   rg   rh   ri   Ú__doc__ÚMIN_SM_VERSIONÚQUANTIZES_INPUTr   r   rl   Úclassmethodr*   r,   ru   r˜   r�   r£   r¨   r²   r   r¼   rk   r!   r%   r#   rJ   rJ   œ   sø   † ÙWà€NØ€Oá�dÔô5Ð!ó 5ó ð5ðn ð Ø!$ØØ#(×#6Ñ#6ØØ/3Ø#$ØØ!ö$ó ð$ðL ô*Aó ð*AðX ô
ó ð
ð  ô
ó ð
ð& ö
ó ð
ð ôó ðð ôRó ðRð ô
ó ô
r%   rJ   c               ó    € V ^8„  d   QhRRRR/# )r   r   Úobjectr   r   r!   )r"   s   "r#   r$   r$   �  s   € ÷ ^ñ ^�Vð ^ ñ ^r%   c                óP   € \        V \        4      ;'       d    V P                  R 8H  # )rJ   )Ú
isinstancer   r’   )r   s   &r#   Ú_is_w4a8rÆ   �  s#   € Ü�fœoÓ.×]Ð]°6×3EÑ3EÐI]Ñ3]Ð]r%   c               ó    € V ^8„  d   QhRRRR/# )r   Úrhsr   r   r!   )r"   s   "r#   r$   r$   ‘  s   € ÷ ñ ˜?ð ¨ñ r%   c                óT   € V P                   P                  '       g   \        R 4      hV # )z>W4A8 GEMM expects RHS W.T. Use F.linear(x, W) or mm(x, W.t()).)r   rN   ÚRuntimeError)rÈ   s   &r#   Ú_resolve_w4a8_rhsrË   ‘  s#   € Ø�;‰;×!×!Ð!ÜÐ[Ó\Ð\Ø€Jr%   c          
     ó,   € V ^8„  d   QhRRRRRRRRR	R/# )
r   Úinput_tensorr   r   r   r<   r   r=   r   r   r!   )r"   s   "r#   r$   r$   —  s:   € ÷ ñ Øðàðð ðð ð	ð
 ñr%   c                ó´   € \         P                  V4      w  rErgpVP                  p	\        V VVVVVVV	P                  V	P
                  VV	P                  R 7      # ))r   r4   r<   r   r   r=   r>   )rJ   r£   r   rG   r   r   r>   )
rÍ   r   r<   r=   r1   r2   r3   r4   r   rt   s
   &&&&      r#   Ú_w4a8_int8_forwardrÏ   —  sf   € ô 5G×4XÑ4XÐY_Ó4`Ñ1€E�)¨Ø�^‰^€FÜØØØØØØØØ×$Ñ$Ø ×2Ñ2ØØ×$Ñ$ôð r%   c                ó†  € V^ ,          p\        V\        4      '       g5   \        P                  P                  P
                  P                  ! V/ VB # VP                  p\        P                  ! VVP                  ^,          VP                  ^ ,          3VP                  '       * R7      p\        VP                  RV4      # )rY   )r\   rN   rJ   )rÅ   r   r*   ÚopsÚatenÚtÚdefaultr   r�   r�   r\   rN   r~   )ÚqtÚargsr)   rÍ   ÚoldÚ
new_paramss   &&&   r#   Ú_handle_w4a8int8_trÙ   ®  s–   € à˜•7€LÜ�l¤O×4Ò4Ü�y‰y�~‰~×Ñ×'Ò'¨Ð8°Ñ8Ð8Ø
×
Ñ
€CÜ×$Ò$ØØ—N‘N 1Õ% s§~¡~°aÕ'8Ð9Ø—~‘~Ô%ô€Jô
 ˜<×.Ñ.Ð0DÀjÓQÐQr%   c                ó:  € V^ ,          V^,          rC\        V4      ^8”  d
   V^,          MRp\        V4      '       g4   \        P                  P                  P
                  ! \        W4V34      !  # \        V\        4      '       d   VP                  4       pVP                  P                  '       d9   \        P                  P                  P                  W4P                  4       V4      # VP                  RVP                  P                  4      p\        W4WV4      # )rY   Nr=   )r[   rÆ   r*   ÚnnÚ
functionalÚlinearr   rÅ   r   r�   r   rN   Úgetrq   rÏ   )rÕ   rÖ   r)   rÍ   r   r<   r=   s   &&&    r#   Ú_handle_w4a8int8_linearrß   ¼  sÏ   € à �7 D¨¥G�&Ü˜$“i !”mˆ4�Ž7¨€DÜ�F×ÒÜ�x‰x×"Ñ"×)Ò)¬?¸LÐRVÐ;WÓ+XÑYÐYÜ�,¤×0Ò0Ø#×.Ñ.Ó0ˆØ‡~�~× × Ð Ü�x‰x×"Ñ"×)Ñ)¨,×8IÑ8IÓ8KÈTÓRÐRØ—
‘
˜;¨¯©×(AÑ(AÓB€IÜ˜l°DÓDÐDr%   c                ó*  € V^ ,          V^,          rC\        V4      '       g   \        P                  ! \        W434      !  # \	        V\
        4      '       d   VP                  4       p\        V4      p\        VVRVP                  P                  R7      # )rY   N©r<   r=   )rÆ   r*   Úmmr   rÅ   r   r�   rË   rÏ   r   rq   )rÕ   rÖ   r)   rÍ   r   s   &&&  r#   Ú_handle_w4a8int8_mmrã   Ê  s{   € à �7 D¨¥G�&Ü�F×ÒÜ�xŠxœ¨,Ð)?Ó@ÑAÐAÜ�,¤×0Ò0Ø#×.Ñ.Ó0ˆÜ˜vÓ&€FÜØØØØ—.‘.×+Ñ+ô	ð r%   c                ó>  € V^ ,          V^,          V^,          rTpVP                  R^4      ^8g  ;'       g    VP                  R^4      ^8g  p\        V\        P                  4      ;'       d    VP	                  4       ^8g  p\        V4      '       d   V'       g	   V'       d)   \        P                  ! \        V4      / \        V4      B # \        V\        4      '       d   VP                  4       p\        V4      p\        VVVVP                  P                  R7      # )rY   ÚbetaÚalphará   )rÞ   rÅ   r*   ÚTensorÚdimrÆ   Úaddmmr   r   r�   rË   rÏ   r   rq   )rÕ   rÖ   r)   r<   rÍ   r   ÚscaledÚmatrix_addends   &&&     r#   Ú_handle_w4a8int8_addmmrì   Ú  sÝ   € à!% a¥¨$¨q­'°4¸µ7˜€DØ�Z‰Z˜ Ó" aÑ'×FÐF¨6¯:©:°g¸qÓ+AÀQÑ+F€FÜ˜t¤U§\¡\Ó2×FÐF°t·x±x³zÀQ±€MÜ�F×ÒŸv¯Ü�{Š{œO¨DÓ1ÐM´_ÀVÓ5LÑMÐMÜ�,¤×0Ò0Ø#×.Ñ.Ó0ˆÜ˜vÓ&€FÜØØØØ—.‘.×+Ñ+ô	ð r%   )&r¾   Ú
__future__r   r�   r   r*   Úcomfy_kitchen.registryr   Úbaser   r   r   r   r	   Úw4a8_packedr
   r   r   r,   r'   Úbfloat16r7   rG   rJ   rÆ   rË   rÏ   rÑ   rÒ   rÓ   rÔ   rÙ   rÝ   rß   râ   rã   ré   rì   r!   r%   r#   Ú<module>rò      s‰  ðñõ #ã Ý !ã å +÷õ ÷ñ ð Ø ØØ$×2Ñ2ØØ+/Ø ØØ÷*ðb %)Ø&*ØØ Ø %§¡÷ð< %)Ø&*Ø $ØØ Ø"Ÿ^™^ØØ Ø,0ØØ$(Ø*.÷!/ôdn
˜ô n
õb^õõñ. �E—I‘I—N‘N×$Ñ$×,Ñ,Ð.@ÓAñ
Ró Bð
Rñ �E—I‘I—N‘N×)Ñ)×1Ñ1Ð3EÓFñ
Eó Gð
Eñ �E—I‘I—N‘N×%Ñ%×-Ñ-Ð/AÓBñó Cðñ �E—I‘I—N‘N×(Ñ(×0Ñ0Ð2DÓEñó Fòr%   