+
    ªåûj�Y  ã                  ón  € R t ^ RIHt ^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RI	t	^ RI
t^RIHtHtHtHtHt  ^ RIHt ]P,                  ! ]4      t^@t^€tRt0 R-mtR R	 ltR
 R ltR R ltR R lt R R lt!RRRR/R R llt"R R lt#R.RRRR/R R lllt$R R lt%R.RRRR/R R  lllt&R! R" lt' ! R# R$]4      t(R% R& lt)]! ]	PT                  PV                  PX                  PZ                  ](4      R' 4       t.R( R) lt/]! ]	PT                  PV                  P`                  PZ                  ](4      R* 4       t1]! ]	PT                  PV                  Pd                  PZ                  ](4      R+ 4       t3]! ]	PT                  PV                  Ph                  PZ                  ](4      R, 4       t5R#   ] d    Rt ELui ; i)/a~  SVDQuant W4A4 quantization layout for tensor cores.

Each quantized linear stores:
  qweight:       (N, K // 2)  int8        packed W4 residual
  scale=wscales: (K // 64, N) bf16/fp16   per-group weight scales
  proj_down:     (K, R)       bf16/fp16   SVD down projection (V^T)
  proj_up:       (N, R)       bf16/fp16   SVD up projection (U)
  smooth_factor: (K,)         bf16/fp16   input-side smoothing

LoRA-style proj_down / proj_up recover the outlier-heavy singular directions
that pure 4-bit quantization cannot represent; the dispatched kernel fuses
activation quantization + low-rank correction + int4 matmul into a single call.
)ÚannotationsN)ÚSequence)Ú	dataclass)ÚBaseLayoutParamsÚQuantizedLayoutÚQuantizedTensorÚdequantize_argsÚregister_layout_op)Úcudag      Æ?c               ó$   € V ^8„  d   QhRRRRRR/# )é   ÚnameÚstrÚdefaultÚboolÚreturn© )Úformats   "Ú^C:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\tensor\svdquant_w4a4.pyÚ__annotate__r   4   s!   € ÷ :ñ :�sð :¨ð :°ñ :ó    c               ó„   € \         P                  ! V 4      pVf   V# VP                  4       P                  4       \        9  # ©N)ÚosÚgetenvÚstripÚlowerÚ_FALSE_ENV_VALUES)r   r   Úvalues   &$ r   Ú_env_enabledr   4   s4   € Ü�IŠI�d‹O€EØ‚}ØˆØ�;‰;‹=×ÑÓ Ô(9Ñ9Ð9r   c               ó   € V ^8„  d   QhRR/# )r   r   z
str | Noner   )r   s   "r   r   r   ;   s   € ÷ Hñ H Jñ Hr   c                 óL   € \        \        P                  P                  R R4      # )Úbackend_overrideN)ÚgetattrÚckÚregistryÚ_thread_localr   r   r   Ú_registry_backend_overrider'   ;   s   € Ü”2—;‘;×,Ñ,Ð.@À$ÓGÐGr   c               ó    € V ^8„  d   QhRRRR/# )r   Úinput_tensorútorch.TensorÚqdatar   )r   s   "r   r   r   ?   s   € ÷ ñ  |ð ¸Lñ r   c                ó2  € \        RRR7      '       g   R# \        4       pVe
   VR8w  d   R# V P                  '       d   VP                  '       g   R# V P                  VP                  8w  d   R# \        f   R# \        \        RR4      '       g   R# \        # )a£  Return the CUDA backend module when SVDQuant can safely bypass registry.

ComfyUI currently disables comfy_kitchen's global CUDA backend on PyTorch
CUDA < 13, but this SVDQuant extension is built locally and works on the
cu128/RTX 5090 environment. The QuantizedTensor path can call the CUDA
implementation directly while still respecting an explicit non-CUDA backend
override such as ``with ck.use_backend("eager")``.
Ú"COMFY_KITCHEN_SVDQUANT_DIRECT_CUDAT)r   Nr
   Ú_EXT_AVAILABLEF)r   r'   Úis_cudaÚdeviceÚcuda_backendr#   )r)   r+   Úoverrides   && r   Ú_direct_cuda_backendr3   ?   s{   € ô Ð<Àd×KÓKÙä)Ó+€HØÒ ¨FÔ 2Ùà××Ð u§}§} }ÙØ×Ñ˜eŸl™lÔ*ÙäÒÙÜ”<Ð!1°5×9Ò9ÙÜÐr   c               ó    € V ^8„  d   QhRRRR/# )r   Útensorr*   r   r   r   )r   s   "r   r   r   [   s   € ÷ ñ  lð °tñ r   c                óª   € \        V \        4      ;'       d=    V P                  R 8H  ;'       d&    \        \	        V P
                  RR4      4      '       * # )ÚTensorCoreSVDQuantW4A4LayoutÚ
transposedF)Ú
isinstancer   Ú_layout_clsr   r#   Ú_params)r5   s   &r   Ú_is_svdquant_w4a4_qtensorr<   [   sP   € ä�6œ?Ó+÷ 	Cð 	CØ×ÑÐ"@Ñ@÷	Cð 	Cä”W˜VŸ^™^¨\¸5ÓAÓBÔBðr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úar*   Úbr   r   r   )r   s   "r   r   r   c   s&   € ÷ Nñ N˜\ð N¨lð N¸tñ Nr   c                ó¸   € V P                   VP                   8H  ;'       d;    V P                  VP                  8H  ;'       d    V P                  VP                  8H  # r   )ÚshapeÚdtyper0   )r>   r?   s   &&r   Ú_same_tensor_metadatarC   c   s?   € Ø�7‰7�a—g‘gÑ×MÐM !§'¡'¨Q¯W©WÑ"4×MÐM¸¿¹ÀQÇXÁXÑ9MÐMr   ÚvalidateFÚtrustc               ó(   € V ^8„  d   QhRRRRRRRR/# )r   ÚweightsúSequence[torch.Tensor]rD   r   rE   r   r   )r   s   "r   r   r   g   s2   € ÷ -ñ -Ø#ð-ð ð-ð ð	-ð
 
ñ-r   c               óæ  € \        V 4      ^ 8X  d   R# \        ;QJ d    R V  4       F  '       d   K   RM	  RM! R V  4       4      '       g   R# V ^ ,          P                  pV R,           EF�  pVP                  p\        VP                  4      \        VP                  4      8w  d    R# \        VP                  VP                  4      '       g    R# \        VP                  VP                  4      '       g    R# VP                  P                  4       VP                  P                  4       8X  d;   VP                  P                  4       VP                  P                  4       8X  d   EK  V'       d   EK  V'       g    R# \        P                  ! VP                  VP                  4      '       g    R# \        P                  ! VP                  VP                  4      '       d   EK�   R# 	  R# )a¬  Return True when split SVDQuant projections can reuse activation quantize.

Qwen split Q/K/V checkpoints store separate weight tensors but share the
exact same activation-side SVDQuant parameters: ``smooth_factor`` and
``proj_down``. When that invariant holds, the expensive activation int4
quantization and LoRA-down matmul can run once and feed all projections.

``validate=True`` compares tensor values when pointers differ. Use it once
at module setup or first forward, then cache the result at the caller. Pass
``trust=True`` only after such a cached validation, for temporary casted
copies whose tensor pointers differ but whose source parameters were
already proven identical.
Fc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5ir   )r<   )Ú.0Úweights   & r   Ú	<genexpr>Ú0svdquant_w4a4_can_share_quant.<locals>.<genexpr>|   s   é € ÐG¹w°VÔ(¨×0Ð0»wùs   ‚T:é   NN)ÚlenÚallr;   r   Úact_unsignedrC   Úsmooth_factorÚ	proj_downÚdata_ptrÚtorchÚequal)rG   rD   rE   ÚbaserL   Úparamss   &$$   r   Úsvdquant_w4a4_can_share_quantrZ   g   sR  € ô& ˆ7ƒ|�qÔÙß‹3ÑG¹wÓG�3�3Š3ÑG¹wÓG×GÒGÙà�1�:×Ñ€DØ˜"—+‘+ˆØ—‘ˆÜ�×#Ñ#Ó$¬¨T×->Ñ->Ó(?Ô?ÚÜ$ V×%9Ñ%9¸4×;MÑ;M×NÒNÚÜ$ V×%5Ñ%5°t·~±~×FÒFÚØ×Ñ×(Ñ(Ó*¨d×.@Ñ.@×.IÑ.IÓ.KÔKØ×Ñ×%Ñ%Ó'¨4¯>©>×+BÑ+BÓ+DÔDâßÚßÚÜ�{Š{˜6×/Ñ/°×1CÑ1C×DÒDÚÜ�{Š{˜6×+Ñ+¨T¯^©^×<Õ<Úñ' ñ( r   c               ó4   € V ^8„  d   QhRRRRRRRRRRR	R
RR/# )r   Úq_xr*   ÚascalesÚlora_actÚ	weight_qtr   Úbiasútorch.Tensor | NoneÚmÚintr   ztuple[torch.Tensor, int]r   )r   s   "r   r   r   —   sN   € ÷ !ñ !Ø	ð!àð!ð ð!ð ð	!ð
 ð!ð ð!ð ñ!r   c               ó  € \         P                  V4      w  rxršp\        \        VP                  R R4      4      pVe   VP
                  ! WWW+VVR7      pM\        P
                  ! WWW+VVR7      p\         P                  V4      pVRV V3# )rR   FN©ÚactÚwgtr]   ÚwscalesÚlora_act_inÚlora_upr`   rR   )r7   Úget_plain_tensorsr   r#   r;   Úscaled_mm_svdquant_w4a4r$   Úget_out_features_from_storage)r\   r]   r^   r_   r`   r1   rb   r+   rh   Ú_smoothÚ
_proj_downÚproj_uprR   ÚoutÚout_featuress   &&&&&&$        r   Ú'_w4a4_forward_from_quantized_activationrs   —   s›   € ô 4P×3aÑ3aÐbkÓ3lÑ0€E�G¨Üœ 	× 1Ñ 1°>À5ÓIÓJ€LØÒØ×2Ò2Ø¨Ø ¸Ø%ô
‰ô ×(Ò(Ø¨Ø ¸Ø%ô
ˆô
 0×MÑMÈeÓT€LØˆr�ˆ7�LÐ Ð r   Úvalidate_shared_quantÚassume_shared_quantc               ó0   € V ^8„  d   QhRRRRRRRRR	RR
R/# )r   r)   r*   rG   rH   Úbiasesú$Sequence[torch.Tensor | None] | Nonert   r   ru   r   útuple[torch.Tensor, ...]r   )r   s   "r   r   r   ³   sL   € ÷ @ñ @Øð@à#ð@ð 1ð@ð
  ð@ð ð@ð ñ@r   c               óˆ  € \        V4      ^ 8X  d   R# Vf   R\        V4      ,          p\        V4      \        V4      8w  d%   \        R\        V4       R\        V4       R24      h\        WVR7      '       g   \        R4      hV^ ,          p\        P	                  V4      w  rgr‰p
\        \        VP                  RR4      4      pV P                  pV P                  RVR,          4      pVP                  ^ ,          pV'       d   V\        ,           pTpMTpRp\        Wö4      pVe   VP                  ! VVV	VVR	7      w  pppM\        P                  ! VVV	VVR	7      w  ppp. p\        WR
R7       FB  w  pp\        VVVVVVVR7      w  ppVP!                  VP                  ! . VRR OVN5!  4       KD  	  \#        V4      # )a  Run split SVDQuant linears while sharing activation quantize.

This is intentionally a runtime grouping helper, not a fused-QKV storage
format. It preserves split Q/K/V checkpoint tensors and only removes the
repeated per-input work that is identical across those projections.
Núgot ú weights but ú biases©rD   rE   ú5SVDQuant weights do not share quantization parametersrR   F©ÚsmoothÚ	lora_downrR   Úlora_xT©Ústrict)rb   r   r   éÿÿÿÿ)rP   Ú
ValueErrorrZ   r7   rk   r   r#   r;   rA   ÚreshapeÚ_GELU_UNSIGNED_SHIFTr3   Úquantize_svdquant_w4a4r$   Úziprs   ÚappendÚtuple)r)   rG   rw   rt   ru   Úfirstr+   Ú_wscalesr�   rT   Ú_proj_uprR   Ú
orig_shapeÚx2drb   Úx_mainrƒ   r1   r\   r]   r^   ÚoutputsrL   r`   Úout2drr   s   &&&$$                     r   Úsvdquant_w4a4_grouped_linearr–   ³   sÇ  € ô ˆ7ƒ|�qÔØˆ	Ø‚~Øœ3˜w›<Õ'ˆÜ
ˆ6ƒ{”c˜'“lÔ"Ü˜4¤ G£˜~¨]¼3¸v»;¸-ÀwÐOÓPÐPÜ(ØÐ7J÷ó ô ÐPÓQÐQà�A�J€EÜ3O×3aÑ3aÐbgÓ3hÑ0€E�V¨Üœ §¡¨~¸uÓEÓF€Là×#Ñ#€JØ
×
Ñ
˜r :¨b¥>Ó
2€CØ�	‰	�!�€AçØÔ+Õ+ˆØ‰àˆØˆä'¨Ó6€LØÒØ!-×!DÒ!DØØØØ%Øô"
ÑˆˆW‘hô "$×!:Ò!:ØØØØ%Øô"
ÑˆˆW�hð €GÜ˜G°D×9‰ˆ�ÜEØ�˜( F¨D°,À!ô
Ñˆˆ|ð 	�‰�u—}’}ÐD j°°" oÐD°|ÓDÖEñ	 :ô
 �‹>Ðr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   ÚtensorsrH   Únatural_dimrc   r   r*   r   )r   s   "r   r   r   ö   s(   € ÷ Cñ CÐ"8ð CÈ#ð CÐR^ñ Cr   c               ó’  a€ V ^ ,          P                  4       o\        ;QJ d    V3R lV  4       F  '       g   K   RM	  RM! V3R lV  4       4      '       d   \        R4      hSR9   d0   \        P                  ! \        V 4      ^ R7      P                  4       # \        P                  ! \        V 4      VR7      P                  4       # )é    c              3  óH   <"  € T F  qP                  4       S8g  x € K  	  R # 5ir   ©Údim)rK   Útrž   s   & €r   rM   Ú'_cat_svdquant_n_axis.<locals>.<genexpr>ø   s   øé € Ð
+¡7˜a�5‰5‹7�cŽ>£7ùs   ƒ"TFz6cannot fuse mixed natural/tile-packed SVDQuant tensorsr�   >   é   é   )rž   Úanyr‡   rV   Úcatr�   Ú
contiguous)r˜   r™   rž   s   &$@r   Ú_cat_svdquant_n_axisr¦   ö   sƒ   ø€ Ø
�!�*�.‰.Ó
€Cß
ƒsÔ
+¡7Ó
+‡s‡s‚sÔ
+¡7Ó
+×+Ò+ÜÐQÓRÐRØ
ˆf„}Ü�yŠyœ˜w›¨QÔ/×:Ñ:Ó<Ð<Ü�9Š9”U˜7“^¨Ô5×@Ñ@ÓBÐBr   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
r   rG   rH   rw   rx   rt   r   ru   r   z<tuple[QuantizedTensor, torch.Tensor | None, tuple[int, ...]]r   )r   s   "r   r   r   ÿ   sB   € ÷ Fñ FØ#ðFà0ðFð  ð	Fð
 ðFð BñFr   c          
     óè  € \        V 4      ^ 8X  d   \        R4      hVf   R\        V 4      ,          p\        V4      \        V 4      8w  d%   \        R\        V 4       R\        V4       R24      h\        WVR7      '       g   \        R4      hV ^ ,          pVP                  pVP                  ^,          p\        VP                  4      p. p. p	. p
. pV  FÐ  p\        P                  V4      w  rÞpppVP                  P                  ^,          V8w  d   \        R4      h\        VP                  P                  4      V8w  d   \        R	4      hVP                  \        P                  V4      4       V	P                  V4       V
P                  V4       VP                  V4       KÒ  	  \        V	^ R
7      p\        V
^R
7      p\        V^ R
7      p\        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       d   RpMŠ. p\        WRR7       FH  w  ppVf.   \        P                  ! VVP                   VP"                  R7      pVP                  V4       KJ  	  \        P$                  ! \'        V4      ^ R7      P)                  4       p\        P+                  VVP,                  \/        V4      V3VP0                  VVP2                  VR7      p\5        VRV4      V\'        V4      3# )a  Fuse split SVDQuant projections into a runtime-only wide projection.

This does not change checkpoint storage. It concatenates the already loaded
split weights along output-N so a caller can execute Q/K/V as one wider
SVDQuant linear and split the output view afterwards.
zexpected at least one weightNr{   r|   r}   r~   r   z<all fused SVDQuant weights must have the same input featuresz?all fused SVDQuant weights must have the same act_unsigned flag)r™   c              3  ó(   "  € T F  qR J x € K
  	  R # 5ir   r   )rK   r`   s   & r   rM   Ú4svdquant_w4a4_fuse_linear_weights.<locals>.<genexpr>.  s   é € Ð
+¡F˜D�4�<£Fùs   ‚FTr„   ©rB   r0   r�   )ÚscaleÚ
orig_dtyper‘   rT   rp   rS   rR   r7   r   )rP   r‡   rZ   r;   r‘   r   rR   r7   rk   rŒ   rm   r¦   rQ   r‹   rV   ÚzerosrB   r0   r¤   r�   r¥   ÚParamsr­   ÚsumrT   rS   r   )rG   rw   rt   ru   rŽ   Úfirst_paramsÚin_featuresrR   rr   ÚqdatasÚscalesÚproj_upsrL   r+   Úwscalern   ro   rp   Úfused_qdataÚfused_scaleÚfused_proj_upÚ
fused_biasÚ
bias_partsr`   ÚnrY   s   &&$$                      r   Ú!svdquant_w4a4_fuse_linear_weightsr½   ÿ   s’  € ô ˆ7ƒ|�qÔÜÐ7Ó8Ð8Ø‚~Øœ3˜w›<Õ'ˆÜ
ˆ6ƒ{”c˜'“lÔ"Ü˜4¤ G£˜~¨]¼3¸v»;¸-ÀwÐOÓPÐPÜ(ØÐ7J÷ó ô ÐPÓQÐQà�A�J€EØ—=‘=€LØ×)Ñ)¨!Õ,€KÜ˜×1Ñ1Ó2€LØ€LØ€FØ€FØ€HÛˆÜ6R×6dÑ6dÐekÓ6lÑ3ˆ�w 
¨GØ�>‰>×$Ñ$ QÕ'¨;Ô6ÜÐ[Ó\Ð\Ü�—‘×+Ñ+Ó,°Ô<ÜÐ^Ó_Ð_Ø×ÑÔ8×VÑVÐW\Ó]Ô^Ø�‰�eÔØ�‰�fÔØ�‰˜Ö ñ ô ' v¸1Ô=€KÜ& v¸1Ô=€KÜ(¨¸qÔA€Mç
ƒsÑ
+¡FÓ
+‡s‡s‚sÑ
+¡FÓ
+×+Ò+Ø‰
àˆ
Ü˜6¸×=‰GˆD�!ØŠ|Ü—{’{ 1¨M×,?Ñ,?È×H\ÑH\Ô]�Ø×Ñ˜dÖ#ñ >ô —Y’Yœu ZÓ0°aÔ8×CÑCÓEˆ
ä)×0Ñ0ØØ×*Ñ*Ü˜Ó% {Ð3Ø×(Ñ(ØØ"×0Ñ0Ø!ð 1ó €Fô 	˜Ð%CÀVÓLØÜˆlÓðð r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
r   r)   r*   Úfused_weightrº   ra   Úoutput_featureszSequence[int]r   ry   r   )r   s   "r   r   r   H  sA   € ÷ 
Cñ 
CØð
Càð
Cð $ð
Cð #ð	
Cð
 ñ
Cr   c                ó¨   € \        V4      '       g   \        R4      h\        WV4      p\        \        P
                  ! V\        V4      RR7      4      # )zGExecute a runtime-fused split projection and return split output views.zCfused_weight must be a TensorCoreSVDQuantW4A4Layout QuantizedTensorr�   r†   )r<   Ú	TypeErrorÚ_w4a4_forwardr�   rV   Úsplit)r)   r¿   rº   rÀ   rq   s   &&&& r   Ú"svdquant_w4a4_fused_grouped_linearrÅ   H  sD   € ô % \×2Ò2ÜÐ]Ó^Ð^Ü
˜°JÓ
?€CÜ”—’˜S¤%¨Ó"8¸bÔAÓBÐBr   c                  ó¾   € ] tR tRtRtRtRt]! RR7       ! R R]4      4       t	]
R R	 l4       t]
R
 R l4       t]
R R l4       t]
R R l4       t]
R R l4       tRtR# )r7   iU  uf  SVDQuant W4A4 weight quantization with low-rank correction.

Note:
    Offline-quantized only â€” `quantize()` raises NotImplementedError because
    SVDQuant factorization requires calibration (smooth_factor, proj_down,
    proj_up) that must be computed from activation statistics. Use the
    DeepCompressor pipeline to produce the pre-quantized tensors.
FT)Úfrozenc                  óh   € ] tR tRt$ RtR]R&   R]R&   R]R&   RtR]R	&   RtR]R
&   R R ltR t	Rt
R# )Ú#TensorCoreSVDQuantW4A4Layout.Paramsik  a  SVDQuant W4A4 parameters.

Inherits `scale` (= wscales), `orig_dtype`, `orig_shape` from
BaseLayoutParams. Adds the three tensors that parameterize the
low-rank correction and input smoothing, plus a logical-transpose flag
used by the aten.t / aten.mm dispatch path.
r*   rT   rp   rS   Fr   rR   r8   c               ó   € V ^8„  d   QhRR/# )r   r   z	list[str]r   )r   s   "r   r   Ú0TensorCoreSVDQuantW4A4Layout.Params.__annotate__z  s   € ÷ 	Fñ 	F Iñ 	Fr   c                	ó
   € . RO# )r¬   ©r¬   rT   rp   rS   r   ©Úselfs   &r   Ú_tensor_fieldsÚ2TensorCoreSVDQuantW4A4Layout.Params._tensor_fieldsz  s   € ÚEÐEr   c                	ó   € R # r   r   rÎ   s   &r   Ú_validate_tensor_fieldsÚ;TensorCoreSVDQuantW4A4Layout.Params._validate_tensor_fields}  s   € ñ r   r   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__rR   r8   rÐ   rÓ   Ú__static_attributes__r   r   r   r¯   rÉ   k  s:   ‡ ñ	ð  ÓØÓØ#Ó#Ø"ˆ�dÓ"Ø ˆ
�DÓ õ	Fö	r   r¯   c               ó    € V ^8„  d   QhRRRR/# )r   r5   r*   r   ztuple[torch.Tensor, Params]r   )r   s   "r   r   Ú)TensorCoreSVDQuantW4A4Layout.__annotate__ƒ  s   € ÷ 
ñ 
àð
ð 
%ñ	
r   c                	ó   € \        R 4      h)zvSVDQuant W4A4 requires offline calibration (DeepCompressor). Load pre-quantized tensors via `from_state_dict` instead.)ÚNotImplementedError)Úclsr5   Úkwargss   &&,r   ÚquantizeÚ%TensorCoreSVDQuantW4A4Layout.quantize‚  s   € ô "ðHó
ð 	
r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r+   r*   rY   r¯   r   r   )r   s   "r   r   rÝ   Ž  s!   € ÷ &ñ &˜|ð &°Vð &Àñ &r   c           
     ó„  € VP                   ^,          pVP                  pVP                  p\        P                  ! W5VR7      p\
        P                  ! WbP                  VP                  R7      w  rxp	\
        P                  ! WqW‚P                  W’P                  RR7      RV p
V
P                  4       P                  4       # )u%  Reconstruct the effective weight W_eff such that plain ``x @ W_eff.T + bias``
reproduces the SVDQuant kernel output to bf16 precision.

Uses the kitchen kernel itself with an identity input rather than
reimplementing dequant in Python. Kitchen weight layout is natural
row-major packed int4 ``(N, K/2)`` â€” the kernel reads it directly, so
this path stays bit-exact with the actual compute path regardless of
per-group scaling / LoRA composition details. Tile-packed storage is
handled by the same kernel path and produces the same logical weight.
r«   )r�   r‚   N)rf   rg   r]   rh   ri   rj   r`   )r‘   r0   r­   rV   Úeyer$   rŠ   rS   rT   rl   r¬   rp   rŸ   r¥   )rà   r+   rY   r²   r0   rB   ræ   r\   r]   r^   Úw_effs   &&&        r   Ú
dequantizeÚ'TensorCoreSVDQuantW4A4Layout.dequantize�  s¥   € ð ×'Ñ'¨Õ*ˆØ—‘ˆØ×!Ñ!ˆä�iŠi˜¸Ô@ˆÜ!#×!:Ò!:Ø×,Ñ,¸×8HÑ8Hô"
Ñˆ�hô ×*Ò*Ø¨¿¹Ø ¯.©.¸tô
ð ˆ;ðˆð �w‰w‹y×#Ñ#Ó%Ð%r   c               ó    € V ^8„  d   QhRRRR/# )r   Úqtensorr   r   zKtuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]r   )r   s   "r   r   rÝ   ¨  s    € ÷ Pñ PØ%ðPà	TñPr   c                	óŒ   € VP                   pVP                  VP                  VP                  VP                  VP
                  3# r   )r;   Ú_qdatar¬   rS   rT   rp   )rà   rë   Úps   && r   rk   Ú.TensorCoreSVDQuantW4A4Layout.get_plain_tensors§  s2   € ð �O‰OˆØ�~‰~˜qŸw™w¨¯©¸¿¹ÀaÇiÁiÐOÐOr   c               ó    € V ^8„  d   QhRRRR/# )r   r+   r*   r   rc   r   )r   s   "r   r   rÝ   ¯  s   € ÷ #ñ #°,ð #À3ñ #r   c                	ó²   € VP                  4       ^8X  d(   \        VP                  ^ ,          4      \        ,          # \        VP                  ^ ,          4      # )r¢   )rž   rc   rA   Ú_TILE_PACKED_BLOCK_N)rà   r+   s   &&r   rm   Ú:TensorCoreSVDQuantW4A4Layout.get_out_features_from_storage®  s<   € à�9‰9‹;˜!ÔÜ�u—{‘{ 1•~Ó&Ô)=Õ=Ð=Ü�5—;‘;˜q•>Ó"Ð"r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r+   r*   rY   r¯   r   zdict[str, torch.Tensor]r   )r   s   "r   r   rÝ   µ  s"   € ÷ 
ñ 
 |ð 
¸Vð 
ÐH_ñ 
r   c           
     ój   € RVRVP                   RVP                  RVP                  RVP                  /# )zÏSerialization mapping.

Suffixes compose onto the owning Parameter's key (typically `*.weight`),
producing for example `transformer_blocks.0.attn.to_q.weight`,
`...weight_scale`, `...weight_proj_down`, etc.
Ú Ú_scalero   r�   Ú_smooth_factorrÍ   )rà   r+   rY   s   &&&r   Ústate_dict_tensorsÚ/TensorCoreSVDQuantW4A4Layout.state_dict_tensors´  s;   € ð �Ø�f—l‘lØ˜&×*Ñ*Ø˜Ÿ™Ø˜f×2Ñ2ð
ð 	
r   r   N)é   r›   )rÕ   rÖ   r×   rØ   rÙ   ÚMIN_SM_VERSIONÚQUANTIZES_INPUTr   r   r¯   Úclassmethodrâ   rè   rk   rm   rù   rÛ   r   r   r   r7   r7   U  sš   † ñð €Nð €Oá�dÔôÐ!ó ó ðð, ô
ó ð
ð ô&ó ð&ð2 ôPó ðPð ô#ó ð#ð
 ô
ó ô
r   r7   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   r)   r*   r_   r   r`   ra   r   r   )r   s   "r   r   r   Ç  s0   € ÷ ;;ñ ;;Øð;;àð;;ð ð;;ð ñ	;;r   c                óx  € \         P                  V4      w  r4rVp\        \        VP                  RR4      4      pV P
                  p	V P                  RV	R,          4      p
V
P
                  ^ ,          pV'       d   V
\        ,           pT
pMT
pRp\        WÃ4      pVe5   VP                  ! VVVVVR7      w  pppVP                  ! WóVVVWrVR7      pM;\        P                  ! VVVVVR7      w  ppp\        P                  ! WóVVVWrVR7      p\         P                  V4      pVRV P                  ! . V	RR OVN5!  # )u,  Compute y = x @ W^T + bias via the int4 kernel.

For layers flagged ``act_unsigned`` (nunchaku convention: post-GELU fc2),
we apply the +0.171875 shift to the main-path activation here at the layer
so it falls into the unsigned [0, 15] quantization grid. LoRA continues to
use the raw un-shifted activation (SVDQuant invariant: LoRA is the residual
between full-precision W and the int4 approximation, computed on the
pre-quantization activation).

Kernel API stays shift-free â€” shift is a Qwen/Flux model-topology constant,
not a quantize-op parameter.
rR   FNr€   re   r†   )r7   rk   r   r#   r;   rA   rˆ   r‰   r3   rŠ   rl   r$   rm   )r)   r_   r`   r+   rh   r�   rT   rp   rR   r‘   r’   rb   r“   rƒ   r1   r\   r]   r^   rq   rr   s   &&&                 r   rÃ   rÃ   Ç  sV  € ô" 2N×1_Ñ1_Ð`iÓ1jÑ.€E�F wÜœ 	× 1Ñ 1°>À5ÓIÓJ€Là×#Ñ#€JØ
×
Ñ
˜r :¨b¥>Ó
2€CØ�	‰	�!�€AçØÔ+Õ+ˆØ‰àˆØˆä'¨Ó6€LØÒØ!-×!DÒ!DØØØØ%Øô"
ÑˆˆW�hð ×2Ò2Ø¨¸Ø ¨'Ø%ô
‰ô "$×!:Ò!:ØØØØ%Øô"
ÑˆˆW�hô ×(Ò(Ø¨¸Ø ¨'Ø%ô
ˆô
 0×MÑMÈeÓT€LØˆr�ˆ7�?Š?Ð:˜J s¨˜OÐ:¨\Ó:Ð:r   c                ó†  € V^ ,          p\        V\        4      '       g5   \        P                  P                  P
                  P                  ! V/ VB # VP                  p\        P                  ! VVP                  ^,          VP                  ^ ,          3VP                  '       * R7      p\        VP                  RV4      # )uÇ   Zero-copy logical transpose â€” flip the ``transposed`` flag.

Lets ``F.linear(x, W)`` decompose into ``x @ W.t()`` without reordering any
storage; ``mm`` / ``addmm`` handlers below unwind the flag.
)r‘   r8   r7   )r9   r   rV   ÚopsÚatenrŸ   r   r;   ÚdataclassesÚreplacer‘   r8   rí   )ÚqtÚargsrá   r)   ÚoldÚ
new_paramss   &&&   r   Ú_handle_w4a4_tr
    s™   € ð ˜•7€LÜ�l¤O×4Ò4Ü�y‰y�~‰~×Ñ×'Ò'¨Ð8°Ñ8Ð8à
×
Ñ
€CÜ×$Ò$ØØ—N‘N 1Õ% s§~¡~°aÕ'8Ð9Ø—~‘~Ô%ô€Jô
 ˜<×.Ñ.Ð0NÐPZÓ[Ð[r   c               ó    € V ^8„  d   QhRRRR/# )r   Úrhsr   r   r   )r   s   "r   r   r     s   € ÷ ñ ˜ð °?ñ r   c                óT   € V P                   P                  '       g   \        R4      hV # )zDReturn rhs unchanged if it is logically transposed (represents W^T).z\SVDQuant W4A4 GEMM expects the RHS to be W.T (stored W). Use F.linear(x, W) or mm(x, W.t()).)r;   r8   ÚRuntimeError)r  s   &r   Ú_resolve_svdquant_rhsr    s+   € à�;‰;×!×!Ð!Üð2ó
ð 	
ð €Jr   c                óø  € V^ ,          V^,          rC\        V4      ^8”  d
   V^,          MRp\        V\        4      '       g4   \        P                  P
                  P                  ! \        W4V34      !  # \        V\        4      '       d   VP                  4       pVP                  P                  '       d9   \        P                  P
                  P                  W4P                  4       V4      # \        W4V4      # )u3   Direct F.linear(input, W, bias) â†’ kitchen kernel.N)rP   r9   r   rV   ÚnnÚ
functionalÚlinearr   rè   r;   r8   rÃ   )r  r  rá   r)   rL   r`   s   &&&   r   Ú_handle_w4a4_linearr  #  s¹   € ð   �7 D¨¥G�&Ü˜$“i !”mˆ4�Ž7¨€Dä�fœo×.Ò.Ü�x‰x×"Ñ"×)Ò)¬?¸LÐRVÐ;WÓ+XÑYÐYÜ�,¤×0Ò0Ø#×.Ñ.Ó0ˆØ‡~�~× × Ð Ü�x‰x×"Ñ"×)Ñ)¨,×8IÑ8IÓ8KÈTÓRÐRÜ˜¨tÓ4Ð4r   c                ó  € V^ ,          V^,          rC\        V\        4      '       g   \        P                  ! \	        W434      !  # \        V\        4      '       d   VP                  4       p\        V4      p\        W4RR7      # )uo   Handle ``mm(x, W.t())`` â€” the decomposition F.linear takes when the weight
is a non-default tensor subclass.
N©r`   )r9   r   rV   Úmmr   rè   r  rÃ   )r  r  rá   r>   r?   s   &&&  r   Ú_handle_w4a4_mmr  2  se   € ð
 ��7�D˜•G€qÜ�aœ×)Ò)Ü�xŠxœ¨!¨Ó0Ñ1Ð1Ü�!”_×%Ò%Ø�L‰L‹NˆÜ˜aÓ €AÜ˜ DÔ)Ð)r   c                ó  € V^ ,          V^,          V^,          rTp\        V\        4      '       g    \        P                  ! \	        W4V34      !  # \        V\        4      '       d   VP                  4       p\        V4      p\        WEVR7      # )z!Handle ``addmm(bias, x, W.t())``.r  )r9   r   rV   Úaddmmr   rè   r  rÃ   )r  r  rá   r`   r>   r?   s   &&&   r   Ú_handle_w4a4_addmmr  @  so   € ð �a•˜$˜q�' 4¨¥7ˆQ€DÜ�aœ×)Ò)Ü�{Š{œO¨T°a¨LÓ9Ñ:Ð:Ü�!”_×%Ò%Ø�L‰L‹NˆÜ˜aÓ €AÜ˜ DÔ)Ð)r   >   Ú0ÚnoÚoffÚfalser   )6rÙ   Ú
__future__r   r  Úloggingr   Úcollections.abcr   r   rV   Úcomfy_kitchenr$   rX   r   r   r   r   r	   Úcomfy_kitchen.backendsr
   r1   Ú	ExceptionÚ	getLoggerrÕ   ÚloggerÚ_INT4_GROUP_SIZErò   r‰   r   r   r'   r3   r<   rC   rZ   rs   r–   r¦   r½   rÅ   r7   rÃ   r  r  rŸ   r   r
  r  r  r  r  r  r  r  r   r   r   Ú<module>r)     sã  ðñõ #ã Û Û 	Ý $Ý !ã ã ÷õ ðÝ;ð 
×	Ò	˜8Ó	$€àÐ ØÐ ØÐ Ú/Ð õ:õHõõ8õNð-ð ð-ð ÷	-õ`!ñ8@ð
 #(ð@ð !&÷@ð @õFCñFð #(ð	Fð
 !&÷Fð FõR
Côm
 ?ô m
õd;;ñ| �E—I‘I—N‘N×$Ñ$×,Ñ,Ð.JÓKñ\ó Lð\õ&ñ �E—I‘I—N‘N×)Ñ)×1Ñ1Ð3OÓPñ5ó Qð5ñ �E—I‘I—N‘N×%Ñ%×-Ñ-Ð/KÓLñ
*ó Mð
*ñ �E—I‘I—N‘N×(Ñ(×0Ñ0Ð2NÓOñ*ó Pò*øðq ô ØƒLðús   ¸F' Æ'	F4Æ3F4