+
    ªåûj  ã                  ól  € R t ^ RIHt ^ RIt^ RIt^ RIHt ^ RIt^ RIt^RI	H
t
HtHtHtHt ]P                  ! ]4      t^@t ! R R]4      tR R lt]! ]P*                  P,                  P.                  P0                  ]4      R	 4       tR
 R lt]! ]P*                  P,                  P6                  P0                  ]4      R 4       t]! ]P*                  P,                  P:                  P0                  ]4      R 4       t]! ]P*                  P,                  P>                  P0                  ]4      R 4       t R# )u/  AWQ W4A16 quantization layout for modulation-style linears.

Each quantized linear stores:
  qweight:        (N, K // 2)  int8       packed uint4 (kitchen-native row-major)
                                          bits 0..3 -> column 2j   (uint4 [0, 15])
                                          bits 4..7 -> column 2j+1 (uint4 [0, 15])
  scale=wscales:  (K // G, N)  bf16/fp16  per-group fp scales
  wzeros:         (K // G, N)  bf16/fp16  per-group fp zero points

Dequantization (per element):
    W[n, k] = (qweight[n, k] - 8) * wscales[k // G, n] + wzeros[k // G, n]

Targets the modulation linears (``img_mod.1`` / ``txt_mod.1``) in
Qwen-Image-Edit and similar topologies â€” small batch, called once per block,
where W4A16 GEMV stays fp16/bf16-accurate while the int4 weights cut both
checkpoint size and resident VRAM by ~4x vs the bf16-dequantized fallback.

Dispatch goes through ``ck.gemv_awq_w4a16``, which has an eager pure-PyTorch
implementation registered as a ``torch.library`` custom op; a CUDA backend
can be added later without changing this layout.
)ÚannotationsN)Ú	dataclass)ÚBaseLayoutParamsÚQuantizedLayoutÚQuantizedTensorÚdequantize_argsÚregister_layout_opc                  ó¨   € ] tR t^2tRtRtRt]! RR7       ! R R]4      4       t	]
R R	 l4       t]
R
 R l4       t]
R R l4       t]
R R l4       tRtR# )ÚTensorCoreAWQW4A16Layoutu#  AWQ W4A16 weight quantization with per-group fp scales + zeros.

Note:
    Offline-quantized only â€” ``quantize()`` raises NotImplementedError.
    Use the upstream AWQ / DeepCompressor calibration pipeline to produce
    the pre-quantized tensors. ``from_state_dict`` is the loading path.
NFT)Úfrozenc                  óT   € ] tR t^Bt$ RtR]R&   ]tR]R&   RtR]R&   R	 R
 lt	R t
RtR# )ÚTensorCoreAWQW4A16Layout.Paramsu  AWQ W4A16 parameters.

Inherits ``scale`` (= wscales), ``orig_dtype``, ``orig_shape`` from
BaseLayoutParams. Adds ``zeros`` (per-group fp zero points) and
``group_size`` (the K-dim quantization group size â€” typically 64,
matching the wscales / wzeros leading shape).
útorch.TensorÚzerosÚintÚ
group_sizeFÚboolÚ
transposedc               ó   € V ^8„  d   QhRR/# )é   Úreturnz	list[str]© )Úformats   "ÚZC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\tensor\awq_w4a16.pyÚ__annotate__Ú,TensorCoreAWQW4A16Layout.Params.__annotate__O   s   € ÷ 	&ñ 	& Iñ 	&ó    c                	ó
   € R R.# ©Úscaler   r   ©Úselfs   &r   Ú_tensor_fieldsÚ.TensorCoreAWQW4A16Layout.Params._tensor_fieldsO   s   € Ø˜WÐ%Ð%r   c                	ó   € R # ©Nr   r    s   &r   Ú_validate_tensor_fieldsÚ7TensorCoreAWQW4A16Layout.Params._validate_tensor_fieldsR   s   € ár   r   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú_DEFAULT_GROUP_SIZEr   r   r"   r&   Ú__static_attributes__r   r   r   ÚParamsr   B   s-   ‡ ñ	ð ÓØ-ˆ
�CÓ-Ø ˆ
�DÓ õ	&ö	r   r0   c               ó   € V ^8„  d   QhRR/# )r   Útensorr   r   )r   s   "r   r   Ú%TensorCoreAWQW4A16Layout.__annotate__W   s   € ÷ 
ñ 
˜lñ 
r   c                	ó   € \        R 4      h)zaAWQ W4A16 requires offline calibration. Load pre-quantized tensors via `from_state_dict` instead.)ÚNotImplementedError)Úclsr2   Úkwargss   &&,r   ÚquantizeÚ!TensorCoreAWQW4A16Layout.quantizeV   s   € ä!ð5ó
ð 	
r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úqdatar   Úparamsr0   r   r   )r   s   "r   r   r3   ^   s!   € ÷ ñ ˜|ð °Vð Àñ r   c                óö  € VP                   w  r4V^,          pVP                  pVP                  \        P                  4      pV^,          P                  \        P                  4      pV^,	          ^,          P                  \        P                  4      p	\        P
                  ! W5\        P                  VP                  R7      p
WŠR&   WšR&   VP                  P                  4       P                  R4      pVP                  P                  4       P                  R4      pV
P                  W5V,          V4      P                  VP                  4      pVR,
          V,          V,           P                  W54      pV# )züReconstruct the bf16/fp16 weight matrix W of shape ``(N, K)``.

Used by the fallback path when the dispatch handler can't run a
quantized matmul (e.g. RHS is not transposed, or operand types don't
match). Stays in fp accumulation on the eager backend.
)ÚdtypeÚdeviceg       @)ºNNN:é    Nr   )r@   :é   Nr   éÿÿÿÿ)Úshaper   ÚtoÚtorchÚint32Úemptyr?   r   ÚtÚ	unsqueezer   ÚviewÚ
orig_dtype)r6   r;   r<   ÚnÚk_halfÚkÚgÚq_i32ÚloÚhiÚw_uintÚscalesr   ÚgroupsÚws   &&&            r   Ú
dequantizeÚ#TensorCoreAWQW4A16Layout.dequantize]   s  € ð —K‘K‰	ˆØ�Q�JˆØ×Ñˆà—‘œŸ™Ó%ˆØ�d�l×ÑœuŸ{™{Ó+ˆØ˜�z˜TÕ!×%Ñ%¤e§k¡kÓ2ˆÜ—’˜Q¬¯©¸U¿\¹\ÔJˆØˆw‰Øˆw‰Ø—‘—‘Ó!×+Ñ+¨BÓ/ˆØ—‘—‘Ó!×+Ñ+¨BÓ/ˆØ—‘˜Q Q¥¨Ó*×-Ñ-¨f×.?Ñ.?Ó@ˆØ�s�l˜fÕ$ uÕ,×2Ñ2°1Ó8ˆØˆr   c               ó   € V ^8„  d   QhRR/# )r   Úqtensorr   r   )r   s   "r   r   r3   v   s   € ÷ 0ñ 0¨ñ 0r   c                	ó`   € VP                   pVP                  VP                  VP                  3# r%   )Ú_paramsÚ_qdatar   r   )r6   r[   Úps   && r   Úget_plain_tensorsÚ*TensorCoreAWQW4A16Layout.get_plain_tensorsu   s$   € à�O‰OˆØ�~‰~˜qŸw™w¨¯©Ð/Ð/r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r;   r   r<   r0   r   zdict[str, torch.Tensor]r   )r   s   "r   r   r3   {   s"   € ÷ 
ñ 
 |ð 
¸Vð 
ÐH_ñ 
r   c                	ó:   € R VRVP                   RVP                  /# )Ú Ú_scaleÚ_zerosr   )r6   r;   r<   s   &&&r   Ústate_dict_tensorsÚ+TensorCoreAWQW4A16Layout.state_dict_tensorsz   s%   € ð �Ø�f—l‘lØ�f—l‘lð
ð 	
r   r   )r(   r)   r*   r+   r,   ÚMIN_SM_VERSIONÚQUANTIZES_INPUTr   r   r0   Úclassmethodr8   rX   r`   rg   r/   r   r   r   r
   r
   2   s„   † ñð €Nð €Oá�dÔôÐ!ó ó ðð& ô
ó ð
ð ôó ðð. ô0ó ð0ð ô
ó ô
r   r
   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Úinput_tensorr   Ú	weight_qtr   Úbiasztorch.Tensor | Noner   r   )r   s   "r   r   r   …   s0   € ÷ ñ Øðàðð ðð ñ	r   c           	     ó¬   € \         P                  V4      w  r4p\        \        VP                  R\
        4      4      p\        P                  ! WWEW&R7      # )z.Compute y = x @ W^T + bias via AWQ W4A16 GEMV.r   )ro   r   )r
   r`   r   Úgetattrr]   r.   ÚckÚgemv_awq_w4a16)rm   rn   ro   r;   ÚwscalesÚwzerosr   s   &&&    r   Ú_awq_forwardrv   …   sN   € ô 6×GÑGÈ	ÓRÑ€E�FÜ”W˜Y×.Ñ.°Ô>QÓRÓS€Jä×ÒØ˜WØôð r   c                ó†  € V^ ,          p\        V\        4      '       g5   \        P                  P                  P
                  P                  ! V/ VB # VP                  p\        P                  ! VVP                  ^,          VP                  ^ ,          3VP                  '       * R7      p\        VP                  RV4      # )u=   Zero-copy logical transpose â€” flip the ``transposed`` flag.)Ú
orig_shaper   r
   )Ú
isinstancer   rF   ÚopsÚatenrI   Údefaultr]   ÚdataclassesÚreplacerx   r   r^   )ÚqtÚargsr7   rm   ÚoldÚ
new_paramss   &&&   r   Ú_handle_awq_trƒ   ”   s˜   € ð ˜•7€LÜ�l¤O×4Ò4Ü�y‰y�~‰~×Ñ×'Ò'¨Ð8°Ñ8Ð8à
×
Ñ
€CÜ×$Ò$ØØ—N‘N 1Õ% s§~¡~°aÕ'8Ð9Ø—~‘~Ô%ô€Jô
 ˜<×.Ñ.Ð0JÈJÓWÐWr   c               ó    € V ^8„  d   QhRRRR/# )r   Úrhsr   r   r   )r   s   "r   r   r   ¤   s   € ÷ ñ ˜/ð ¨oñ r   c                óT   € V P                   P                  '       g   \        R4      hV # )zDReturn rhs unchanged if it is logically transposed (represents W^T).zXAWQ W4A16 GEMM expects the RHS to be W.T (stored W). Use F.linear(x, W) or mm(x, W.t()).)r]   r   ÚRuntimeError)r…   s   &r   Ú_resolve_awq_rhsrˆ   ¤   s+   € à�;‰;×!×!Ð!Üð2ó
ð 	
ð €Jr   c                óø  € V^ ,          V^,          rC\        V4      ^8”  d
   V^,          MRp\        V\        4      '       g4   \        P                  P
                  P                  ! \        W4V34      !  # \        V\        4      '       d   VP                  4       pVP                  P                  '       d9   \        P                  P
                  P                  W4P                  4       V4      # \        W4V4      # )u-   Direct F.linear(input, W, bias) â†’ AWQ GEMV.N)Úlenry   r   rF   ÚnnÚ
functionalÚlinearr   rX   r]   r   rv   )r   r€   r7   rm   Úweightro   s   &&&   r   Ú_handle_awq_linearr�   ®   s¹   € ð   �7 D¨¥G�&Ü˜$“i !”mˆ4�Ž7¨€Dä�fœo×.Ò.Ü�x‰x×"Ñ"×)Ò)¬?¸LÐRVÐ;WÓ+XÑYÐYÜ�,¤×0Ò0Ø#×.Ñ.Ó0ˆØ‡~�~× × Ð Ü�x‰x×"Ñ"×)Ñ)¨,×8IÑ8IÓ8KÈTÓRÐRÜ˜¨dÓ3Ð3r   c                ó  € V^ ,          V^,          rC\        V\        4      '       g   \        P                  ! \	        W434      !  # \        V\        4      '       d   VP                  4       p\        V4      p\        W4RR7      # )uJ   ``mm(x, W.t())`` â€” F.linear's decomposition for tensor subclass weights.N©ro   )ry   r   rF   Úmmr   rX   rˆ   rv   )r   r€   r7   ÚaÚbs   &&&  r   Ú_handle_awq_mmr•   ½   se   € ð ��7�D˜•G€qÜ�aœ×)Ò)Ü�xŠxœ¨!¨Ó0Ñ1Ð1Ü�!”_×%Ò%Ø�L‰L‹NˆÜ˜Ó€AÜ˜ 4Ô(Ð(r   c                ó  € V^ ,          V^,          V^,          rTp\        V\        4      '       g    \        P                  ! \	        W4V34      !  # \        V\        4      '       d   VP                  4       p\        V4      p\        WEVR7      # )z``addmm(bias, x, W.t())``.r‘   )ry   r   rF   Úaddmmr   rX   rˆ   rv   )r   r€   r7   ro   r“   r”   s   &&&   r   Ú_handle_awq_addmmr˜   É   so   € ð �a•˜$˜q�' 4¨¥7ˆQ€DÜ�aœ×)Ò)Ü�{Š{œO¨T°a¨LÓ9Ñ:Ð:Ü�!”_×%Ò%Ø�L‰L‹NˆÜ˜Ó€AÜ˜ 4Ô(Ð(r   )!r,   Ú
__future__r   r}   Úloggingr   rF   Úcomfy_kitchenrr   Úbaser   r   r   r   r   Ú	getLoggerr(   Úloggerr.   r
   rv   rz   r{   rI   r|   rƒ   rˆ   r�   r�   r’   r•   r—   r˜   r   r   r   Ú<module>rŸ      s  ðñõ* #ã Û Ý !ã ã ÷õ ð 
×	Ò	˜8Ó	$€àÐ ôN
˜ô N
õfñ �E—I‘I—N‘N×$Ñ$×,Ñ,Ð.FÓGñXó HðXõñ �E—I‘I—N‘N×)Ñ)×1Ñ1Ð3KÓLñ4ó Mð4ñ �E—I‘I—N‘N×%Ñ%×-Ñ-Ð/GÓHñ)ó Ið)ñ �E—I‘I—N‘N×(Ñ(×0Ñ0Ð2JÓKñ)ó Lò)r   