+
    �åûjÌ=  ã                   ó¨  € ^ RI t ^ RIHu Ht ^ RIHt ^@t^€t^t	^t
^tRtR R ltR R ltR R	 ltR
 R ltR R ltR R ltR R ltRR R lltR R R llt] P,                  P/                  RR!R7      RR R ll4       t]P2                  RR l4       t] P,                  P/                  RR!R7      R R R ll4       t]P2                  R R l4       tR# )"é    N)Úregistryg      Æ?c                ó<   € V ^8„  d   QhR\         R\         R\         /# )é   ÚaÚbÚreturn)Úint)Úformats   "ÚaC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\backends\eager\svdquant.pyÚ__annotate__r   A   s!   € ÷ ñ ”ð œð ¤ñ ó    c                 ó   € V ) V,          ) # )N© )r   r   s   &&r   Ú	_ceil_divr   A   s   € ØˆR�1�Wˆ:Ðr   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r   Úvaluesr   ©ÚtorchÚTensor)r
   s   "r   r   r   E   s"   € ÷ +ñ +¤§¡ð +´%·,±,ñ +r   c                óŠ  € V P                   R,          ^,          ^ 8w  d    \        RV P                   R,           24      hV R,          P                  \        P                  4      ^,          pV R,          P                  \        P                  4      ^,          pW^,          ,          P                  \        P
                  4      # )a>  Pack (..., K) int4 values into (..., K // 2) int8 (low = even column).

Storage-level codec: handles the full 4-bit field; the caller decides which
quantizer emission range to use. Inputs may contain any value that fits in
a nibble (signed [-8, 7] or unsigned [0, 15]); values outside get masked
via the & 0x0F below.
zlast dim must be even, got éÿÿÿÿ).:r   Nr   ).:é   Nr   )ÚshapeÚ
ValueErrorÚtor   Úint32Úint8)r   ÚloÚhis   &  r   Ú_pack_int4_row_majorr    E   sŽ   € ð ‡|�|�BÕ˜!Õ˜qÔ ÜÐ6°v·|±|ÀBÕ7GÐ6HÐIÓJÐJØ	�	Õ	×	Ñ	œeŸk™kÓ	*¨TÕ	1€BØ	�	Õ	×	Ñ	œeŸk™kÓ	*¨TÕ	1€BØ˜•'�N×ÑœuŸz™zÓ*Ð*r   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# ©r   Úpackedr   r   )r
   s   "r   r   r   T   s&   € ÷ Bñ B¤5§<¡<ð B´E·L±Lñ Br   c                óª  € V P                  \        P                  4      pV^,          pV^,	          ^,          p\        P                  ! V^8¬  V^,
          V4      p\        P                  ! V^8¬  V^,
          V4      p\        P                  ! W#.RR7      pVP
                  ! . V P                  RR ORN5!  P                  \        P                  4      # )u   Inverse of _pack_int4_row_major with signed-nibble interpretation.

Storage-level codec: returns int8 across the full signed nibble range
[-8, 7]. This is wider than the quantizer's emission range [-7, 7] by
design â€” the codec must accept any bit pattern that could land in the
nibble.
©ÚdimNr   )r   r   r   ÚwhereÚstackÚreshaper   r   ©r#   Úx32r   r   Ústackeds   &    r   Ú_unpack_int4_row_majorr-   T   s¤   € ð �)‰)”E—K‘KÓ
 €CØ	ˆt�€BØ
��(�dÕ	€BÜ	�Š�R˜1‘W˜b 2�g rÓ	*€BÜ	�Š�R˜1‘W˜b 2�g rÓ	*€BÜ�kŠk˜2˜(¨Ô+€GØ�?Š?Ð2˜FŸL™L¨¨"Ð-Ð2¨rÓ2×5Ñ5´e·j±jÓAÐAr   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# r"   r   )r
   s   "r   r   r   e   s&   € ÷ 
Bñ 
B¤E§L¡Lð 
B´U·\±\ñ 
Br   c                ó"  € V P                  \        P                  4      pV^,          pV^,	          ^,          p\        P                  ! W#.RR7      pVP                  ! . V P
                  RR ORN5!  P                  \        P                  4      # )uØ   Inverse of _pack_int4_row_major with unsigned-nibble interpretation.

Storage-level codec: returns int8 across the full unsigned nibble range
[0, 15] (used by the u4.s4 MMA path â€” post-GELU+shift fc2 activations).
r%   Nr   )r   r   r   r(   r)   r   r   r*   s   &    r   Ú_unpack_uint4_row_majorr0   e   sp   € ð �)‰)”E—K‘KÓ
 €CØ	ˆt�€BØ
��(�dÕ	€BÜ�kŠk˜2˜(¨Ô+€GØ�?Š?Ð2˜FŸL™L¨¨"Ð-Ð2¨rÓ2×5Ñ5´e·j±jÓAÐAr   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# r"   r   )r
   s   "r   r   r   r   s"   € ÷ ñ ¬U¯\©\ð ¼e¿l¹lñ r   c                óÎ  € V P                  4       ^8w  d   V # V P                  w  rr4V\        \        ,          8w  d   \	        RV 24      hV\        \
        ^,          ,          8w  d   \	        RV 24      hV P                  WV\        \
        ^,          4      P                  ^ ^^^^4      P                  4       P                  V\        ,          V\
        ^,          ,          4      # )z=Convert kitchen_tile_packed_w4a4 weight to natural (N, K//2).z unexpected tile-packed N quads: z"unexpected tile-packed byte axis: )	r&   r   Ú_TILE_PACKED_BLOCK_NÚ_TILE_PACKED_INTERLEAVEr   Ú_INT4_GROUP_SIZEÚviewÚpermuteÚ
contiguous)r#   Ún_tilesÚk_groupsÚn_quadsÚbytes_per_quads   &    r   Ú _tile_packed_weight_to_row_majorr=   r   sÆ   € à‡z�zƒ|�qÔØˆØ17·±Ñ.€G�wØÔ&Ô*AÕAÔAÜÐ;¸G¸9ÐEÓFÐFØÔ0Ô4DÈÕ4IÕJÔJÜÐ=¸nÐ=MÐNÓOÐOØ�;‰;Ø˜7Ô$;Ô=MÐQRÕ=Róç�gˆa��A�q˜!ÓŸZ™Z›\¯$©$ØÔ&Õ&¨Ô4DÈÕ4IÕ(Jó+ðr   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r   Úwscalesr   r   )r
   s   "r   r   r   ‚   s"   € ÷ ñ ¬E¯L©Lð ¼U¿\¹\ñ r   c                óì   € V P                  4       ^8w  d   V # V P                  ^^ ^4      P                  4       P                  V P                  ^,          V P                  ^ ,          \
        ,          4      # )zBConvert tile-packed wscales (N/128, K/G, 128) to natural (K/G, N).©r&   r7   r8   r6   r   r3   )r?   s   &r   Ú_tile_packed_scales_to_naturalrB   ‚   s[   € à‡{�{ƒ}˜ÔØˆØ�?‰?˜1˜a Ó#×.Ñ.Ó0×5Ñ5Ø�‰�aÕ˜'Ÿ-™-¨Õ*Ô-AÕAóð r   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r   Úlora_upr   r   )r
   s   "r   r   r   ‹   s"   € ÷ ñ ¬U¯\©\ð ¼e¿l¹lñ r   c                óì   € V P                  4       ^8w  d   V # V P                  ^ ^^4      P                  4       P                  V P                  ^ ,          \
        ,          V P                  ^,          4      # )z>Convert tile-packed proj_up (N/128, R, 128) to natural (N, R).rA   )rD   s   &r   Ú_tile_packed_lora_up_to_naturalrF   ‹   s[   € à‡{�{ƒ}˜ÔØˆØ�?‰?˜1˜a Ó#×.Ñ.Ó0×5Ñ5Ø�‰�aÕÔ/Õ/°·±¸qÕ1Aóð r   c                ó2  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R\         P                  R,          R\        \         P                  \         P                  \         P                  3,          /# ©	r   ÚxÚsmoothÚ	lora_downÚpad_sizeÚact_unsignedÚlora_xNr   ©r   r   r	   ÚboolÚtuple)r
   s   "r   r   r   ”   s„   € ÷ D'ñ D'Ü‡|�|ðD'ä�L‰LðD'ô �|‰|ðD'ô ð	D'ô
 ðD'ô �L‰L˜4ÕðD'ô Œ5�<‰<œŸ™¤u§|¡|Ð3Õ4ñD'r   c                ó†  € V P                  4       ^8w  d"   \        R\        V P                  4       24      hV P                  w  rgVP                  ^,           \        pWx,          ^ 8w  d   \        RV RV 24      h\        Wc4      V,          p	Ve   TMT p
V
P                  4       VP                  4       ,          pW,          pVP                  WgV,          V4      pVP                  4       P                  RR7      P                  RR7      pV'       d   \        M\        pV'       d   ^ M\        ) pWï,          pVVP                  R4      ,          P                  4       P                  VV4      P                  \         P"                  4      pVP%                  Wg4      p\'        V4      pW–8”  dZ   W–,
          p\(        P*                  ! V^ ^ ^ V34      p\(        P*                  ! V^ ^ ^ V34      p\(        P*                  ! V^ ^ ^ V34      pVP-                  4       P/                  4       P                  V P0                  4      pVVV3# )u  Quantize activations to int4 with smoothing + separate LoRA down projection.

Args:
    x: (M, K) bf16/fp16 â€” main-path input (may be shifted by caller for unsigned).
    smooth: (K,) per-channel smoothing factor applied before quantization.
    lora_down: (K, R) low-rank down projection weight. The eager reference
        here runs the matmul in fp32 for numerical stability; the CUDA
        backend takes a lower-precision bf16 matmul path for memory/launch
        savings. Do not use this eager result as a bitwise oracle for the
        CUDA output â€” it is a high-precision reference, not a backend parity
        target.
    pad_size: pad M to a multiple of this (default 256) to match downstream kernels.
    act_unsigned: if True, quantize into uint4 [0, 15] (scale=max/15) instead of
        signed int4 [-7, 7] (scale=max/7). Selects MMA grid for downstream u4.s4.
    lora_x: (M, K) bf16/fp16 â€” input for LoRA matmul. Defaults to x. Pass
        separately when caller pre-shifts x (SVDQuant LoRA is mathematically
        defined on pre-quantization, pre-shift activations).

Returns:
    q_x: (M_pad, K // 2) int8 packed (2 int4 per byte, same layout as weight).
    ascales: (K // 64, M_pad) same dtype as x â€” per-row per-group scales.
    lora_act: (M_pad, R) fp32 LoRA activations.
zexpected 2D input, got shape zK=z not divisible by group_size=r%   g»½×Ùß|Û=)Úminr   )r&   r   rQ   r   r5   r   Úfloatr6   ÚabsÚamaxÚclampÚ
_UINT4_MAXÚ	_INT4_MAXÚ	unsqueezeÚroundr   r   r   r)   r    ÚFÚpadÚtr8   Údtype)rI   rJ   rK   rL   rM   rN   ÚmÚkÚgroupÚm_padÚlora_srcÚlora_actÚx_smoothÚgroupsÚabsmaxÚqmaxÚqminÚscalesÚq_valsÚq_packedr]   Úascaless   &&&&&&                r   Úquantize_svdquant_w4a4ro   ”   s×  € ð> 	‡u�uƒw�!„|ÜÐ8¼¸q¿w¹w»Ð8HÐIÓJÐJØ�7‰7�D€AØ‡O�O�AÖÜ€EØ…y�A„~Ü˜2˜a˜SÐ =¸e¸WÐEÓFÐFÜ�aÓ" XÕ-€Eð  Ò+‰v°€HØ�~‰~Ó )§/¡/Ó"3Õ3€Hð
 �z€HØ�]‰]˜1 5�j¨%Ó0€FØ�Z‰Z‹\×Ñ 2ÐÓ&×,Ñ,°Ð,Ó7€F÷
 &�:¬9€Dß‰1¤9 *€DØ�]€FØ�v×'Ñ'¨Ó+Õ+×2Ñ2Ó4×:Ñ:¸4ÀÓF×IÑIÌ%Ï*É*ÓU€FØ�^‰^˜AÓ!€FÜ# FÓ+€Hà„yØ�iˆÜ—5’5˜ A q¨!¨S >Ó2ˆÜ—’�v  1 a¨˜~Ó.ˆÜ—5’5˜ A q¨!¨S >Ó2ˆà�h‰h‹j×#Ñ#Ó%×(Ñ(¨¯©Ó1€GØ�W˜hÐ&Ð&r   c                ó2  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\        R
\         P                  /	# ©r   ÚactÚwgtrn   r?   Úlora_act_inrD   ÚbiasNrM   r   ©r   r   rP   )r
   s   "r   r   r   Û   s’   € ÷ Gñ GÜ	�‰ðGä	�‰ðGô �\‰\ðGô �\‰\ð	Gô
 —‘ðGô �\‰\ðGô �,‰,˜Õ
ðGô ðGô ‡\�\ñGr   c                ó€  € \        V4      p\        V4      p\        V4      pV P                  w  r‰VP                  ^ ,          p
V	^,          p\        pVP
                  p\        V4      P                  V4      pVP                  W«V,          V4      pVP                  4       P                  R4      pVV,          P                  W«4      pV'       d   \        V 4      M
\        V 4      pVP                  V4      P                  W‹V,          V4      pVP                  4       P                  R4      pVV,          P                  W‹4      pVVP                  4       ,          pVP                  4       VP                  4       P                  4       ,          pVVP                  VP
                  4      ,           pVe
   VV,           pV# )ué  SVDQuant W4A4 int4 GEMM + LoRA up + optional bias (eager reference impl).

Semantic mirror of the CUDA path; used as a high-precision reference and on
non-CUDA devices. Numerical notes:

* Quantization grid and emission range match the CUDA kernel exactly
  (signed absmax/7, unsigned absmax/15 â€” see module header).
* The LoRA-up branch here accumulates in fp32 for stability. The CUDA
  wrapper takes a bf16 addmm_ path for memory/launch-count savings and
  drops the fp32 lora_act_in precision. This eager output is therefore a
  high-precision reference, not a bit-parity oracle for the CUDA backend.
  Tests that compare the two paths should use tolerances consistent with
  bf16 matmul (~8e-3 abs).

Args:
    act: (M, K // 2) int8 packed activations from quantize_svdquant_w4a4.
        Bit-pattern interpretation depends on act_unsigned (signed [-7,7]
        vs unsigned [0,15]).
    wgt: (N, K // 2) int8 packed weights (kitchen natural row-major) or
        (N/128, K/64, 32, 128) kitchen_tile_packed_w4a4.
    ascales: (K // 64, M) per-row per-group activation scales.
    wscales: (K // 64, N) natural or (N/128, K/64, 128) tile-packed.
    lora_act_in: (M, R) fp32 LoRA down-projection activations.
    lora_up: (N, R) natural or (N/128, R, 128) tile-packed.
    bias: (N,) bias or None.
    act_unsigned: if True, interpret packed activations as unsigned [0, 15]
        (matches the u4.s4 MMA path used for post-GELU+shift fc2 layers).

Returns:
    out: (M, N) in the dtype of wscales/lora_up.
r   )r=   rB   rF   r   r5   r_   r-   r   r6   r^   rZ   r0   rT   )rr   rs   rn   r?   rt   rD   ru   rM   r`   Úk_halfÚnra   rb   Úcompute_dtypeÚwgt_intÚwgt_gÚwscales_bngÚwgt_fpÚact_intÚascales_mngÚact_fpÚoutÚlora_contributions   &&&&&&&&               r   Úscaled_mm_svdquant_w4a4r„   Û   sm  € ôR +¨3Ó
/€CÜ,¨WÓ5€GÜ-¨gÓ6€Gà—	‘	�I€AØ�	‰	�!�€AØ��
€AÜ€EØ—M‘M€Mô % SÓ)×,Ñ,¨]Ó;€GØ�L‰L˜ �J¨Ó.€EØ—)‘)“+×'Ñ'¨Ó+€KØ�kÕ!×'Ñ'¨Ó-€F÷ /;Ô% cÔ*Ô@VÐWZÓ@[€GØ�j‰j˜Ó'×,Ñ,¨Q°Uµ
¸EÓB€GØ—)‘)“+×'Ñ'¨Ó+€KØ˜Õ#×)Ñ)¨!Ó/€Fà
�6—8‘8“:Õ
€Cð $×)Ñ)Ó+¨g¯m©m«o×.?Ñ.?Ó.AÕAÐØ
Ð!×$Ñ$ S§Y¡YÓ/Õ
/€CàÒØ�D�jˆØ€Jr   z%comfy_kitchen::quantize_svdquant_w4a4)Úmutates_argsc                ó2  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        R\        R\         P                  R,          R\        \         P                  \         P                  \         P                  3,          /# rH   rO   )r
   s   "r   r   r   /  s{   € ÷ ñ Ü‡|�|ðä�L‰Lðô �|‰|ðô ð	ô
 ðô �L‰L˜4Õðô Œ5�<‰<œŸ™¤u§|¡|Ð3Õ4ñr   c                 ó^   € R V RVRVRVRVRV/p\         P                  ! RVR7      pV! R/ VB # )	rI   rJ   rK   rL   rM   rN   ro   ©Úkwargsr   ©r   Úget_implementation)rI   rJ   rK   rL   rM   rN   r‰   Úimpls   &&&&&&  r   Ú_op_quantize_svdquant_w4a4r�   .  sJ   € ð 	ˆQØ�&Ø�YØ�HØ˜Ø�&ð€Fô ×&Ò&Ð'?ÈÔO€DÙ‰>�&‰>Ðr   c                 ó²  € V P                   w  rgVP                   ^,          p\        Wc4      V,          p	\        P                  ! W—^,          \        P                  V P
                  R7      p
\        P                  ! V\        ,          W�P                  V P
                  R7      p\        P                  ! W˜\        P                  V P
                  R7      pW«V3# )r   ©r_   Údevice)	r   r   r   Úemptyr   r�   r5   r_   Úfloat32)rI   rJ   rK   rL   rM   rN   r`   ra   Úrrc   Úq_xrn   re   s   &&&&&&       r   Ú_op_quantize_svdquant_w4a4_faker•   C  s�   € ð �7‰7�D€AØ�‰˜Õ€AÜ�aÓ" XÕ-€EÜ
�+Š+�e !�V¬5¯:©:¸a¿h¹hÔ
G€CÜ�kŠk˜!Ô/Õ/°¿g¹gÈaÏhÉhÔW€GÜ�{Š{˜5¬5¯=©=ÀÇÁÔJ€HØ˜Ð!Ð!r   z&comfy_kitchen::scaled_mm_svdquant_w4a4c                ó2  € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R	\        R
\         P                  /	# rq   rv   )r
   s   "r   r   r   Q  s‡   € ÷ ñ Ü	�‰ðä	�‰ðô �\‰\ðô �\‰\ð	ô
 —‘ðô �\‰\ðô �,‰,˜Õ
ðô ðô ‡\�\ñr   c                 óf   € R V RVRVRVRVRVRVRV/p\         P                  ! RVR	7      p	V	! R
/ VB # )rr   rs   rn   r?   rt   rD   ru   rM   r„   rˆ   r   rŠ   )
rr   rs   rn   r?   rt   rD   ru   rM   r‰   rŒ   s
   &&&&&&&&  r   Ú_op_scaled_mm_svdquant_w4a4r˜   P  sO   € ð 	ˆs�E˜3 	¨7°I¸wØ�{ I¨wØ��n lð€Fô
 ×&Ò&Ð'@ÈÔP€DÙ‰>�&‰>Ðr   c                 ó  € V P                   ^ ,          pVP                  4       ^8X  d   VP                   ^ ,          \        ,          MVP                   ^ ,          p	\        P                  ! W‰VP
                  V P                  R7      # )r   r�   )r   r&   r3   r   r‘   r_   r�   )
rr   rs   rn   r?   rt   rD   ru   rM   r`   ry   s
   &&&&&&&&  r   Ú _op_scaled_mm_svdquant_w4a4_fakerš   d  sU   € ð 	�	‰	�!�€AØ/2¯w©w«y¸A¬~ˆ�	‰	�!�Ô+Ö+À3Ç9Á9ÈQÅ<€AÜ�;Š;�q 7§=¡=¸¿¹ÔDÐDr   )é   FN)NFr   )r   Útorch.nn.functionalÚnnÚ
functionalr\   Úcomfy_kitchen.registryr   r5   r3   r4   rY   rX   Ú_GELU_UNSIGNED_SHIFTr   r    r-   r0   r=   rB   rF   ro   r„   ÚlibraryÚ	custom_opr�   Úregister_faker•   r˜   rš   r   r   r   Ú<module>r¤      sö   ðó^ ß Ð å +àÐ ØÐ ØÐ ð €	Ø€
ØÐ õõ+õBõ"
Bõõ õ÷D'÷NGðf ‡�×ÑÐ@ÈrÐÓRöó Sðð( ×)Ñ)ó	"ó *ð	"ð ‡�×ÑÐAÐPRÐÓSöó Tðð& ×*Ñ*óEó +òEr   