+
    ªåûjÏ&  ã                  óú  € R t ^ RIHt ^ RIt^ RIHt ^ RIt^ RIt^ RI	H
t
HtHt ^RIHtHtHtHtHt ]P&                  ! ]4      t ! R R]4      t]! ]P.                  P0                  P2                  P4                  ]4      R 4       tR	 R
 lt]! ]P.                  P0                  P:                  P4                  ]4      R 4       t]! ]P.                  P0                  P>                  P4                  ]4      R 4       t R# )z8NVFP4 (E2M1) block quantization layout for tensor cores.)ÚannotationsN)Ú	dataclass)ÚF4_E2M1_MAXÚF8_E4M3_MAXÚroundup)ÚBaseLayoutParamsÚQuantizedLayoutÚQuantizedTensorÚdequantize_argsÚregister_layout_opc                  óê   € ] tR t^tRtRt]! RR7       ! R R]4      4       t]	RR R ll4       t
]	R	 R
 l4       t]	R R l4       t]	R R l4       t]	R R l4       t]	R R l4       t]	R R l4       tRtR# )ÚTensorCoreNVFP4Layouta:  NVFP4 E2M1 block quantization with per-tensor and block scaling.
Auto-pads to 16x16 alignment

Note:
    Requires SM >= 10.0 (Blackwell) for hardware-accelerated matmul.
    Shape operations (view, reshape, transpose) are not supported due to
    packed format and block scales - they fall back to dequantization.
T)Úfrozenc                  óF   € ] tR t^#t$ RtR]R&   RtR]R&   R R ltR	 tR
t	R# )ÚTensorCoreNVFP4Layout.ParamszˆNVFP4 layout parameters.

Inherits scale, orig_dtype, orig_shape from BaseLayoutParams.
Adds block_scale for per-block scaling factors.
útorch.TensorÚblock_scaleFÚboolÚ
transposedc               ó   € V ^8„  d   QhRR/# )é   Úreturnz	list[str]© )Úformats   "ÚVC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\comfy_kitchen\tensor\nvfp4.pyÚ__annotate__Ú)TensorCoreNVFP4Layout.Params.__annotate__-   s   € ÷ 	,ñ 	, Iñ 	,ó    c                ó
   € RR.# )z5Override to include block_scale in tensor operations.Úscaler   r   ©Úselfs   &r   Ú_tensor_fieldsÚ+TensorCoreNVFP4Layout.Params._tensor_fields-   s   € à˜]Ð+Ð+r   c           	     	óÞ   € \        V P                  \        P                  4      '       dC   \        P                  V R V P                  P                  \        P                  RR7      4       R# R# )r   T)ÚdtypeÚnon_blockingN)Ú
isinstancer   ÚtorchÚTensorÚobjectÚ__setattr__ÚtoÚfloat32r    s   &r   Ú_validate_tensor_fieldsÚ4TensorCoreNVFP4Layout.Params._validate_tensor_fields1   sD   € Ü˜$Ÿ*™*¤e§l¡l×3Ò3Ü×"Ñ" 4¨°$·*±*·-±-ÄeÇmÁmÐbf°-Ó2gÖhñ 4r   r   N)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__r   r"   r.   Ú__static_attributes__r   r   r   ÚParamsr   #   s$   ‡ ñ	ð
 "Ó!Ø ˆ
�DÓ õ	,ö	ir   r7   Nc               ó$   € V ^8„  d   QhRRRRRR/# )r   Útensorr   r   z!torch.Tensor | float | str | Noner   ztuple[torch.Tensor, Params]r   )r   s   "r   r   Ú"TensorCoreNVFP4Layout.__annotate__6   s(   € ÷ ñ àðð 1ðð
 
%ñr   c                	ó‚  € VP                  4       ^8w  d   \        RVP                  4        R24      hVP                  p\        VP                  4      pVe   VR8X  d;   \
        P                  ! VP                  4       4      \        \        ,          ,          p\        V\
        P                  4      '       g   \
        P                  ! V4      pVP                  VP                  \
        P                  R7      pV P!                  V4      pWe8g  p\"        P$                  ! WVR7      w  r‰V P'                  VVVV	R7      p
WŠ3# )r   zNVFP4 requires 2D tensor, got ÚDÚrecalculate)Údevicer%   )Úpad_16x)r   Ú
orig_dtypeÚ
orig_shaper   )ÚdimÚ
ValueErrorr%   ÚtupleÚshaper(   ÚamaxÚabsr   r   r'   r)   r9   r,   r>   r-   Úget_padded_shapeÚckÚquantize_nvfp4r7   )Úclsr9   r   Úkwargsr@   rA   Úpadded_shapeÚneeds_paddingÚqdatar   Úparamss   &&&,       r   ÚquantizeÚTensorCoreNVFP4Layout.quantize5   sý   € ð �:‰:‹<˜1ÔÜÐ=¸f¿j¹j»l¸^È1ÐMÓNÐNà—\‘\ˆ
Ü˜6Ÿ<™<Ó(ˆ
àŠ=˜E ]Ô2Ü—J’J˜vŸz™z›|Ó,´¼kÕ0IÕJˆEä˜%¤§¡×.Ò.Ü—L’L Ó'ˆEØ—‘ §¡´U·]±]�ÓCˆà×+Ñ+¨JÓ7ˆØ$Ñ2ˆä×.Ò.¨vÀmÔTÑˆà—‘ØØ!Ø!Ø#ð	 ó 
ˆð ˆ}Ðr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   rO   r   rP   r7   r   r   )r   s   "r   r   r:   W   s&   € ÷ _ñ _˜|ð _°Vð _Àñ _r   c                	ón   € \         P                  ! WP                  VP                  VP                  4      # ©N)rI   Údequantize_nvfp4r   r   r@   ©rK   rO   rP   s   &&&r   Ú
dequantizeÚ TensorCoreNVFP4Layout.dequantizeV   s'   € ä×"Ò" 5¯,©,¸×8JÑ8JÈF×L]ÑL]Ó^Ð^r   c               ó    € V ^8„  d   QhRRRR/# )r   Úqtensorr	   r   z/tuple[torch.Tensor, torch.Tensor, torch.Tensor]r   )r   s   "r   r   r:   [   s    € ÷ Rñ RØ%ðRà	8ñRr   c                	óp   € VP                   VP                  P                  VP                  P                  3# rU   )Ú_qdataÚ_paramsr   r   )rK   r[   s   &&r   Úget_plain_tensorsÚ'TensorCoreNVFP4Layout.get_plain_tensorsZ   s)   € ð �~‰~˜wŸ™×4Ñ4°g·o±o×6QÑ6QÐQÐQr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   rO   r   rP   r7   r   zdict[str, torch.Tensor]r   )r   s   "r   r   r:   a   s"   € ÷ 
ñ 
 |ð 
¸Vð 
ÐH_ñ 
r   c                ó:   € RVRVP                   RVP                  /# )u7   Return key suffix â†’ tensor mapping for serialization.Ú Ú_scaleÚ_scale_2)r   r   rW   s   &&&r   Ústate_dict_tensorsÚ(TensorCoreNVFP4Layout.state_dict_tensors`   s'   € ð �Ø�f×(Ñ(Ø˜Ÿ™ð
ð 	
r   c               ó    € V ^8„  d   QhRRRR/# ©r   rA   útuple[int, ...]r   r   )r   s   "r   r   r:   j   s   € ÷ 6ñ 6¨/ð 6¸oñ 6r   c                	óŠ   € \        V4      ^8w  d   \        R\        V4       R24      hVw  r#\        V^4      \        V^4      3# )r   zNVFP4 requires 2D shape, got r<   )ÚlenrC   r   )rK   rA   ÚrowsÚcolss   &&  r   rH   Ú&TensorCoreNVFP4Layout.get_padded_shapei   sH   € äˆz‹?˜aÔÜÐ<¼SÀ»_Ð<MÈQÐOÓPÐPØ‰
ˆÜ˜˜bÓ!¤7¨4°Ó#4Ð5Ð5r   c               ó    € V ^8„  d   QhRRRR/# ri   r   )r   s   "r   r   r:   q   s   € ÷ +ñ +¨?ð +¸ñ +r   c                	óV   € V P                  V4      pV^ ,          V^,          ^,          3# )é    )rH   )rK   rA   Úpaddeds   && r   Úget_storage_shapeÚ'TensorCoreNVFP4Layout.get_storage_shapep   s)   € à×%Ñ% jÓ1ˆØ�q•	˜6 !�9¨�>Ð*Ð*r   c               ó    € V ^8„  d   QhRRRR/# )r   Ústorage_shaperj   r   r   )r   s   "r   r   r:   v   s   € ÷ 8ñ 8¸?ð 8Èñ 8r   c                ó4   € V^ ,          V^,          ^,          3# )zGCompute logical (padded) shape from storage shape by reversing packing.r   )rK   rw   s   &&r   Úget_logical_shape_from_storageÚ4TensorCoreNVFP4Layout.get_logical_shape_from_storageu   s   € ð ˜aÕ  -°Õ"2°QÕ"6Ð7Ð7r   r   )é
   rr   rU   )r0   r1   r2   r3   r4   ÚMIN_SM_VERSIONr   r   r7   ÚclassmethodrQ   rX   r_   rf   rH   rt   ry   r6   r   r   r   r   r      sÁ   † ñð €Ná�dÔôiÐ!ó ió ðið" öó ðð@ ô_ó ð_ð ôRó ðRð
 ô
ó ð
ð ô6ó ð6ð ô+ó ð+ð ô8ó ô8r   r   c                ó  € V^ ,          p\        V\        4      '       g5   \        P                  P                  P
                  P                  ! V/ VB # VP                  P                  pV^,          V^ ,          3p\        P                  VP                  P                  VP                  P                  VVP                  P                  VP                  P                  '       * R7      p\        VP                  RV4      # )z3Handle transpose as a logical no-op for NVFP4.
    )r   r@   rA   r   r   r   )r'   r	   r(   ÚopsÚatenÚtÚdefaultr^   rA   r   r7   r   r@   r   r   r]   )ÚqtÚargsrL   Úinput_tensorÚ	old_shapeÚ	new_shapeÚ
new_paramss   &&&    r   Ú_handle_nvfp4_transposer‰   ~   sÐ   € ð ˜•7€LÜ�l¤O×4Ò4Ü�y‰y�~‰~×Ñ×'Ò'¨Ð8°Ñ8Ð8à×$Ñ$×/Ñ/€IØ˜1•˜y¨�|Ð,€Iä&×-Ñ-Ø×"Ñ"×(Ñ(Ø×'Ñ'×2Ñ2ØØ ×(Ñ(×4Ñ4Ø#×+Ñ+×6Ñ6Ô6ð .ó €Jô ˜<×.Ñ.Ð0GÈÓTÐTr   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Úresultr   Úorig_mÚintÚorig_nr   r   )r   s   "r   r   r   •   s0   € ÷ ñ Øðàðð ðð ñ	r   c                ó„   € V P                   ^ ,          V8w  g   V P                   ^,          V8w  d   V RV1RV13,          # V # )z7Slice padded matmul output back to original dimensions.N)rE   )r‹   rŒ   rŽ   s   &&&r   Ú_slice_to_original_shaper�   •   s@   € ð ‡|�|�A…˜&Ô  F§L¡L°¥O°vÔ$=Ø�g�v�g˜w ˜wÐ&Õ'Ð'Ø€Mr   c           
     ó*  € V^ ,          V^,          rC\        V\        4      '       d   \        V\        4      '       g   \        P                  ! \	        V4      !  # VP
                  P                  4       ^8w  d   \        P                  ! \	        V4      !  # \        VP                  RR4      p\        VP                  RR4      pV'       g	   V'       g3   \        P                  R4       \        P                  ! \	        V4      !  # \        P                  V4      w  rxp	\        P                  V4      w  r«pVP                  RVP                  P                  4      p \        P                   ! VV
VVV	VVR7      pVP                  P"                  ^ ,          pVP                  P"                  ^,          p\%        WïV4      #   \&        \(        3 dA   p\        P+                  RT R24       \        P                  ! \	        T4      !  u Rp?# Rp?ii ; i)	uB  NVFP4 matrix multiplication: output = a @ b

When b is logically transposed (from a prior .t() call), this works directly
with scaled_mm_nvfp4 since that kernel computes a @ b_phys.T, which equals
a @ b_logical when b_logical = b_phys.T.

This handles the common torch.compile decomposition: linear(x, w) â†’ mm(x, w.t())
r   FzINVFP4 mm: unsupported transpose configuration, falling back to dequantizeÚ	out_dtype)Útensor_scale_aÚtensor_scale_bÚblock_scale_aÚblock_scale_br’   zNVFP4 mm failed: ú , falling back to dequantizationN)r'   r	   r(   Úmmr
   r]   rB   Úgetattrr^   ÚloggerÚdebugr   r_   Úgetr@   rI   Úscaled_mm_nvfp4rA   r�   ÚRuntimeErrorÚ	TypeErrorÚwarning)rƒ   r„   rL   ÚaÚbÚa_transposedÚb_transposedÚa_qdataÚscale_ar•   Úb_qdataÚscale_br–   r’   r‹   rŒ   rŽ   Úes   &&&               r   Ú_handle_nvfp4_mmrª       s™  € ð ��7�D˜•G€qô �qœ/×*Ò*¬z¸!¼_×/MÒ/MÜ�xŠxœ¨Ó.Ñ/Ð/ð 	‡x�x‡|�|ƒ~˜ÔÜ�xŠxœ¨Ó.Ñ/Ð/ä˜1Ÿ9™9 l°EÓ:€LÜ˜1Ÿ9™9 l°EÓ:€LçŸ<ä�‰Ð`ÔaÜ�xŠxœ¨Ó.Ñ/Ð/ä&;×&MÑ&MÈaÓ&PÑ#€G�mÜ&;×&MÑ&MÈaÓ&PÑ#€G�mØ—
‘
˜;¨¯	©	×(<Ñ(<Ó=€Ið0Ü×#Ò#ØØØ"Ø"Ø'Ø'Øô
ˆð —‘×%Ñ% aÕ(ˆØ—‘×%Ñ% aÕ(ˆÜ'¨¸Ó?Ð?øäœ)Ð$ô 0Ü�‰Ð*¨1¨#Ð-MÐNÔOÜ�xŠxœ¨Ó.Ñ/Õ/ûð0ús   ÅA"G ÇHÇ5HÈHÈHc                ó  € V^ ,          V^,          rC\        V4      ^8”  d
   V^,          MRp\        V\        4      '       d   \        V\        4      '       g4   \        P                  P
                  P                  ! \        W4V34      !  # VP                  P                  4       ^8w  d4   \        P                  P
                  P                  ! \        W4V34      !  # \        VP                  RR4      p\        VP                  RR4      pV'       g	   V'       dI   \        P                  R4       \        P                  P
                  P                  ! \        W4V34      !  # \        P                  V4      w  r‰p
\        P                  V4      w  r¼pVP!                  RVP                  P"                  4      p \$        P&                  ! VVV	VV
VVVR7      pVP                  P(                  ^ ,          pVP                  P(                  ^ ,          p\+        VVV4      #   \,        \.        3 dW   p\        P1                  RT R24       \        P                  P
                  P                  ! \        Y4T34      !  u Rp?# Rp?ii ; i)	z NVFP4 linear: output = input @ weight.T + bias

Uses ck.scaled_mm_nvfp4 for hardware-accelerated NVFP4 matmul.
Output is sliced to original (non-padded) shape.
Nr   FzMNVFP4 linear: unsupported transpose configuration, falling back to dequantizer’   )r“   r”   r•   r–   Úbiasr’   zNVFP4 scaled_mm failed: r—   )rl   r'   r	   r(   ÚnnÚ
functionalÚlinearr
   r]   rB   r™   r^   rš   r›   r   r_   rœ   r@   rI   r�   rA   r�   rž   rŸ   r    )rƒ   r„   rL   r…   Úweightr¬   Úinput_transposedÚweight_transposedÚinput_qdatar¦   r•   Úweight_qdatar¨   r–   r’   r‹   rŒ   rŽ   r©   s   &&&                r   Ú_handle_nvfp4_linearrµ   Ô   s  € ð   �7 D¨¥G�&Ü˜$“i !”mˆ4�Ž7¨€Dô �|¤_×5Ò5¼*ÀVÌ_×:]Ò:]Ü�x‰x×"Ñ"×)Ò)¬?¸LÐRVÐ;WÓ+XÑYÐYð ×Ñ×ÑÓ  AÔ%Ü�x‰x×"Ñ"×)Ò)¬?¸LÐRVÐ;WÓ+XÑYÐYä˜|×3Ñ3°\À5ÓIÐÜ §¡°¸eÓDÐß×,Ü�‰ÐdÔeÜ�x‰x×"Ñ"×)Ò)¬?¸LÐRVÐ;WÓ+XÑYÐYä*?×*QÑ*QÐR^Ó*_Ñ'€K˜-Ü+@×+RÑ+RÐSYÓ+ZÑ(€L˜=Ø—
‘
˜;¨×(<Ñ(<×(GÑ(GÓH€IðZä×#Ò#ØØØ"Ø"Ø'Ø'ØØô	
ˆð ×%Ñ%×0Ñ0°Õ3ˆØ—‘×*Ñ*¨1Õ-ˆÜ'¨°¸Ó?Ð?øäœ)Ð$ô ZÜ�‰Ð1°!°Ð4TÐUÔVÜ�x‰x×"Ñ"×)Ò)¬?¸LÐRVÐ;WÓ+XÑYÕYûðZús    Æ;A$H  È JÈ1AJÉ<JÊJ)!r4   Ú
__future__r   ÚloggingÚdataclassesr   r(   Úcomfy_kitchenrI   Úcomfy_kitchen.float_utilsr   r   r   Úbaser   r   r	   r
   r   Ú	getLoggerr0   rš   r   r   r€   r�   r‚   r‰   r�   r˜   rª   r¯   rµ   r   r   r   Ú<module>r½      så   ðÙ >Ý "ã Ý !ã ã ß GÑ G÷õ ð 
×	Ò	˜8Ó	$€ôa8˜Oô a8ñN �E—I‘I—N‘N×$Ñ$×,Ñ,Ð.CÓDñUó EðUõ,ñ �E—I‘I—N‘N×%Ñ%×-Ñ-Ð/DÓEñ00ó Fð00ñf �E—I‘I—N‘N×)Ñ)×1Ñ1Ð3HÓIñ/Zó Jò/Zr   