+
    ÑåûjÊŠ  ã                   ó6  € ^ RI t ^ RIHt ^ RIHt ^ RIHt ^ RIt^ RIH	t	H
t
 ^ RIHtHtHtHtHtHtHtHtHtHtHtHtHtHt . ROt]! R
R4      t ! R R]P8                  4      tR]P<                  3R R llt ! R R]4      t  ! R R	]4      t!Rs"R t#Rs$R t%R# )é    N)Ú
namedtuple)ÚCallable©ÚAny)Ú)sparse_semi_structured_from_dense_cutlassÚ'sparse_semi_structured_to_dense_cutlass)Ú_FP8_E4M3_DTYPESÚfallback_dispatcherÚsemi_sparse_addmmÚsemi_sparse_cloneÚsemi_sparse_detachÚsemi_sparse_indicesÚsemi_sparse_linearÚsemi_sparse_mmÚsemi_sparse_scaled_mmÚsemi_sparse_tÚsemi_sparse_toÚsemi_sparse_to_copyÚsemi_sparse_valuesÚsemi_sparse_viewÚSparseSemiStructuredTensorÚ!SparseSemiStructuredTensorCUTLASSÚ$SparseSemiStructuredTensorCUSPARSELTÚ_SEMI_STRUCTURED_SPARSE_CONFIGz=sparse_min_rows sparse_min_cols dense_min_rows dense_min_colsc                   ón  a € ] tR t^+t o Rt^ tRtRtRt. ROt	]
RV 3R lR ll4       tV 3R lR ltV 3R lR lt]V 3R	 lR
 l4       t]P"                  P$                  t]V 3R lR l4       t]RV 3R lR ll4       t]V 3R lR l4       tR t]]3V 3R lR ll4       tRR/V 3R lR lltV 3R ltRtV tR# )r   a�  
This class implements semi-structured sparsity as a Tensor subclass.

Semi-structured sparsity describes a sparsity pattern where n in every 2n elements are sparse,
depending on the datatype. It is also referred to as 2:4 sparsity or fine-grained
structured sparsity.

There are two backends available for semi_structured sparsity, either cuSPARSELt or CUTLASS.
This class is meant to serve as a base class for both implementations. SparseSemiStructuredCUTLASS
and SparseSemiStructuredCUSPARSELT both inherit from this class and define three backend-specific items.
Note that as such, this class cannot be instantiated directly.

-`_DTYPE_SHAPE_CONSTRAINTS` - A dictionary holding backend specific dense/sparse min shape constraints
- `def from_dense()` - backend specific compression routines
- `def _mm()` - backend specific mm op (either torch._cslt_sparse_mm or torch._sparse_semi_structured_(mm|addmm))
Fc                ó  <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R	S[R
S[/	# )é   ÚshapeÚpackedNÚmetaÚpacked_tÚmeta_tÚcompressed_swizzled_bitmaskÚfuse_transpose_cusparseltÚalg_id_cusparseltÚrequires_grad)ÚtorchÚSizeÚTensorÚboolÚint)ÚformatÚ__classdict__s   "€ÚXC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\torch\sparse\semi_structured.pyÚ__annotate__Ú'SparseSemiStructuredTensor.__annotate__Q   s¡   ø€ ÷ Rñ Rá�z‰zðRñ —‘˜tÕ#ðRñ �l‰l˜TÕ!ð	Rñ
 —,‘, Õ%ðRñ —‘˜tÕ#ðRñ &+§\¡\°DÕ%8ðRñ $(ðRñ ðRñ ñRó    c
           	     óÜ  € V P                   '       gT   \        P                  ! R\        ^R7       RV n         V P	                  4        \
        P                  P                  V 4       Ve   Tp
MVe   Tp
M\        R4      h\
        P                  P                  V VV
P                  V
P                  V
P                  V	R7      pW+n        W;n        WKn        W[n        Wkn        W{n        W‹n        V# )a“  
Create a new instance of the tensor subclass from the compressed sparse representation.

We have the option to create the subclass with the compressed representations of both X and X', for training.
For inference, we only need a single representation (either X or X'), while the corresponding other set will be None.

Depending on the backend selected, certain fields will be set to None. (CUSPARSELT vs CUTLASS)

Args:
    shape: The shape of the original dense tensor
    packed: The compressed representation of the original dense tensor
    meta: The metadata of the original dense tensor, if it is stored separately
    packed_t: The compressed representation of the transposed original dense tensor
    meta_t: The metadata of the transposed original dense tensor, if it is stored separately
    compressed_swizzled_bitmask: The masks used by the CUTLASS backend to determine which threads should
                                 participate in the computation. Used for pointwise ops.
    fuse_transpose_cusparselt: When running with cuSPARSELt, we have the option to fuse a transposition
                               with a matmul, which is useful in the case of 2:4 sparse training.
    alg_id_cusparselt: The algorithm id to use when using cuSPARSELT, will have an effect on performance

Returns:
    torch.Tensor: A torch.Tensor wrapper subclass.

Raises:
    ValueError: If all of the tensor arguments are None.
zøThe PyTorch API of SparseSemiStructuredTensor is in prototype stage and will change in the near future. Please open a Github issue for features requests and see our documentation on the torch.sparse module for further information about the project.©Ú
stacklevelTz3At least one of packed or packed_t must be provided)ÚdeviceÚdtypeÚlayoutr&   )Ú_PROTOTYPE_WARNING_SHOWNÚwarningsÚwarnÚUserWarningÚ_load_dispatch_tabler'   Ú_dynamoÚallow_in_graphÚ
ValueErrorr)   Ú_make_wrapper_subclassr5   r6   r7   r   r    r!   r"   r#   r$   r%   )Úclsr   r   r    r!   r"   r#   r$   r%   r&   Úprevious_tensorÚtensors   &&&&&&&&&&  r.   Ú__new__Ú"SparseSemiStructuredTensor.__new__P   sã   € ðN ×+×+Ð+Ü�MŠMðHô
 Øõ	ð ,0ˆCÔ(ð
 ×$Ñ$Ô&ô �M‰M×(Ñ(¨Ô-àÒØ$‰OØÒ!Ø&‰OäÐRÓSÐSä—‘×4Ñ4ØØØ"×)Ñ)Ø!×'Ñ'Ø"×)Ñ)Ø'ð 5ó 
ˆð ŒØŒØ"ŒØŒØ-HÔ*Ø+DÔ(Ø#4Ô Øˆr1   c                ó    <€ V ^8„  d   QhRS[ /# ©r   Úreturn)Ústr)r,   r-   s   "€r.   r/   r0   ¥   s   ø€ ÷ @ñ @™#ñ @r1   c                óˆ   € \        V R 4      '       g   \        R4      hV P                  P                   RV P                   R2# )r   ztensor has no shape attributez(shape=Ú))ÚhasattrÚAssertionErrorÚ	__class__Ú__name__r   )Úselfs   &r.   Ú__repr__Ú#SparseSemiStructuredTensor.__repr__¥   s=   € Ü�t˜W×%Ò%Ü Ð!@ÓAÐAØ—.‘.×)Ñ)Ð*¨'°$·*±*°¸QÐ?Ð?r1   c                óx   <€ V ^8„  d   QhRS[ S[S[,          S[ S[P                  S[S[S[3,          3,          /# rG   )ÚtupleÚlistrI   r'   r(   r*   r+   )r,   r-   s   "€r.   r/   r0   ª   s5   ø€ ÷ *ñ *á	‰t‘C�y™%¡§
¡
©D±#±tÐ ;Õ<Ð<Õ	=ñ*r1   c                ó¬   a € \        \        V 3R  lS P                  4      4      pS P                  S P                  S P
                  S P                  3pW3# )c                 ó    <€ \        SV 4      R J# ©N)Úgetattr)ÚxrP   s   &€r.   Ú<lambda>Ú?SparseSemiStructuredTensor.__tensor_flatten__.<locals>.<lambda>®   s   ø€ œW T¨1Ó-°TÑ9r1   )rU   ÚfilterÚ	__slots__r   r$   r%   r&   )rP   Úinner_tensorsÚtensor_metas   f  r.   Ú__tensor_flatten__Ú-SparseSemiStructuredTensor.__tensor_flatten__ª   sT   ø€ ô ÜÔ9¸4¿>¹>ÓJó
ˆð �J‰JØ×*Ñ*Ø×"Ñ"Ø×Ñð	
ˆð Ð)Ð)r1   c                ól   <€ V ^8„  d   QhRS[ S[P                  S[S[S[3,          RS[P
                  /# )r   r`   rH   )rT   r'   r(   r*   r+   r)   )r,   r-   s   "€r.   r/   r0   ¹   s7   ø€ ÷ 
ñ 
ñ ™5Ÿ:™:¡t©S±$Ð6Õ7ð
ñ 
�‰ñ
r1   c                óÎ   € Vw  rVrxV ! VVP                  R R4      VP                  RR4      VP                  RR4      VP                  RR4      VP                  RR4      VVVR7	      # )r   Nr    r!   r"   r#   ©	r   r   r    r!   r"   r#   r$   r%   r&   )Úget)	rA   r_   r`   Ú
outer_sizeÚouter_strider   r$   r%   r&   s	   &&&&&    r.   Ú__tensor_unflatten__Ú/SparseSemiStructuredTensor.__tensor_unflatten__¸   s€   € ð NYÑJˆÐ*;áØØ ×$Ñ$ X¨tÓ4Ø×"Ñ" 6¨4Ó0Ø"×&Ñ& z°4Ó8Ø ×$Ñ$ X¨tÓ4Ø(5×(9Ñ(9Ø-¨tó)ð '@Ø/Ø'ô
ð 	
r1   c                ó    <€ V ^8„  d   QhRS[ /# rG   r   )r,   r-   s   "€r.   r/   r0   Ó   s   ø€ ÷ Tñ T¹cñ Tr1   c                óÊ   € VP                   V P                  9  d&   \        V P                   R VP                   R24      hV P                  VP                   ,          ! WW44      # )zI only supports a specific set of operations, can't perform requested op (rK   )Ú_overloadpacketÚSPARSE_DISPATCHÚNotImplementedErrorrO   )rA   ÚfuncÚtypesÚargsÚkwargss   &&&&&r.   Ú__torch_dispatch__Ú-SparseSemiStructuredTensor.__torch_dispatch__Ò   sb   € à×Ñ s×':Ñ':Ô:Ü%Ø—<‘<�.ð !/Ø/3¯}©}¨o¸Qð@óð ð ×"Ñ" 4×#7Ñ#7Ö8¸ÀdÓSÐSr1   Nc                ó   <€ V ^8„  d   QhRR/# )r   rH   N© )r,   r-   s   "€r.   r/   r0   Ü   s   ø€ ÷ Bñ BÀñ Br1   c                ó*  € \        V RR4      Efƒ   \        P                  P                  P                  \
        \        P                  P                  P                  \        \        P                  P                  P                  \        \        P                  P                  P                  \        \        P                  P                  P                  \        \        P                  P                  P                  \        \        P                  P                  P                  \         \        P                  P                  P"                  \$        \        P                  P                  P&                  \$        \        P                  P                  P(                  \*        \        P                  P                  P,                  \.        \        P                  P                  P0                  \2        \        P                  P                  P4                  \6        \        P                  P                  P8                  \:        \        P                  P                  P<                  \>        /V n         Ve   V P@                  PC                  V4       R# R# R# )zD
Loads the op overload sparse dispatch table for the current class.
rn   N)"rY   r'   ÚopsÚatenÚvaluesr   Úindicesr   Úis_same_sizer
   Údetach_Údetachr   Útr   Úviewr   Úmmr   ÚmatmulÚaddmmr   Úlinearr   Ú_to_copyr   Ú
_scaled_mmr   Úcloner   Útor   rn   Úupdate)rA   Úcustom_dispatch_tables   &&r.   r<   Ú/SparseSemiStructuredTensor._load_dispatch_tableÛ   sm  € ô
 �3Ð)¨4Ó0Ó8ä—	‘	—‘×%Ñ%Ô'9Ü—	‘	—‘×&Ñ&Ô(;Ü—	‘	—‘×+Ñ+Ô-@Ü—	‘	—‘×&Ñ&Ô(;Ü—	‘	—‘×%Ñ%Ô'9Ü—	‘	—‘× Ñ ¤-Ü—	‘	—‘×#Ñ#Ô%5Ü—	‘	—‘×!Ñ!¤>Ü—	‘	—‘×%Ñ%¤~Ü—	‘	—‘×$Ñ$Ô&7Ü—	‘	—‘×%Ñ%Ô'9Ü—	‘	—‘×'Ñ'Ô)<Ü—	‘	—‘×)Ñ)Ô+@Ü—	‘	—‘×$Ñ$Ô&7Ü—	‘	—‘×!Ñ!¤>ð#ˆCÔð" %Ò0Ø×#Ñ#×*Ñ*Ð+@ÖAñ 1ñ% 9r1   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# )r   Úoriginal_tensorrH   N©r'   r)   )r,   r-   s   "€r.   r/   r0   ö   s    ø€ ÷ (ñ (¹u¿|¹|ð (ÐPTñ (r1   c           	     ó¶  € VP                   '       g   \        RVP                   R24      hVP                  4       ^8w  d   \        RVP                  4        R24      hVP	                  4       '       g   \        R4      hVP
                  V P                  9  d   \        RVP
                   RV  R24      hVP                  w  r#V P                  VP
                  ,          P                  pV P                  VP
                  ,          P                  pW$8  g#   W$,          '       g   W58  g   W5,          '       d    \        R	VP                   R
V RV R24      hR# )zO
Assert that the given tensor is valid for semi-structured sparse compression.
zError original_tensor.device= z= is not supported! Only CUDA tensors are currently supported.zError original_tensor.dim = z; is not supported! Only 2d tensors are currently supported.zXError original_tensor is not contiguous!Only contiguous tensors are currently supported.zError original_tensor.dtype z is not a supported dtype for Ú!zError original_tensor.shape zV is not supported! Both dimensions must be larger than or equal to and a multiple of (z, rK   N)
Úis_cudaÚRuntimeErrorr5   ÚdimÚis_contiguousr6   Ú_DTYPE_SHAPE_CONSTRAINTSr   Úsparse_min_rowsÚsparse_min_cols)rA   rŽ   ÚmÚnÚmin_rowsÚmin_colss   &&    r.   Ú _validate_device_dim_dtype_shapeÚ;SparseSemiStructuredTensor._validate_device_dim_dtype_shapeõ   sr  € ð ×&×&Ð&ÜØ0°×1GÑ1GÐ0Hð I=ð =óð ð ×ÑÓ  AÔ%ÜØ.¨×/BÑ/BÓ/DÐ.Eð F;ð ;óð ð ×,Ñ,×.Ò.ÜðCóð ð × Ñ ¨×(DÑ(DÔDÜØ.¨×/DÑ/DÐ.EÐEcÐdgÐchÐhiÐjóð ð
 ×$Ñ$‰ˆØ×/Ñ/°×0EÑ0EÕF×VÑVˆØ×/Ñ/°×0EÑ0EÕF×VÑVˆØŒ<˜1Ÿ<œ<¨1¬<¸1¿<¼<äØ.¨×/DÑ/DÐ.Eð FVØV^ÐU_Ð_aÐbjÐakÐklðnóð ñ <Hr1   c           	     óª   € V P                   R,          p\        P                  ! V \        P                  ! WP                  V P
                  R7      4      # )é   ©r6   r5   éÿÿÿÿ)r   r'   r‚   Úeyer6   r5   )rP   Úcols   & r.   Úto_denseÚ#SparseSemiStructuredTensor.to_dense   s3   € Ø�j‰j˜�nˆÜ�xŠx˜œeŸiši¨·:±:ÀdÇkÁkÔRÓSÐSr1   c                ó>   <€ V ^8„  d   QhRS[ P                  RS[RR/# )r   rŽ   Úalg_idrH   r   ©r'   r)   r+   )r,   r-   s   "€r.   r/   r0   %  s-   ø€ ÷ "ñ "áŸ™ð"ñ ð"ð 
&ñ	"r1   c                ó   € \         hrX   ©ro   ©rA   rŽ   r¨   s   &&&r.   Ú
from_denseÚ%SparseSemiStructuredTensor.from_dense$  s
   € ô "Ð!r1   Úbiasc                óv   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  /# )r   ÚBr¯   NrH   r�   )r,   r-   s   "€r.   r/   r0   ,  s:   ø€ ÷ "ñ "á�<‰<ð"ñ �l‰l˜TÕ!ð	"ñ 
�‰ñ"r1   c               ó   € \         hrX   r«   )rP   r±   r¯   rs   s   &&$,r.   Ú_mmÚSparseSemiStructuredTensor._mm,  s
   € ô "Ð!r1   c                ó¬  <€ V ^8„  d   Qh/ S[ ;R&   S[S[P                  S[3,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[S[3,          ;R&   S[P                  R,          ;R	&   S[P                  R,          ;R
&   S[P                  R,          ;R&   S[P                  R,          ;R&   S[P                  R,          ;R&   S[;R&   S[ ;R&   # )r   Ú_DEFAULT_ALG_IDr–   Ú_FORCE_CUTLASSÚ_FUSE_TRANSPOSEr8   ÚBACKENDrn   Nr   r    r!   r"   r#   r$   r%   )	r+   Údictr'   r6   r   r*   rI   r   r)   )r,   r-   s   "€r.   r/   r0   +   s  ø‡ ‚ ñ$ Ññ% ñ& #¡5§;¡;Ñ0NÐ#NÕOÑOñ' ñ( Ñ ñ) ñ* Ñ!ñ+ ñ, #Ñ*ñ- ñ0 �Lñ1 ñ2 ™(¡HÐ,Õ-Ñ-ñ3 ñ6 �L‰L˜4ÕÑñ7 ñ8 �,‰,˜Õ
Ññ9 ñ: �l‰l˜TÕ!Ñ!ñ; ñ< �L‰L˜4ÕÑñ= ñ> "'§¡°Õ!4Ñ4ñ? ñ@  $Ñ#ñA ñB ÑòC r1   rw   )r   r    r!   r"   r#   )Fr   FrX   )rO   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r¶   r·   r¸   r8   r^   ÚstaticmethodrD   rQ   ra   Úclassmethodri   r'   Ú_CÚ_disabled_torch_function_implÚ__torch_function__rt   r<   r�   r¥   r­   r³   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r-   s   @r.   r   r   +   s  ø‡ € ñð" €Oà €NØ!€OØ%*Ðò X€Ià÷Rñ Ró ðR÷h@ð @÷
*ð *ð ÷
ó ð
ð. Ÿ™×?Ñ?Ðà÷Tó ðTð ÷Bñ Bó ðBð2 ÷(ó ð(òTTð ð &÷"ñ "ó ð"ð"ð %)÷	"ò "÷C ƒ r1   Fc                ó\   € V ^8„  d   QhR\         P                  R\        R\        R\        /# )r   rŽ   Ú
transposedr¨   rH   )r'   r)   r*   r+   r   )r,   s   "r.   r/   r/   6  s@   € ÷ DFñ DFÜ—\‘\ðDFäðDFô ðDFô  ñ	DFr1   c                ó  € V'       d   \         P                  ! R\        ^R7       \        P                  '       d   \
        P                  P                  M\
        P                  P                  pVP                  WR7      # )a	  
This function converts a dense tensor into a sparse semi-structured tensor.
It will return a SparseSemiStructuredTensor, a subclass of torch.Tensor.

This function will check to ensure the dense tensor has the right dtype, size, dims, and device.
We currently only support semi-structured sparse tensors for 2d CUDA tensors.
Additionally, your tensor must be a positive multiple of the minimum sparse block size, given in
`_DTYPE_TO_SHAPE_CONSTRAINTS` for each dtype (float32, float16, bfloat16, int8).

Args:
    original_tensor (Tensor): the dense tensor to convert
    transposed (bool, optional): deprecated arg to be removed in another release. Do not use.
    alg_id (int, optional): the algorithm id to use for cuSPARSELt matmul. Defaults to 0.
        Can be obtained via ``torch._cslt_sparse_mm_search``.
Returns:
    SparseSemiStructuredTensor: A sparse semi-structured tensor created from the given original_tensor
Raises:
    None
Example:
    >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA)
    >>> A = torch.Tensor([0, 0, 1, 1]).tile((128, 32)).half().cuda()
    tensor([[0., 0., 1.,  ..., 0., 1., 1.],
            [0., 0., 1.,  ..., 0., 1., 1.],
            [0., 0., 1.,  ..., 0., 1., 1.],
            ...,
            [0., 0., 1.,  ..., 0., 1., 1.],
            [0., 0., 1.,  ..., 0., 1., 1.],
            [0., 0., 1.,  ..., 0., 1., 1.]], device='cuda:0', dtype=torch.float16)
    >>> A_sparse = to_sparse_semi_structured(A)
    SparseSemiStructuredTensor(shape=torch.Size([128, 128]))
    >>> A_sparse.values()
    tensor([[1., 1., 1.,  ..., 1., 1., 1.],
            [1., 1., 1.,  ..., 1., 1., 1.],
            [1., 1., 1.,  ..., 1., 1., 1.],
            ...,
            [1., 1., 1.,  ..., 1., 1., 1.],
            [1., 1., 1.,  ..., 1., 1., 1.],
            [1., 1., 1.,  ..., 1., 1., 1.]], device='cuda:0', dtype=torch.float16),
    >>> A_sparse.indices()
    tensor([[-4370, -4370, -4370,  ..., -4370, -4370, -4370],
            [-4370, -4370, -4370,  ..., -4370, -4370, -4370],
            [-4370, -4370, -4370,  ..., -4370, -4370, -4370],
            ...,
            [-4370, -4370, -4370,  ..., -4370, -4370, -4370],
            [-4370, -4370, -4370,  ..., -4370, -4370, -4370],
            [-4370, -4370, -4370,  ..., -4370, -4370, -4370]], device='cuda:0', dtype=torch.int16))
z­Setting transpose from `to_sparse_semi_structured` is deprecated and will be removed in a future release. `SparseSemiStructuredTensor` only support contiguous input tensors.r3   )r¨   )
r9   r:   ÚFutureWarningr   r·   r'   Úsparser   r   r­   )rŽ   rÉ   r¨   ÚSPARSE_SUBCLASSs   &&& r.   Úto_sparse_semi_structuredrÎ   6  sg   € ÷h Ü�ŠðRô Øõ	
ô &×4×4Ð4ô 	�‰×6Ò6ä�\‰\×>Ñ>ð ð ×%Ñ% oÐ%ÓEÐEr1   c                   óZ  a a€ ] tR tRt oRtRt]P                  ]! ^^€^^4      ]P                  ]! ^ ^@^^4      ]P                  ]! ^ ^@^^4      ]P                  ]! ^ ^ ^^4      /t]]P                  3V3R lR ll4       tV 3R lt]RV3R lR ll4       tR	R
RR/V3R lR lltRtVtV ;t# )r   i}  aŒ  
This class implements semi-structured sparsity for the CUTLASS backend.


In this implementation, the specified elements and metadata are stored separately,
in packed and meta respectively.

When _FORCE_CUTLASS is set, or when cuSPARSELt is not available, this subclass calls into _sparse_semi_structured_(mm|addmm) and
sparse_semi_structured_from_dense for conversion to the compressed format.
Úcutlassc                ó>   <€ V ^8„  d   QhRS[ P                  RS[RR/# )r   rŽ   r¨   rH   r   r©   )r,   r-   s   "€r.   r/   Ú.SparseSemiStructuredTensorCUTLASS.__annotate__’  s-   ø€ ÷ 
ñ 
áŸ™ð
ñ ð
ð 
-ñ	
r1   c           
     ó†   € V P                  V4       \        V4      w  ppV ! VP                  VVR R R VP                  R7      # )N©r   r    r!   r"   r#   r&   )r�   r   r   r&   )rA   rŽ   r¨   Úsparse_tensor_cutlassÚmeta_tensor_cutlasss   &&&  r.   r­   Ú,SparseSemiStructuredTensorCUTLASS.from_dense‘  sW   € ð 	×,Ñ,¨_Ô=ô 6°oÓFñ	
Ø!Øñ Ø×!Ñ!Ø(Ø$ØØØ(,Ø)×7Ñ7ô
ð 	
r1   c                óä   <€ V P                   e   V P                  f   \        R4      hV P                   P                  ^8X  d!   \	        V P                  V P                   4      # \
        SV `  4       # )Nz meta and packed must not be None)r    r   rM   Úndimr   Úsuperr¥   )rP   rN   s   &€r.   r¥   Ú*SparseSemiStructuredTensorCUTLASS.to_dense§  sc   ø€ Ø�9‰9Ò §¡Ò 3Ü Ð!CÓDÐDð �y‰y�~‰~ Ô"ô	 4Ø—‘Ø—	‘	óð	
ô ‘Ñ!Ó#ð	
r1   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# ©r   rŽ   rH   r   r�   )r,   r-   s   "€r.   r/   rÒ   ´  s%   ø€ ÷ K
ñ K
Ù#Ÿl™lðK
à	%ñK
r1   c           
     óp   € \         P                  ! WRR7      w  pppppV ! VP                  VVVVVRR7      # )a¼  
This function takes in a unpruned dense tensor and runs a (branchless) static sort across a 4x4 tile.

It greedily picks the largest values in the tile, upholding the 2:4 sparsity constraint across both rows and columns.
The algorithm used to prune the matrix is implemented in `_sparse_semi_structured_tile`.

Then it creates the packed and meta tensors for the compressed sparse representation of the pruned dense tensor.
It also calculates the packed_t and meta_t tensors for the compressed sparse representation of the transposed
pruned dense tensor.
Since we cannot transpose the compressed representations, we store both for the fw/bw pass respectively.

Finally, this function also computes a compressed swizzled bitmask that encodes the sparsity pattern.
This can be used in the backward pass to mask the gradients.

::

    [9 1 7 4]                       [9 0 7 0]
    [1 2 3 0]                       [0 2 0 0]
    [8 3 5 4] -> prune 4x4 tile  -> [8 0 0 4] -> pack to CUTLASS semi-structured -> packed
    [1 2 6 2]                       [0 0 6 2]                                    -> metadata

                                              -> pack to transposed CUTLASS      -> packed_t
                                                 semi-structured representation  -> metadata_t

                                              -> compute swizzled bitmask        -> compressed_swizzled_bitmask

The equivalent PyTorch code to create the same five outputs from the dense tensor can be found below::

    from torch.sparse import SparseSemiStructuredTensorCUTLASS
    from torch.sparse._semi_structured_conversions import (
        _sparse_semi_structured_tile,
        _compute_compressed_swizzled_bitmask,
    )

    pruned = _sparse_semi_structured_tile(dense)
    packed_cutlass, meta_cutlass = sparse_semi_structured_from_dense_cutlass(
        pruned
    )
    packed_t_cutlass, meta_t_cutlass = (
        sparse_semi_structured_from_dense_cutlass(pruned.t().contiguous())
    )
    bitmask = _compute_compressed_swizzled_bitmask(pruned)

    SparseSemiStructuredTensorCUTLASS(
        dense.shape,
        packed_cutlass,
        meta_cutlass,
        packed_t_cutlass,
        meta_t_cutlass,
        bitmask,
    )
T©Ú	algorithmÚuse_cutlassFrÔ   )r'   Ú_sparse_semi_structured_tiler   ©rA   rŽ   rà   r   r    r!   r"   r#   s   &&&     r.   Úprune_dense_static_sortÚ9SparseSemiStructuredTensorCUTLASS.prune_dense_static_sort³  sV   € ô~ ×.Ò.Ø¸dô
ñ	
ØØØØØ'ñ Ø×!Ñ!ØØØØØ(CØô
ð 	
r1   r¯   NÚshould_transpose_denseFc                ó|   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[RS[ P                  /# ©r   r±   r¯   Nræ   rH   ©r'   r)   r*   )r,   r-   s   "€r.   r/   rÒ     sD   ø€ ÷ !ñ !á�<‰<ð!ñ �l‰l˜TÕ!ð	!ñ
 !%ð!ñ 
�‰ñ!r1   c          
     óH  € \        V\        4      '       d   \        R 4      hV P                  P                  pV P
                  ^8w  g   VP
                  ^8w  d   \        RV R24      hV P                  e   V P                  f   \        RV R24      h\        4        V P                  VP                  ,          p\        P                  P                  P                  VV P                  V P                  VV P                   ^ ,          VP"                  VP$                  V4      # )úZ`SparseSemiStructuredTensor @ SparseSemiStructuredTensor` is not supported by the hardwareÚ`ú)` matmul: Broadcasting is not implementedú$` matmul: operation is not supported)Ú
isinstancer   r?   rN   rO   rÙ   ro   r   r    Ú_ensure_cutlass_mm_registeredr–   r6   r'   ry   Úsemi_structuredÚ
cutlass_mmr   Údense_min_rowsÚdense_min_cols)rP   r±   r¯   ræ   rs   Úcls_nameÚconstraintss   &&$$,  r.   r³   Ú%SparseSemiStructuredTensorCUTLASS._mm  sþ   € ô �aÔ3×4Ò4ÜØlóð ð —>‘>×*Ñ*ˆØ�9‰9˜Œ>˜QŸV™V qœ[Ü%Ø�H�:ÐFÐGóð ð �;‰;Ò $§)¡)Ò"3Ü%Ø�H�:ÐAÐBóð ô *Ô+Ø×7Ñ7¸¿¹Õ@ˆKÜ—9‘9×,Ñ,×7Ñ7ØØ—‘Ø—	‘	ØØ—
‘
˜1•Ø×*Ñ*Ø×*Ñ*Ø&ó	ð 	r1   rw   ©Ú )rO   r»   r¼   r½   r¾   r¹   r'   Úint8r   Úfloat16Úbfloat16Úfloat32r–   rÀ   r   r¶   r­   r¥   rä   r³   rÅ   rÆ   Ú__classcell__)rN   r-   s   @@r.   r   r   }  sÖ   ù‡ € ñ	ð €Gà�
‰
Ñ2°2°s¸BÀÓCØ�‰Ñ5°b¸"¸aÀÓCØ�‰Ñ6°r¸2¸qÀ!ÓDØ�‰Ñ5°b¸"¸aÀÓCð	 Ðð ð 1×@Ñ@÷
ñ 
ó ð
õ*

ð ÷K
ñ K
ó ðK
ðZ!ð %)ð	!ð
 (-÷!÷ !ò !r1   c                   ór  a € ] tR tRt o RtRt]P                  ]! ^ ^ ^^4      ]P                  ]! ^ ^ ^^4      ]P                  ]! ^ ^ ^^4      ]P                  ]! ^^^^4      ]P                  ]! ^^^^4      /t]]P                   3V 3R lR ll4       t]RV 3R lR ll4       tRR	R
R/V 3R lR lltRtV tR	# )r   i%  ab  
The cuSPARSELt backend expects the specified elements and the metadata to be stored in a single tensor:
packed = [ specified elements of original tensor | metadata ]
For an original tensor of size (m, k) we expect the first m * k // 2 elements to be the kept elements
The rest of the tensor is metadata. Since there is only one tensor, we only use the packed and packed_t
attributes respectively.

cuSPARSELt also supports transposition fusion, which is necessary for performant 2:4 sparse training, as well
as specifying alg_id, a config that affects the performance of the matmul depending on matmul sizes.
Ú
cusparseltc                ó>   <€ V ^8„  d   QhRS[ P                  RS[RR/# )r   rŽ   r¨   rH   r   r©   )r,   r-   s   "€r.   r/   Ú1SparseSemiStructuredTensorCUSPARSELT.__annotate__;  s-   ø€ ÷ 
ñ 
áŸ™ð
ñ ð
ð 
0ñ	
r1   c                ó²   € V P                  V4       V ! VP                  \        P                  ! V4      R R R R \        P
                  VVP                  R7	      # )Nre   )r�   r   r'   Ú_cslt_compressr   r¸   r&   r¬   s   &&&r.   r­   Ú/SparseSemiStructuredTensorCUSPARSELT.from_dense:  sW   € ð 	×,Ñ,¨_Ô=áØ!×'Ñ'Ü×'Ò'¨Ó8ØØØØ(,Ü&@×&PÑ&PØ$Ø)×7Ñ7ô

ð 
	
r1   c                ó8   <€ V ^8„  d   QhRS[ P                  RR/# rÝ   r�   )r,   r-   s   "€r.   r/   r  O  s!   ø€ ÷ >
ñ >
Ù#Ÿl™lð>
à	%ñ>
r1   c           
     óü   € \         P                  ! WRR7      w  pppppVP                  VP                  ^ ,          R4      pVP                  VP                  ^,          R4      pV ! VP                  VVVVVRR7      # )a¶  
This function does the same thing as described in SparseSemiStructuredCUTLASS, but uses the cuSPARSELt metadata
layout and sparse matmul.

The only functional difference is that cuSPARSELt stores `metadata` and `packed` together into a single tensor.

::

    [9 1 7 4]                       [9 0 7 0]
    [1 2 3 0]                       [0 2 0 0]
    [8 3 5 4] -> prune 4x4 tile  -> [8 0 0 4] -> pack to cuSPARSELT semi-structured -> packed
    [1 2 6 2]                       [0 0 6 2]

                                              -> pack to transposed cuSPARSELt      -> packed_t
                                                 semi-structured representation

                                              -> compute swizzled bitmask           -> compressed_swizzled_bitmask

The equivalent PyTorch code to create the same three outputs from the dense tensor can be found below::

    from torch.sparse import SparseSemiStructuredTensorCUSPARSELT
    from torch.sparse._semi_structured_conversions import (
        _sparse_semi_structured_tile,
        _compute_compressed_swizzled_bitmask,
    )

    pruned = _sparse_semi_structured_tile(dense)
    packed_cusparselt = torch._cslt_compress(pruned)
    packed_t_cusparselt = torch._cslt_compress(pruned.t().contiguous())
    bitmask = _compute_compressed_swizzled_bitmask(pruned)

    SparseSemiStructuredTensorCUSPARSELT(
        dense.shape, packed_cutlass, None, packed_t_cutlass, None, bitmask
    )
Frß   rÔ   r¢   )r'   râ   r�   r   rã   s   &&&     r.   rä   Ú<SparseSemiStructuredTensorCUSPARSELT.prune_dense_static_sortN  sŽ   € ôZ ×.Ò.Ø¸eô
ñ	
ØØØØØ'ð —‘˜_×2Ñ2°1Õ5°rÓ:ˆØ—=‘= ×!6Ñ!6°qÕ!9¸2Ó>ˆñ Ø×!Ñ!ØØØØØ(CØô
ð 	
r1   r¯   Nræ   Fc                ó|   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[RS[ P                  /# rè   ré   )r,   r-   s   "€r.   r/   r  �  sD   ø€ ÷ 4ñ 4á�<‰<ð4ñ �l‰l˜TÕ!ð	4ñ
 !%ð4ñ 
�‰ñ4r1   c               óL  € \        V\        4      '       d   \        R 4      hV P                  ^8w  g   VP                  ^8w  d$   \	        RV P
                  P                   R24      hVP                  V P                  8w  dj   \	        RV P
                  P                   R\        V P                  4       R\        VP                  4       RV P                   RVP                   R24      hVe…   VP                  V P                  8w  dj   \	        RV P
                  P                   R\        V P                  4       R\        VP                  4       RV P                   R	VP                   R
24      hV P                  \        9   d]   \	        RV P
                  P                   R\        V P                  4       R\        VP                  4       RV P                   R2	4      hV P                  f$   \	        RV P
                  P                   R24      h\        4        V P                  VP                  ,          p\        P                  P                   P#                  VV P                  VV P                  ^ ,          VP$                  VP&                  VV P(                  V4	      # )rë   rì   rí   z` matmul: trying to do `A=z @ B=z`, with A.dtype=z and B.dtype=zH. This operation is only supported when A and B have the same data type.z + C`, with A.dtype=B.dtype=z and C.dtype=zK. This operation is only supported when A, B and C have the same data type.z`, with A.dtype=B.dtype=z_. mm is not supported for float8_e4m3fn/float8_e4m3fnuz, please use `torch._scaled_mm` instead.rî   )rï   r   r?   rÙ   ro   rN   rO   r6   rT   r   r	   r   Ú _ensure_cusparselt_mm_registeredr–   r'   ry   rñ   Úcusparselt_mmró   rô   r%   )rP   r±   r¯   ræ   rs   rö   s   &&$$, r.   r³   Ú(SparseSemiStructuredTensorCUSPARSELT._mm�  so  € ô �aÔ3×4Ò4ÜØlóð ð �9‰9˜Œ>˜QŸV™V qœ[Ü%Ø�D—N‘N×+Ñ+Ð,Ð,UÐVóð ð �7‰7�d—j‘jÔ Ü%Ø�D—N‘N×+Ñ+Ð,Ð,FÄuÈTÏZÉZÓGXÐFYÐY^Ô_dÐef×elÑelÓ_mÐ^nð o Ø $§
¡
˜|¨=¸¿¹¸	ð BYðYóð ð
 Ò §
¡
¨d¯j©jÔ 8Ü%Ø�D—N‘N×+Ñ+Ð,Ð,FÄuÈTÏZÉZÓGXÐFYÐY^Ô_dÐef×elÑelÓ_mÐ^nð o(Ø(,¯
©
 |°=ÀÇÁÀ	ð J\ð\óð ð �:‰:Ô)Ô)Ü%Ø�D—N‘N×+Ñ+Ð,Ð,FÄuÈTÏZÉZÓGXÐFYÐY^Ô_dÐef×elÑelÓ_mÐ^nð o(Ø(,¯
©
 |ð 4pðpóð ð
 �;‰;ÒÜ%Ø�D—N‘N×+Ñ+Ð,Ð,PÐQóð ô -Ô.Ø×7Ñ7¸¿¹Õ@ˆKÜ—9‘9×,Ñ,×:Ñ:ØØ—‘ØØ—
‘
˜1•Ø×*Ñ*Ø×*Ñ*Ø&Ø×&Ñ&Ø&ó
ð 
r1   rw   rø   )rO   r»   r¼   r½   r¾   r¹   r'   Úfloat8_e4m3fnr   Úfloat8_e4m3fnuzrú   rû   rü   r–   rÀ   r   r¶   r­   rä   r³   rÅ   rÆ   rÇ   s   @r.   r   r   %  sç   ø‡ € ñ	ð €Gà×ÑÑ;¸BÀÀBÈÓKØ×ÑÑ=¸bÀ"ÀbÈ"ÓMØ�
‰
Ñ2°2°r¸2¸rÓBØ�‰Ñ5°b¸"¸aÀÓCØ�‰Ñ6°r¸2¸qÀ!ÓDð Ðð ð 1×@Ñ@÷
ñ 
ó ð
ð& ÷>
ñ >
ó ð>
ð@4ð %)ð	4ð
 (-÷4÷ 4ð 4r1   c                 óÌ   aa€ \         '       d   R# Rs ^ RIHp  R R loV ! RRR7      ! S4      pVP                  R R	 l4       p^ R
IHoHp VV3R lpVP                  W44       R# )z¸Lazily register the cutlass_mm custom op.

Registration is deferred to avoid importing torch.library at module load
time, since torch.sparse is imported early during ``import torch``.
NT©Ú	custom_opc                óö   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R\        R	\        R
\         P                  /	# )r   Údenser   r    r¯   NÚout_featuresr›   rœ   ræ   rH   ©r'   r)   r+   r*   )r,   s   "r.   r/   Ú3_ensure_cutlass_mm_registered.<locals>.__annotate__Ö  s†   € ÷ Gñ GÜ�|‰|ðGä—‘ðGô �l‰lðGô �l‰l˜TÕ!ð	Gô
 ðGô ðGô ðGô !%ðGô 
�‰ñGr1   c                 ó„  € V P                   w  r‰V) V,          p
V	) V,          pV
^ 8g  ;'       g    V^ 8g  pT pV'       d/   \        P                  P                  P	                  V ^ V^ V
34      pV'       d   VP                  4       MTpVf   \        P                  ! WV4      pM\        P                  ! W1W.4      pV'       dY   V'       d   TMT	pVRV P                  ^^ V4      pV'       d   VP                  4       P                  4       # VP                  4       # V'       d   VP                  4       P                  4       # T# )r   N)
r   r'   ÚnnÚ
functionalÚpadr€   Ú_sparse_semi_structured_mmÚ_sparse_semi_structured_addmmÚnarrowÚ
contiguous)r  r   r    r¯   r  r›   rœ   ræ   r™   rš   Úto_pad_mÚto_pad_nÚneed_padÚdense_paddedÚmm_inputÚresÚout_colss   &&&&&&&&         r.   Ú_cutlass_mm_implÚ7_ensure_cutlass_mm_registered.<locals>._cutlass_mm_implÖ  s  € ð �{‰{‰ˆØ�B˜(•?ˆØ�B˜(•?ˆØ˜q‘=×1Ð1 H°¡MˆØˆßÜ Ÿ8™8×.Ñ.×2Ñ2°5¸1¸hÈÈ8Ð:TÓUˆLß'=�<—>‘>Ô#À<ˆØŠ<Ü×2Ò2°6ÀÓJ‰Cä×5Ò5°dÀDÓSˆCßß2‘q¸ˆHØ�m�|Ð$×+Ñ+¨A¨q°(Ó;ˆCß+A�3—5‘5“7×%Ñ%Ó'ÐWÀsÇ~Á~ÓGWÐWß'=ˆs�u‰u‹w×!Ñ!Ó#ÐFÀ3ÐFr1   zsemi_structured::cutlass_mm©Úmutates_argsc                óö   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R,          R\        R\        R\        R	\        R
\         P                  /	# )r   r  r   r    r¯   Nr  r›   rœ   Útranspose_denserH   r  )r,   s   "r.   r/   r  ÷  s{   € ÷ 
ñ 
Ü�|‰|ð
ä—‘ð
ô �l‰lð
ô �l‰l˜TÕ!ð	
ô
 ð
ô ð
ô ð
ô ð
ô 
�‰ñ
r1   c                 óÔ   € V'       d   V P                   ^ ,          MV P                   ^,          pV'       d   W„3MWH3p	\        P                  ! V	V P                  V P                  R7      # ©r   r¡   ©r   r'   Úemptyr6   r5   )
r  r   r    r¯   r  r›   rœ   r,  r&  Úoutput_shapes
   &&&&&&&&  r.   Ú_cutlass_mm_fakeÚ7_ensure_cutlass_mm_registered.<locals>._cutlass_mm_fakeö  sS   € ÷ &5�5—;‘;˜q–>¸%¿+¹+Àa½.ˆç(7ˆXÑ$¸lÐ=Uð 	ô �{Š{ØØ—+‘+Ø—<‘<ô
ð 	
r1   )Ú_has_unrecognized_tensor_typesÚFunctionalTensorModec                 óŽ   <€ S! V4      '       d   \         # V ;_uu_ 4        S! V/ VB uuR R R 4       #   + '       g   i     R # ; irX   )ÚNotImplemented)ÚmodeÚ_oprq   rr   rs   r'  r4  s   &&&&&€€r.   Ú_cutlass_mm_decompÚ9_ensure_cutlass_mm_registered.<locals>._cutlass_mm_decomp  s2   ø€ Ù)¨%×0Ò0Ü!Ð!ßŠTÙ# TÐ4¨VÑ4÷ �T�T‹Tús	    3³A	rw   )Ú_cutlass_mm_registeredÚtorch.libraryr  Úregister_fakeÚ#torch._subclasses.functional_tensorr4  r5  Úregister_torch_dispatch)r  rò   r2  r5  r:  r'  r4  s        @@r.   rð   rð   É  si   ù€ ÷ ÓÙØ!Ðå'õGñ8 Ð8ÀrÕJØó€Jð ×Ñô
ó ð
÷(ö5ð ×&Ñ&Ð';ÖPr1   c                 ó†   € \         '       d   R# Rs ^ RIHp  V ! RR
R7      RR R ll4       pVP                  R R	 l4       pR# )z,Lazily register the cusparselt_mm custom op.NTr  zsemi_structured::cusparselt_mmr)  c                óî   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\        R\        R\        R\        R	\        R
\        R\         P                  /
# ©r   r  r   r¯   Nr  r›   rœ   Úfuse_transposer¨   ræ   rH   r  )r,   s   "r.   r/   Ú6_ensure_cusparselt_mm_registered.<locals>.__annotate__+  s�   € ÷  ñ  Ü�|‰|ð ä—‘ð ô �l‰l˜TÕ!ð ô ð	 ô
 ð ô ð ô ð ô ð ô !%ð ô 
�‰ñ r1   c	                 ó  € V P                   w  ršV	) V,          pV
) V,          pV^ 8g  ;'       g    V^ 8g  pT pV'       d/   \        P                  P                  P	                  V ^ V^ V34      pV'       d   VP                  4       MTp\        P                  ! VVVVVR7      pV'       dW   V'       d   T	MT
pV'       d"   VP                  ^ ^ V4      P                  4       # VP                  ^^ V4      P                  4       # V# )r   )r¯   Útranspose_resultr¨   )	r   r'   r  r  r  r€   Ú_cslt_sparse_mmr  r  )r  r   r¯   r  r›   rœ   rD  r¨   ræ   r™   rš   r   r!  r"  r#  r$  r%  r&  s   &&&&&&&&&         r.   r  Ú7_ensure_cusparselt_mm_registered.<locals>.cusparselt_mm*  sä   € ð �{‰{‰ˆØ�B˜(•?ˆØ�B˜(•?ˆØ˜q‘=×1Ð1 H°¡MˆØˆßÜ Ÿ8™8×.Ñ.×2Ñ2°5¸1¸hÈÈ8Ð:TÓUˆLß'=�<—>‘>Ô#À<ˆÜ×#Ò#ØØØØ+Øô
ˆ÷ ß2‘q¸ˆHßØ—z‘z ! Q¨Ó1×<Ñ<Ó>Ð>à—z‘z ! Q¨Ó1×<Ñ<Ó>Ð>Øˆ
r1   c                óî   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R,          R\        R\        R\        R\        R	\        R
\        R\         P                  /
# rC  r  )r,   s   "r.   r/   rE  N  s�   € ÷ 
ñ 
Ü�|‰|ð
ä—‘ð
ô �l‰l˜TÕ!ð
ô ð	
ô
 ð
ô ð
ô ð
ô ð
ô !%ð
ô 
�‰ñ
r1   c	                 óÔ   € V'       d   V P                   ^ ,          MV P                   ^,          p	V'       d   W“3MW93p
\        P                  ! V
V P                  V P                  R7      # r.  r/  )r  r   r¯   r  r›   rœ   rD  r¨   ræ   r&  r1  s   &&&&&&&&&  r.   Ú_cusparselt_mm_fakeÚ=_ensure_cusparselt_mm_registered.<locals>._cusparselt_mm_fakeM  sS   € ÷ &<�5—;‘;˜q–>ÀÇÁÈQÅˆç(6ˆXÑ$¸\Ð<Tð 	ô �{Š{ØØ—+‘+Ø—<‘<ô
ð 	
r1   rw   )F)Ú_cusparselt_mm_registeredr=  r  r>  )r  r  rL  s      r.   r  r  !  sL   € ÷ !Ó ÙØ $Ðå'áÐ/¸bÔAö ó Bð ðD × Ñ ô
ó !ò
r1   )r   r   r   rÎ   )&r9   Úcollectionsr   Úcollections.abcr   Útypingr   r'   Ú)torch.sparse._semi_structured_conversionsr   r   Ú!torch.sparse._semi_structured_opsr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   r)   r   r¶   rÎ   r   r   r<  rð   rN  r  rw   r1   r.   Ú<module>rU     s®   ðã Ý "Ý $Ý ã ÷÷÷ ÷ ÷ ò$€ñ ",Ø$ØCó"Ð ôH" §¡ô H"ðZ Ø,×<Ñ<÷DFôNeÐ(Bô eôP^Ð+Eô ^ðB Ð òRQðj "Ð ô@
r1   