+
    ƒ°ñj¡   ã                   ó„   € ^ RI HtHt ^ RIt^ RItR t ! R R4      tR R ltRR R	 llt	RR
 R llt
R R ltR R ltR# )é    )ÚOptionalÚTupleNc           	     ó
  € . p\        V 4       FM  p\        V4       F;  p	\        V4       F)  p
VP                  Wƒ,          W”,          W¥,          34       K+  	  K=  	  KO  	  \        P                  ! Wv\        P                  R7      # )zDKernel spatial offsets in the same order as the CUDA/Triton kernels.©ÚdeviceÚdtype)ÚrangeÚappendÚtorchÚtensorÚint32)ÚKwÚKhÚKdÚDwÚDhÚDdr   ÚoffsetsÚvxÚvyÚvzs   &&&&&&&    Ú?C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\trellis2\flexgemm.pyÚcompute_kernel_offsetsr      sa   € à€GÜ�BŽiˆÜ˜–)ˆBÜ˜B–i�Ø—‘ ¥¨­°"µ'Ð:Ö;ó  ó ñ ô �<Š<˜´e·k±kÔBÐBó    c                   óL   a € ] tR t^t o RtV 3R lR ltRtV 3R lR ltRtV t	R# )	ÚTorchHashMapz2Sorted-array hashmap backed by torch.searchsorted.c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )é   ÚkeysÚvalues©r   ÚTensor)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚTorchHashMap.__annotate__   s#   ø€ ÷ +ñ +™UŸ\™\ð +±5·<±<ñ +r   c                óÔ   € \         P                  ! VP                  \         P                  4      4      w  V n        pW#,          V n        V P                  P                  4       V n        R # )N)r   ÚsortÚtoÚlongÚsorted_keysÚsorted_valsÚnumelÚ_n)Úselfr   r    Úorders   &&& r   Ú__init__ÚTorchHashMap.__init__   sE   € Ü"'§*¢*¨T¯W©W´U·Z±ZÓ-@Ó"AÑˆÔ˜%Ø!�=ˆÔØ×"Ñ"×(Ñ(Ó*ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Ú	flat_keysÚreturnr!   )r#   r$   s   "€r   r%   r&      s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r   c                ó:  € VP                   ^ ,          p\        P                  ! V3RVP                  \        P                  R7      pV P
                  ^ 8X  g   V^ 8X  d   V# \        ^ W P                  4       EF  p\        W@P                  ,           V4      pWV P                  \        P                  4      p\        P                  ! V P                  V4      pWpP
                  8  pVP                  V P
                  ^,
          R7       W€P                  V,          V8H  ,          p	V	P                  4       '       g   KÇ  V	P                  RR7      ^ ,          p
V P                   Wz,          ,          P                  \        P                  4      W4V
,           &   EK"  	  V# )r   r   )ÚmaxT)Úas_tupleéÿÿÿÿ)Úshaper   Úfullr   r   r.   r	   Ú_LOOKUP_CHUNKÚminr)   r*   Úsearchsortedr+   Úclamp_ÚanyÚnonzeror,   )r/   r4   ÚNÚoutÚsÚeÚ
flat_chunkÚidxÚin_rangeÚfoundÚ	found_idxs   &&         r   Úlookup_flatÚTorchHashMap.lookup_flat   s*  € Ø�O‰O˜AÕˆÜ�jŠj˜!˜˜r¨)×*:Ñ*:Ä%Ç+Á+ÔNˆØ�7‰7�aŒ<˜1 œ6ØˆJÜ�q˜!×/Ñ/×0ˆAÜ�A×*Ñ*Õ*¨AÓ.ˆAØ" Q˜×*Ñ*¬5¯:©:Ó6ˆJÜ×$Ò$ T×%5Ñ%5°zÓBˆCØŸW™W‘}ˆHØ�J‰J˜4Ÿ7™7 Q�;ˆJÔ'Ø× 0Ñ 0°Õ 5¸Ñ CÕDˆEØ�y‰y�{Œ{Ø!ŸM™M°4˜MÓ8¸Õ;�	Ø%)×%5Ñ%5°cµnÕ%E×%HÑ%HÌÏÉÓ%U�˜	•MÔ"ñ 1ð ˆ
r   )r.   r+   r,   Ni  € )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r1   r<   rK   Ú__static_attributes__Ú__classdictcell__)r$   s   @r   r   r      s#   ø‡ € Ù<÷+ð +ð €M÷ö r   r   c                ó8   € V ^8„  d   QhR\         P                  /# )r   Úcoordsr!   )r#   s   "r   r%   r%   .   s   € ÷ ñ ä�L‰Lñr   c                 ó  € VP                   pVP                  ^ ,          p\        WVWxWšV4      pVP                  ^ ,          p\        P                  ! V^,          V,          V^,          V	,          V^,          V
,          .\        P
                  VR7      pW#,          V,          W4,          pp\        P                  ! WÎ3\        P
                  VR7      p\        ^\        V\        RV^(,          ,          4      4      4      p\        ^ VV4       EFs  p\        VV,           V4      pVVV1^ 3,          P                  4       pVVV1R3,          R,          VR,          ,           V,
          pVR,          VR,          VR,          pppV^ 8¬  VV8  ,          V^ 8¬  ,          VV8  ,          V^ 8¬  ,          VV8  ,          pVP                  4       P                  V4      pVP                  VP                  4       P                  V4      4      P                  V4      P                  VR	,          V,          4       VP                  V( R
4       V P                  VP!                  R
4      4      P#                  VV,
          V4      VVV% EKv  	  V# )r   ©r   r   ºé   é   Ng      ÀA)ºNNNNr[   )Nr[   r[   ).r   ).rY   ).r   )r[   Nr9   )r   r:   r   r   r   r   Úemptyr7   r=   Úintr	   r*   Úmul_Úadd_Úmasked_fill_rK   ÚreshapeÚview)ÚhashmaprU   ÚWÚHÚDr   r   r   r   r   r   r   ÚMr   ÚVÚcenterÚWHDÚHDÚneighborÚchunkrD   rE   ÚbÚcandÚxÚyÚzÚ	in_boundsÚflats   &&&&&&&&&&&                  r   Úbuild_submanifold_neighbor_mapru   .   s  € ð �]‰]€FØ�‰�Q�€AÜ$ R¨R°R¸VÓD€GØ�‰�aÕ€AÜ�\Š\˜B !�G r�>¨B°!­G°r­>¸BÀ!½GÀr½>ÐJÔRW×R]ÑR]ÐflÔm€FØ�e�a�i˜�ˆ€Cä�{Š{˜A˜6¬¯©¸VÔD€Hä�”3�qœ#Ð/°1°rµ6Õ:Ó;Ó<Ó=€Eä�1�a˜×ˆÜ��E•	˜1ÓˆØ�1�Q�3˜�6�N×ÑÓ!ˆØ�a˜�c˜3�hÕ 
Õ+¨g°jÕ.AÕAÀFÕJˆØ�v•,  V¥¨d°6­lˆaˆ1ˆØ˜!‘V  A¡Õ&¨!¨q©&Õ1°Q¸±UÕ;¸qÀA¹vÕFÈ!ÈaÉ%ÕPˆ	Ø�v‰v‹x�}‰}˜RÓ ˆØ�	‰	�!—&‘&“(—-‘- Ó"Ó#×(Ñ(¨Ó+×0Ñ0°°7µ¸cÕ1AÔBØ×Ñ˜9˜* bÔ)Ø×+Ñ+¨D¯L©L¸Ó,<Ó=×BÑBÀ1ÀqÅ5È!ÓLˆ��1‹ñ  ð €Or   c                ó<   € V ^8„  d   QhR\         R\         R\         /# )r   Úsafety_fractionÚmin_gbÚmax_gb)Úfloat)r#   s   "r   r%   r%   N   s(   € ÷ ?ñ ?äð?ô ð?ô ñ	?r   c                ó†   € \         P                  P                  V 4      R,          p\        V\	        WA,          V4      4      # )z<Pick a chunk-memory budget (in GB) for sparse conv batching.é   @)ÚcomfyÚmodel_managementÚget_free_memoryr7   r=   )r   rw   rx   ry   Úfree_gbs   &&&& r   Úget_recommended_chunk_memr�   N   s6   € ô ×$Ñ$×4Ñ4°VÓ<À	ÕJ€GÜˆv”s˜7Õ4°fÓ=Ó>Ð>r   c                ót  € V ^8„  d   QhR\         P                  R\         P                  R\        R\         P                  R\        \         P                  ,          R\        \         P                  ,          R\        R\        R	\
        \         P                  \        \         P                  ,          3,          /	# )
r   ÚfeatsrU   r:   ÚweightÚbiasÚneighbor_cacheÚdilationÚcache_neighbor_mapr5   )r   r"   Útupler   Úboolr   )r#   s   "r   r%   r%   X   s¢   € ÷ Nñ NÜ�<‰<ðNä�L‰LðNô ðNô �L‰Lð	Nô
 ”5—<‘<Õ
 ðNô œUŸ\™\Õ*ðNô ðNô ðNô Œ5�<‰<œ¤%§,¡,Õ/Ð/Õ0ñNr   c                 óš  € V P                   ^ ,          ^ 8X  dE   VP                   ^ ,          p\        P                  ! ^ V3V P                  V P                  R7      R3# Vw  ršpVP                   w  rŒrÞpWÍ,          V,          pV P                  pRpVfÿ   Wš,          V,          pW«,          pTp^pVR,          P                  4       V,          pVP                  VR,          VR7       VP                  VR,          VR7       VP                  VR,          VR7       \        P                  ! VP                   ^ ,          \        P                  VR7      p\        VV4      pV'       d)   \        VWW«WÍVV^ ,          V^,          V^,          4      pMRpMTpV P                   ^ ,          pVP                  VVV,          4      P                  p\        P                  ! VVVV P                  R7      p\        V4      pVV,          V P                  4       ,          pVR	,          p\        ^\!        VV,          4      4      p \#        V V4      p \%        ^ VV 4       F¼  p!\#        V!V ,           V4      p"V"V!,
          p#Vf,   \        VVV!V" WšW¼WÞV^ ,          V^,          V^,          4      p$MVV!V" p$V$P'                  ^ 4      p%V V%,          p&V&P)                  V$R
,          ^ 8  ^ 4       V&P                  V#VV,          4      p'\        P*                  ! V'V4      VV!V"% K¾  	  Ve2   VVP-                  ^ 4      P/                  VP                  4      ,          pVV3# )r   r   N)ÚalpharW   ©r[   r   ©r[   rY   ©r[   r   )r[   é   r|   )r[   r[   N)r:   r   r\   r   r   r*   r_   Úaranger   r   ru   rb   ÚTr�   Úelement_sizer7   r]   r=   r	   Ú	clamp_minr`   ÚmatmulÚ	unsqueezer)   )(rƒ   rU   r:   r„   r…   r†   r‡   rˆ   ÚCord   re   rf   r   r   r   ÚCirh   r   rc   Úb_strideÚx_strideÚy_strideÚz_strider4   Úvalsrl   ÚN_ptsÚweight_TÚoutputÚmax_chunk_mem_gbÚmem_per_rowÚmax_chunk_memÚ
chunk_sizeÚstartÚendÚactual_chunkÚneighbor_chunkÚ	chunk_idxÚgatheredÚgathered_flats(   &&&&&&&&                                r   Úsparse_submanifold_conv3dr¬   X   sà  € ð ‡{�{�1…~˜ÔØ�\‰\˜!�_ˆÜ�{Š{˜A˜r˜7¨5¯<©<¸u¿{¹{ÔKÈTÐQÐQà�G€Aˆ!àŸ™Ñ€BˆB�BØ
��"�€AØ�\‰\€Fà€GØÒØ•5˜1•9ˆØ•5ˆØˆØˆà˜4•L×%Ñ%Ó'¨(Õ2ˆ	Ø�‰�v˜d•|¨8ˆÔ4Ø�‰�v˜d•|¨8ˆÔ4Ø�‰�v˜d•|¨8ˆÔ4Ü�|Š|˜FŸL™L¨�O´5·;±;ÀvÔNˆÜ˜y¨$Ó/ˆçÜ5Ø˜ A¨"°"Ø˜•˜X a�[¨(°1­+ó‰Hð
 ‰Hà!ˆà�K‰K˜�N€Eà�{‰{˜2˜q 2�vÓ&×(Ñ(€Hä�[Š[˜ ¨6¸¿¹ÔE€Fô 1°Ó8ÐØ�b•&˜5×-Ñ-Ó/Õ/€KØ$¨	Õ2€MÜ�Qœ˜M¨KÕ7Ó8Ó9€JÜ�Z Ó'€Jä�q˜% Ö,ˆÜ�%˜*Õ$ eÓ,ˆØ˜U•{ˆàÒÜ;Ø˜  cÐ*¨A°!¸Ø˜•˜X a�[¨(°1­+ó‰Nð
 & e¨CÐ0ˆNà"×,Ñ,¨QÓ/ˆ	Ø˜Õ#ˆØ×Ñ˜n¨ZÕ8¸1Ñ<¸aÔ@Ø Ÿ™ l°A¸µFÓ;ˆÜ!ŸLšL¨¸ÓAˆˆu�SÒñ! -ð$ ÒØ�$—.‘. Ó#×&Ñ& v§|¡|Ó4Õ4ˆà�8ÐÐr   c                óî   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\        \         P                  \         P                  \        \         P                  3,          /# )r   rƒ   rU   r:   Úfactorr5   )r   r"   r‰   r]   r   )r#   s   "r   r%   r%   ©   s\   € ÷ "<ñ "<Ü�<‰<ð"<ä�L‰Lð"<ô ð"<ô ð	"<ô
 Œ5�<‰<œŸ™¤u¬e¯l©lÐ:Õ;ñ"<r   c           	     óV  a€ VR	,          P                  4       pVR
,          P                  4       pV3R lV 4       w  rgpVP                  SRR7      p	WF,          V	R	,          ,           V,          V	R,          ,           V,          V	R,          ,           p
\        P                  ! V
RR7      w  r¼\        P                  ! VP
                  ^ ,          V P
                  R,          3V P                  V P                  R7      pVP                  ^ WÀ4       \        P                  ! VP
                  ^ ,          3V P                  V P                  R7      pVP                  ^ V\        P                  ! WÀP                  R7      4       WÞP                  R4      ,          pW¸,          pVP                  VRR7      pVV,          pVP                  VRR7      pVV,          pVP                  VRR7      p\        P                  ! VVVV.RR7      P                  \        P                  4      pVVWgV3V3# )a=  Average-pool a sparse volume by `factor` along the three spatial axes.

coords are (batch, x, y, z) and shape is the spatial extent (X, Y, Z), as for
sparse_submanifold_conv3d. Returns the pooled (feats, coords, shape) plus the
fine->coarse index, which is what sparse_upsample3d_nearest needs to invert the pooling.
c              3   ód   <"  € T F%  p\        V4      S,           ^,
          S,          x € K'  	  R# 5i)rY   N)r]   )Ú.0rD   r®   s   & €r   Ú	<genexpr>Ú%sparse_pool3d_mean.<locals>.<genexpr>·   s&   øé € ÐJÁE¸qœC ›F V�O¨aÕ/°F×:Ò:ÃEùs   ƒ-0Úfloor)Úrounding_modeT)Úreturn_inverser   )r   )Údimr�   )r[   rX   rŽ   r�   r9   )r*   Údivr   ÚuniqueÚzerosr:   r   r   Ú
index_add_Ú	ones_liker–   Ústackr)   r   )rƒ   rU   r:   r®   ÚbatchÚfineÚout_xÚout_yÚout_zÚcoarsert   Úunique_flatÚindexÚpooledÚcountsrr   Úrestrq   rp   rn   Ú
out_coordss   &&&f                 r   Úsparse_pool3d_meanrÊ   ©   s´  ø€ ð �4�L×ÑÓ€EØ�&�>×ÑÓ €DÜJÁEÓJÑ€E�%à�X‰X�f¨GˆXÓ4€FØ�]˜V D�\Õ)¨UÕ2°V¸Dµ\ÕAÀUÕJÈVÐTXÍ\ÕY€DÜŸš d¸4Ô@Ñ€Kä�[Š[˜+×+Ñ+¨AÕ.°·±¸BµÐ@ÈÏÉÐ]b×]hÑ]hÔi€FØ
×Ñ�a˜Ô&Ü�[Š[˜+×+Ñ+¨AÕ.Ð0¸¿¹ÈUÏ[É[ÔY€FØ
×Ñ�a˜¤§¢°¿[¹[Ô IÔJØ
×Ñ˜rÓ"Õ"€FàÕ€AØ�?‰?˜5°ˆ?Ó8€DØˆu�€AØ�8‰8�E¨ˆ8Ó1€DØˆu�€AØ�‰� gˆÓ.€AÜ—’˜a  A q˜\¨rÔ2×5Ñ5´e·k±kÓB€Jà�: ¨eÐ4°eÐ;Ð;r   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r   rƒ   Ú
pool_indexr5   r!   )r#   s   "r   r%   r%   Î   s0   € ÷ ñ ¤U§\¡\ð ¼u¿|¹|ð ÔPU×P\ÑP\ñ r   c                ó   € W,          # )z÷Invert sparse_pool3d_mean's spatial mapping: each fine voxel takes its parent's feature.

`pool_index` is the index returned by the matching sparse_pool3d_mean call, so the
output is aligned with that call's input coords (the UNet's skip coords).
© )rƒ   rÌ   s   &&r   Úsparse_upsample3d_nearestrÏ   Î   s   € ð ÕÐr   )Ngš™™™™™É?g      Ð?g      à?)T)Útypingr   r   r   Úcomfy.model_managementr}   r   r   ru   r�   r¬   rÊ   rÏ   rÎ   r   r   Ú<module>rÒ      s;   ðß "Û ã òC÷ñ õ:÷@?÷Nõb"<÷Jr   