+
    
åûjo  ã                   óî  € ^ RI Ht ^ RIHtHtHt ^ RIt^ RIHu H	t
 ^ RIHtHt ^RIHt ^RIHt ^RIHtHtHt ^RIHt ^R	IHtHt ^R
IHtHt . R6OtR R lt]P<                  P?                  R4       R R lt ]P<                  P?                  R4       R R lt!]P<                  P?                  R4       R R lt"]P<                  P?                  R4       R7R R llt#]P<                  P?                  R4        ! R R ]PH                  4      t% ! R! R"]PH                  4      t& ! R# R]PH                  4      t'R$ R% lt(R&]R'R8R(^/t) ! R) R]4      t* ! R* R]4      t+ ! R+ R]4      t,]! 4       ]! R,]*PZ                  3R-7      R.RR/R/R0 R1 ll4       4       t.]! 4       ]! R,]+PZ                  3R-7      R.RR/R/R2 R3 ll4       4       t/]! 4       ]! R,],PZ                  3R-7      R.RR/R/R4 R5 ll4       4       t0R# )9é    )Úpartial)ÚAnyÚCallableÚOptionalN)ÚnnÚTensor)ÚVideoClassification)Ú_log_api_usage_once)Úregister_modelÚWeightsÚWeightsEnum)Ú_KINETICS400_CATEGORIES)Ú_ovewrite_named_paramÚhandle_legacy_interface)ÚPatchMergingÚSwinTransformerBlockÚSwinTransformer3dÚSwin3D_T_WeightsÚSwin3D_S_WeightsÚSwin3D_B_Weightsc                óØ   € V ^8„  d   QhR\         \        ,          R\         \        ,          R\         \        ,          R\        \         \        ,          \         \        ,          3,          /# )é   Ú
shift_sizeÚsize_dhwÚwindow_sizeÚreturn)ÚlistÚintÚtuple)Úformats   "ÚeC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\torchvision\models\video\swin_transformer.pyÚ__annotate__r"       sJ   € ÷ 	#ñ 	#Ü”S•	ð	#Ü%)¬#¥Yð	#Ü=AÄ#½Yð	#ä
Œ4”�9”dœ3•iÐÕ ñ	#ó    c                 ór   € \        ^4       F&  pW,          W#,          8:  g   K  W,          W#&   ^ W&   K(  	  W 3# ©é   ©Úrange)r   r   r   Úis   &&& r!   Ú_get_window_and_shift_sizer*       s9   € ô �1ŽXˆØ�;˜+�.Ö(à%�[ˆK‰NØˆJ‹Mñ	 ð Ð"Ð"r#   r*   c                ó†   € V ^8„  d   QhR\         P                  R\         P                  R\        \        ,          R\        /# )r   Úrelative_position_bias_tableÚrelative_position_indexr   r   )Útorchr   r   r   )r    s   "r!   r"   r"   /   s:   € ÷ 
"ñ 
"Ü"'§,¡,ð
"ÜINÏÉð
"ÜdhÔilÕdmð
"äñ
"r#   c                 ó  € V^ ,          V^,          ,          V^,          ,          pV VRV1RV13,          P                  4       ,          pVP                  W3R4      pVP                  ^^ ^4      P                  4       P	                  ^ 4      pV# )r   Néÿÿÿÿ)ÚflattenÚviewÚpermuteÚ
contiguousÚ	unsqueeze)r,   r-   r   Ú
window_volÚrelative_position_biass   &&&  r!   Ú_get_relative_position_biasr8   /   sŽ   € ð ˜Q• +¨a¥.Õ0°;¸qµ>ÕA€Jà9Ø   ¨[¨j¨[Ð 8Õ9×AÑAÓCõÐð 4×8Ñ8¸ÐQSÓTÐØ3×;Ñ;¸A¸qÀ!ÓD×OÑOÓQ×[Ñ[Ð\]Ó^ÐØ!Ð!r#   r8   c          	      óÀ   € V ^8„  d   QhR\         \        \        \        3,          R\         \        \        \        3,          R\         \        \        \        3,          /# )r   r   Ú
patch_sizer   )r   r   )r    s   "r!   r"   r"   ?   sI   € ÷ 1ñ 1¤5¬¬c´3¨Õ#7ð 1ÄUÌ3ÔPSÔUXÈ=ÕEYð 1Ô^cÔdgÔilÔnqÐdqÕ^rñ 1r#   c                 óÎ   € \        ^4       Uu. uF2  q!V,          W,          W,          ,          ,
          W,          ,          NK4  	  ppV^ ,          V^,          V^,          3# u upi r%   r'   )r   r:   r)   Úpad_sizes   &&  r!   Ú_compute_pad_size_3dr=   ?   sX   € ÜW\Ð]^ÔW_Ó`ÑW_ÐRS˜A• ¥¨z­}Õ!<Õ<À
Å×MÐMÑW_€HÐ`Ø�A�;˜ � X¨a¥[Ð0Ð0ùò as   Ž8A"r=   c                óØ   € V ^8„  d   QhR\         R\        \        \        \        3,          R\        \        \        \        3,          R\        \        \        \        3,          R\         /# )r   Úxr   r   r   r   )r   r   r   )r    s   "r!   r"   r"   G   s[   € ÷ &ñ &Üð&ä”Cœœc�MÕ"ð&ô ”sœC¤�}Õ%ð&ô ”cœ3¤�mÕ$ð	&ô
 ñ&r#   c           
      óV  € V P                   ! V!  pV^ ,          V^ ,          ,          V^,          V^,          ,          ,          V^,          V^,          ,          ,          p\        ^4       Uu. uF*  p^ W&,          ) 3W&,          ) W6,          ) 3W6,          ) R33NK,  	  pp^ pV^ ,           Fc  p	V^,           FS  p
V^,           FC  pW„V	^ ,          V	^,          1V
^ ,          V
^,          1V^ ,          V^,          13&   V^,          pKE  	  KU  	  Ke  	  VP                  V^ ,          V^ ,          ,          V^ ,          V^,          V^,          ,          V^,          V^,          V^,          ,          V^,          4      pVP                  ^ ^^^^^4      P	                  WR^ ,          V^,          ,          V^,          ,          4      pVP                  ^4      VP                  ^4      ,
          pVP                  V^ 8g  \        R4      4      P                  V^ 8H  \        R4      4      pV# u upi )r   Nç        g      YÀ)Ú	new_zerosr(   r2   r3   Úreshaper5   Úmasked_fillÚfloat)r?   r   r   r   Ú	attn_maskÚnum_windowsr)   ÚslicesÚcountÚdÚhÚws   &&&&        r!   Ú_compute_attention_mask_3drM   G   s  € ð —’˜XÑ&€IØ˜A•; +¨a¥.Õ0°X¸aµ[ÀKÐPQÅNÕ5RÕSÐW_Ð`aÕWbÐfqÐrsÕftÕWtÕu€Kô �q”óñ ˆAð	 �•�Ð Ø�nˆ_˜z�}˜nÐ-Ø�mˆ^˜TÐ"ó	
ñ
 ð ð ð €EØ�A�YˆYˆØ˜—�ˆAØ˜A—Y�Y�ØCH˜!˜A�$  1¥˜+ q¨¥t¨a°­d {°A°aµD¸1¸Q½4°KÐ?Ñ@Ø˜•
’ó ó ñ ð —‘Ø���{ 1•~Õ%Ø�A�Ø���{ 1•~Õ%Ø�A�Ø���{ 1•~Õ%Ø�A�ó€Ið ×!Ñ! ! Q¨¨1¨a°Ó3×;Ñ;Ø •^ k°!¥nÕ4°{À1µ~ÕEó€Ið ×#Ñ# AÓ&¨×)<Ñ)<¸QÓ)?Õ?€IØ×%Ñ% i°1¡n´e¸F³mÓD×PÑPÐQZÐ^_ÑQ_ÔafÐgjÓakÓl€IØÐùò;s   Á-0H&rM   Tc                ó  € V ^8„  d   QhR\         R\         R\         R\         R\        \        ,          R\        R\        \        ,          R\        R	\        R
\        \         ,          R\        \         ,          R\
        R\         /# )r   ÚinputÚ
qkv_weightÚproj_weightr7   r   Ú	num_headsr   Úattention_dropoutÚdropoutÚqkv_biasÚ	proj_biasÚtrainingr   )r   r   r   rE   r   Úbool)r    s   "r!   r"   r"   s   s«   € ÷ mñ mÜðmäðmô ðmô #ð	mô
 ”c•ðmô ðmô ”S•	ðmô ðmô ðmô ”vÕðmô œÕðmô ðmô ñmr#   c                óŽ
  € V P                   w  rÍrïp\        WÞV3V^ ,          V^,          V^,          34      p\        P                  ! V ^ ^ ^ V^,          ^ V^,          ^ V^ ,          34      pVP                   w  pppppVVV3p\	        V4      ^ 8”  d5   \
        P                  ! VV^ ,          ) V^,          ) V^,          ) 3RR7      pV^ ,          V^ ,          ,          V^,          V^,          ,          ,          V^,          V^,          ,          ,          pVP                  VV^ ,          V^ ,          ,          V^ ,          V^,          V^,          ,          V^,          V^,          V^,          ,          V^,          V4      pVP                  ^ ^^^^^^^4      P                  VV,          V^ ,          V^,          ,          V^,          ,          V4      p\        P                  ! VW4      pVP                  VP                  ^ 4      VP                  ^4      ^VVV,          4      P                  ^^ ^^^4      pV^ ,          V^,          V^,          pppVVV,          R,          ,          pVP                  VP                  RR	4      4      pVV,           p\	        V4      ^ 8”  dù   \        VV^ ,          V^,          V^,          3V^ ,          V^,          V^,          3V^ ,          V^,          V^,          34      pVP                  VP                  ^ 4      V,          VVVP                  ^4      VP                  ^4      4      pVVP                  ^4      P                  ^ 4      ,           pVP                  R	VVP                  ^4      VP                  ^4      4      p\        P                   ! VR	R7      p\        P"                  ! VW{R7      pVP                  V4      P                  ^^4      P                  VP                  ^ 4      VP                  ^4      V4      p\        P                  ! VW*4      p\        P"                  ! VW‹R7      pVP                  VV^ ,          V^ ,          ,          V^,          V^,          ,          V^,          V^,          ,          V^ ,          V^,          V^,          V4      pVP                  ^ ^^^^^^^4      P                  VVVVV4      p\	        V4      ^ 8”  d2   \
        P                  ! VV^ ,          V^,          V^,          3RR7      pVRRV1RV1RV1R3,          P%                  4       pV# )
a‚  
Window based multi-head self attention (W-MSA) module with relative position bias.
It supports both of shifted and non-shifted window.
Args:
    input (Tensor[B, T, H, W, C]): The input tensor, 5-dimensions.
    qkv_weight (Tensor[in_dim, out_dim]): The weight tensor of query, key, value.
    proj_weight (Tensor[out_dim, out_dim]): The weight tensor of projection.
    relative_position_bias (Tensor): The learned relative position bias added to attention.
    window_size (List[int]): 3-dimensions window size, T, H, W .
    num_heads (int): Number of attention heads.
    shift_size (List[int]): Shift size for shifted window attention (T, H, W).
    attention_dropout (float): Dropout ratio of attention weight. Default: 0.0.
    dropout (float): Dropout ratio of output. Default: 0.0.
    qkv_bias (Tensor[out_dim], optional): The bias tensor of query, key, value. Default: None.
    proj_bias (Tensor[out_dim], optional): The bias tensor of projection. Default: None.
    training (bool, optional): Training flag used by the dropout parameters. Default: True.
Returns:
    Tensor[B, T, H, W, C]: The output tensor after shifted window attention.
)ÚshiftsÚdims)Údim)ÚprW   ºNNNN)é   r   r&   g      à¿éþÿÿÿr0   )Úshaper=   ÚFÚpadÚsumr.   Úrollr2   r3   rC   ÚlinearÚsizeÚmatmulÚ	transposerM   r5   ÚsoftmaxrT   r4   )rO   rP   rQ   r7   r   rR   r   rS   rT   rU   rV   rW   ÚbÚtrK   rL   Úcr<   r?   Ú_ÚtpÚhpÚwpÚpadded_sizerG   ÚqkvÚqÚkÚvÚattnrF   s   &&&&&&&&&&&&                   r!   Úshifted_window_attention_3drx   s   s�  € ðB —K‘K�M€Aˆ!�ä# Q¨1 I°¸AµÀÈAÅÐP[Ð\]ÕP^Ð/_Ó`€HÜ	�Šˆe�a˜˜A˜x¨�{¨A¨x¸­{¸A¸xÈ½{ÐKÓL€AØ—w‘wÑ€A€rˆ2ˆr�1Ø�r˜2�,€Kô ˆ:ƒ˜ÔÜ�JŠJ�q :¨a¥= .°:¸aµ=°.À:ÈaÅ=À.Ð!QÐXaÔbˆð 
�Q�˜; q�>Õ	)¨k¸!­nÀÈAÅÕ.NÕOÐS^Ð_`ÕSaÐepÐqrÕesÕSsÕtð ð 	
�‰Ø	Ø�A�˜+ a�.Õ(Ø�A�Ø�A�˜+ a�.Õ(Ø�A�Ø�A�˜+ a�.Õ(Ø�A�Ø	ó		€Að 	
�	‰	�!�Q˜˜1˜a  A qÓ)×1Ñ1Ø	ˆK�˜ Q�¨+°a­.Õ8¸;Àq½>ÕIÈ1ó	€Aô
 �(Š(�1�jÓ
+€CØ
�+‰+�a—f‘f˜Q“i §¡¨£¨A¨y¸!¸y½.Ó
I×
QÑ
QÐRSÐUVÐXYÐ[\Ð^_Ó
`€CØ�!�f�c˜!•f˜c !�fˆ!€q€AØ	ˆQ�)�^ Õ$Õ$€AØ�8‰8�A—K‘K  BÓ'Ó(€DàÐ(Õ(€Dä
ˆ:ƒ˜Ôä.ØØ˜�^˜[¨�^¨[¸­^Ð<Ø˜�^˜[¨�^¨[¸­^Ð<Ø˜�]˜J q�M¨:°a­=Ð9ó	
ˆ	ð �y‰y˜Ÿ™ › kÕ1°;À	È1Ï6É6ÐRSË9ÐVW×V\ÑV\Ð]^ÓV_Ó`ˆØ�i×)Ñ)¨!Ó,×6Ñ6°qÓ9Õ9ˆØ�y‰y˜˜Y¨¯©¨q«	°1·6±6¸!³9Ó=ˆä�9Š9�T˜rÔ"€DÜ�9Š9�TÐ.ÔB€Dà�‰�A‹× Ñ   AÓ&×.Ñ.¨q¯v©v°a«y¸!¿&¹&À»)ÀQÓG€AÜ	�Š��KÓ+€AÜ	�	Š	�!�wÔ2€Að 	
�‰Ø	Ø�A�˜+ a�.Õ(Ø�A�˜+ a�.Õ(Ø�A�˜+ a�.Õ(Ø�A�Ø�A�Ø�A�Ø	ó		€Að 	
�	‰	�!�Q˜˜1˜a  A qÓ)×1Ñ1°!°R¸¸RÀÓC€Aô ˆ:ƒ˜ÔÜ�JŠJ�q *¨Q¥-°¸AµÀ
È1ÅÐ!NÐU^Ô_ˆð 	
ˆ!ˆRˆaˆR��!��R�a�R˜Ð
Õ×&Ñ&Ó(€AØ€Hr#   rx   c                   óŽ   a a€ ] tR t^æt oRtRV3R lV 3R llltV3R lR ltV3R lR ltV3R lR	 ltV3R
 lR lt	Rt
VtV ;t# )ÚShiftedWindowAttention3dz*
See :func:`shifted_window_attention_3d`.
c                ón   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[S[ ,          RS[ RS[RS[RS[RS[R	R
/	# )r   r\   r   r   rR   rU   rV   rS   rT   r   N)r   r   rX   rE   )r    Ú__classdict__s   "€r!   r"   Ú%ShiftedWindowAttention3d.__annotate__ë   sm   ø€ ÷ .ñ .áð.ñ ™#•Yð.ñ ™•Ið	.ñ
 ð.ñ ð.ñ ð.ñ !ð.ñ ð.ð 
ñ.r#   c	                óx  <€ \         S	V `  4        \        V4      ^8w  g   \        V4      ^8w  d   \        R4      hW n        W0n        W@n        Wpn        W€n        \        P                  ! W^,          VR7      V n        \        P                  ! WVR7      V n        V P                  4        V P                  4        R# )r&   z.window_size and shift_size must be of length 2)ÚbiasN)ÚsuperÚ__init__ÚlenÚ
ValueErrorr   r   rR   rS   rT   r   ÚLinearrs   ÚprojÚ#define_relative_position_bias_tableÚdefine_relative_position_index)
Úselfr\   r   r   rR   rU   rV   rS   rT   Ú	__class__s
   &&&&&&&&&€r!   r�   Ú!ShiftedWindowAttention3d.__init__ë   s�   ø€ ô 	‰ÑÔÜˆ{Ó˜qÔ ¤C¨
£O°qÔ$8ÜÐMÓNÐNà&ÔØ$ŒØ"ŒØ!2ÔØŒä—9’9˜S¨¥'°Ô9ˆŒÜ—I’I˜c¨YÔ7ˆŒ	à×0Ñ0Ô2Ø×+Ñ+Ö-r#   c                ó   <€ V ^8„  d   QhRR/# ©r   r   N© )r    r|   s   "€r!   r"   r}     s   ø€ ÷ Kñ K°Tñ Kr#   c                ó¦  € \         P                  ! \        P                  ! ^V P                  ^ ,          ,          ^,
          ^V P                  ^,          ,          ^,
          ,          ^V P                  ^,          ,          ^,
          ,          V P
                  4      4      V n        \         P                  P                  V P                  RR7       R# )r   ç{®Gáz”?©ÚstdN)	r   Ú	Parameterr.   Úzerosr   rR   r,   ÚinitÚtrunc_normal_)rˆ   s   &r!   r†   Ú<ShiftedWindowAttention3d.define_relative_position_bias_table  s•   € ä,.¯LªLÜ�KŠKØ�T×%Ñ% aÕ(Õ(¨1Õ,°°T×5EÑ5EÀaÕ5HÕ1HÈ1Õ1LÕMÐQRÐUY×UeÑUeÐfgÕUhÕQhÐklÕQlÕmØ—‘óó-
ˆÔ)ô 	�‰×Ñ˜d×?Ñ?ÀTÐÖJr#   c                ó   <€ V ^8„  d   QhRR/# rŒ   r�   )r    r|   s   "€r!   r"   r}     s   ø€ ÷ Qñ Q°ñ Qr#   c           	     ó"  € \        ^4       Uu. uF*  p\        P                  ! V P                  V,          4      NK,  	  pp\        P                  ! \        P
                  ! V^ ,          V^,          V^,          RR7      4      p\        P                  ! V^4      pVR,          VR,          ,
          pVP                  ^^^ 4      P                  4       pVR;;,          V P                  ^ ,          ^,
          ,          uu&   VR;;,          V P                  ^,          ^,
          ,          uu&   VR	;;,          V P                  ^,          ^,
          ,          uu&   VR;;,          ^V P                  ^,          ,          ^,
          ^V P                  ^,          ,          ^,
          ,          ,          uu&   VR;;,          ^V P                  ^,          ,          ^,
          ,          uu&   VP                  R
4      pV P                  RV4       R# u upi )r&   Úij)ÚindexingNr-   )r^   r^   N)r^   Nr^   )r^   r^   r   )r^   r^   r_   )r^   r^   r   r0   )r(   r.   Úaranger   ÚstackÚmeshgridr1   r3   r4   rd   Úregister_buffer)rˆ   r)   Ú
coords_dhwÚcoordsÚcoords_flattenÚrelative_coordsr-   s   &      r!   r‡   Ú7ShiftedWindowAttention3d.define_relative_position_index  s�  € äAFÀqÄÓJÁ¸A”e—l’l 4×#3Ñ#3°AÕ#6Ö7Áˆ
ÐJÜ—’Ü�NŠN˜: a�=¨*°Q­-¸ÀA½ÐQUÔVó
ˆô Ÿš v¨qÓ1ˆØ(¨Õ4°~ÀjÕ7QÕQˆØ)×1Ñ1°!°Q¸Ó:×EÑEÓGˆØ˜×  D×$4Ñ$4°QÕ$7¸!Õ$;Õ;Ó Ø˜×  D×$4Ñ$4°QÕ$7¸!Õ$;Õ;Ó Ø˜×  D×$4Ñ$4°QÕ$7¸!Õ$;Õ;Ó à˜×  Q¨×)9Ñ)9¸!Õ)<Õ%<¸qÕ%@ÀQÈ×IYÑIYÐZ[ÕI\ÕE\Ð_`ÕE`Õ$aÕaÓ Ø˜×  A¨×(8Ñ(8¸Õ(;Õ$;¸aÕ$?Õ?Ó à"1×"5Ñ"5°bÓ"9ÐØ×ÑÐ6Ð8OÖPùò Ks   Ž0Hc                óJ   <€ V ^8„  d   QhRS[ S[,          RS[P                  /# )r   r   r   )r   r   r.   r   )r    r|   s   "€r!   r"   r}   #  s'   ø€ ÷ yñ y±d¹3µið yÁEÇLÁLñ yr#   c                óD   € \        V P                  V P                  V4      # )N)r8   r,   r-   )rˆ   r   s   &&r!   Úget_relative_position_biasÚ3ShiftedWindowAttention3d.get_relative_position_bias#  s   € Ü*¨4×+LÑ+LÈd×NjÑNjÐlwÓxÐxr#   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# ©r   r?   r   ©r   )r    r|   s   "€r!   r"   r}   &  s   ø€ ÷ 
ñ 
™ð 
¡Fñ 
r#   c                óì  € VP                   w  r#rEpW4V.pV P                  P                  4       V P                  P                  4       r‡\	        W†V4      w  rxV P                  V4      p	\        VV P                  P                  V P                  P                  V	VV P                  VV P                  V P                  V P                  P                  V P                  P                  V P                  R 7      # ))r   rS   rT   rU   rV   rW   )ra   r   Úcopyr   r*   r¦   rx   rs   Úweightr…   rR   rS   rT   r   rW   )
rˆ   r?   rn   rl   rK   rL   r   r   r   r7   s
   &&        r!   ÚforwardÚ ShiftedWindowAttention3d.forward&  sÃ   € ØŸ™‰ˆˆa�AØ˜!�9ˆØ"&×"2Ñ"2×"7Ñ"7Ó"9¸4¿?¹?×;OÑ;OÓ;Q�Zä"<¸ZÐS^Ó"_Ñˆà!%×!@Ñ!@ÀÓ!MÐä*ØØ�H‰H�O‰OØ�I‰I×ÑØ"ØØ�N‰NØ!Ø"×4Ñ4Ø—L‘LØ—X‘X—]‘]Ø—i‘i—n‘nØ—]‘]ô
ð 	
r#   )rS   rT   rR   r…   rs   r,   r   r   )TTrA   rA   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r�   r†   r‡   r¦   r®   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r‰   r|   s   @@r!   rz   rz   æ   sG   ù‡ € ñ÷.õ .÷6Kð K÷Qð Q÷&yð y÷
÷ 
ð 
r#   rz   c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚPatchEmbed3diA  a#  Video to Patch Embedding.

Args:
    patch_size (List[int]): Patch token size.
    in_channels (int): Number of input channels. Default: 3
    embed_dim (int): Number of linear projection output channels. Default: 96.
    norm_layer (nn.Module, optional): Normalization layer. Default: None
c                ó~   <€ V ^8„  d   QhRS[ S[,          RS[RS[RS[S[RS[P
                  3,          ,          RR/# )r   r:   Úin_channelsÚ	embed_dimÚ
norm_layer.r   N)r   r   r   r   r   ÚModule)r    r|   s   "€r!   r"   ÚPatchEmbed3d.__annotate__K  sS   ø€ ÷ &ñ &á™•Ið&ñ ð&ñ ð	&ñ
 ™X c©2¯9©9 nÕ5Õ6ð&ð 
ñ&r#   c                ó8  <€ \         SV `  4        \        V 4       V^ ,          V^,          V^,          3V n        \        P
                  ! VVV P                  V P                  R7      V n        Ve   V! V4      V n        R# \        P                  ! 4       V n        R# )r   )Úkernel_sizeÚstrideN)	r€   r�   r
   Útuple_patch_sizer   ÚConv3dr…   ÚnormÚIdentity)rˆ   r:   r¼   r½   r¾   r‰   s   &&&&&€r!   r�   ÚPatchEmbed3d.__init__K  s|   ø€ ô 	‰ÑÔÜ˜DÔ!Ø!+¨A¥°
¸1µ¸zÈ!½}Ð MˆÔä—I’IØØØ×-Ñ-Ø×(Ñ(ô	
ˆŒ	ð Ò!Ù" 9Ó-ˆDŽIäŸš›ˆDŽIr#   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# r©   rª   )r    r|   s   "€r!   r"   rÀ   a  ó   ø€ ÷ 
ñ 
™ð 
¡Fñ 
r#   c           
     óL  € VP                  4       w   r#rE\        W4V3V P                  4      p\        P                  ! V^ V^,          ^ V^,          ^ V^ ,          34      pV P                  V4      pVP                  ^ ^^^^4      pV P                  e   V P                  V4      pV# )zForward function.)rg   r=   rÄ   rb   rc   r…   r3   rÆ   )rˆ   r?   rn   rl   rK   rL   r<   s   &&     r!   r®   ÚPatchEmbed3d.forwarda  s“   € ð Ÿ™›‰ˆˆ1�Ü'¨¨q¨	°4×3HÑ3HÓIˆÜ�EŠE�!�a˜ !� a¨°!­°a¸À!½ÐEÓFˆØ�I‰I�a‹LˆØ�I‰I�a˜˜A˜q !Ó$ˆØ�9‰9Ò Ø—	‘	˜!“ˆAØˆr#   )rÆ   r…   rÄ   )r&   é`   N)
r°   r±   r²   r³   r´   r�   r®   rµ   r¶   r·   r¸   s   @@r!   rº   rº   A  s#   ù‡ € ñ÷&õ &÷,
÷ 
ð 
r#   rº   c            	       ój   a a€ ] tR tRt oRtRRRRRRR]R3	V3R lV 3R	 llltV3R
 lR ltRtVt	V ;t
# )r   in  a  
Implements 3D Swin Transformer from the `"Video Swin Transformer" <https://arxiv.org/abs/2106.13230>`_ paper.
Args:
    patch_size (List[int]): Patch size.
    embed_dim (int): Patch embedding dimension.
    depths (List(int)): Depth of each Swin Transformer layer.
    num_heads (List(int)): Number of attention heads in different layers.
    window_size (List[int]): Window size.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.0.
    dropout (float): Dropout rate. Default: 0.0.
    attention_dropout (float): Attention dropout rate. Default: 0.0.
    stochastic_depth_prob (float): Stochastic depth rate. Default: 0.1.
    num_classes (int): Number of classes for classification head. Default: 400.
    norm_layer (nn.Module, optional): Normalization layer. Default: None.
    block (nn.Module, optional): SwinTransformer Block. Default: None.
    downsample_layer (nn.Module): Downsample layer (patch merging). Default: PatchMerging.
    patch_embed (nn.Module, optional): Patch Embedding layer. Default: None.
g      @rA   çš™™™™™¹?i�  Nc                ó‚  <€ V ^8„  d   QhRS[ S[,          RS[RS[ S[,          RS[ S[,          RS[ S[,          RS[RS[RS[R	S[R
S[RS[S[RS[P                  3,          ,          RS[S[RS[P                  3,          ,          RS[RS[P                  3,          RS[S[RS[P                  3,          ,          RR/# )r   r:   r½   ÚdepthsrR   r   Ú	mlp_ratiorT   rS   Ústochastic_depth_probÚnum_classesr¾   .ÚblockÚdownsample_layerÚpatch_embedr   N)r   r   rE   r   r   r   r¿   )r    r|   s   "€r!   r"   ÚSwinTransformer3d.__annotate__‚  s  ø€ ÷ J+ñ J+á™•IðJ+ñ ðJ+ñ ‘S•	ð	J+ñ
 ™•9ðJ+ñ ™#•YðJ+ñ ðJ+ñ ðJ+ñ !ðJ+ñ  %ðJ+ñ ðJ+ñ ™X c©2¯9©9 nÕ5Õ6ðJ+ñ ™ ¡b§i¡i Õ0Õ1ðJ+ñ # 3©¯	©	 >Õ2ðJ+ñ ™h s©B¯I©I ~Õ6Õ7ðJ+ð  
ñ!J+r#   c                óÜ  <€ \         SV `  4        \        V 4       W n        Vf   \	        \
        \        R7      pVf   \	        \        P                  RR7      pVf   \        pV! WVR7      V n
        \        P                  ! VR7      V n        . p\        V4      p^ p\        \        V4      4       Fÿ  p. pV^V,          ,          p\        VV,          4       F  pV	\!        V4      ,          V^,
          ,          pTP#                  T! TVV,          TV Uu. uF  pV^,          ^ 8X  d   ^ MV^,          NK  	  upVVVVV\        R7
      4       V^,          pK�  	  VP#                  \        P$                  ! V!  4       V\        V4      ^,
          8  g   Kæ  VP#                  V! VV4      4       EK  	  \        P$                  ! V!  V n        V^\        V4      ^,
          ,          ,          V n        V! V P(                  4      V n        \        P,                  ! ^4      V n        \        P0                  ! V P(                  V
4      V n        V P5                  4        F‰  p\7        V\        P0                  4      '       g   K%  \        P8                  P;                  VP<                  RR7       VP>                  f   K`  \        P8                  PA                  VP>                  4       K‹  	  R # u upi )	N)Ú
attn_layergñhãˆµøä>)Úeps)r:   r½   r¾   )r]   )r   r   rÒ   rT   rS   rÓ   r¾   rÚ   r�   r�   )!r€   r�   r
   rÔ   r   r   rz   r   Ú	LayerNormrº   r×   ÚDropoutÚpos_droprd   r(   r‚   rE   ÚappendÚ
SequentialÚfeaturesÚnum_featuresrÆ   ÚAdaptiveAvgPool3dÚavgpoolr„   ÚheadÚmodulesÚ
isinstancer”   r•   r­   r   Úzeros_)rˆ   r:   r½   rÑ   rR   r   rÒ   rT   rS   rÓ   rÔ   r¾   rÕ   rÖ   r×   ÚlayersÚtotal_stage_blocksÚstage_block_idÚi_stageÚstager\   Úi_layerÚsd_probrL   Úmr‰   s   &&&&&&&&&&&&&&&          €r!   r�   ÚSwinTransformer3d.__init__‚  sE  ø€ ô" 	‰ÑÔÜ˜DÔ!Ø&ÔàŠ=ÜÔ0Ô=UÔVˆEàÒÜ ¤§¡°4Ô8ˆJàÒÜ&ˆKñ '°*Ð^hÔiˆÔÜŸ
š
 WÔ-ˆŒà"$ˆÜ  ›[ÐØˆäœS ›[Ö)ˆGØ%'ˆEØ˜a �jÕ(ˆCÜ  ¨¥Ö1�à/´%¸Ó2GÕGÐK]Ð`aÕKaÕb�Ø—‘ÙØØ! 'Õ*Ø$/ÙOZÓ#[É{È!¨°1­¸Ô)9¡A¸qÀA½vÒ$EÉ{Ñ#[Ø"+Ø 'Ø*;Ø.5Ø#-Ü#;ôôð  !Õ#’ñ# 2ð$ �M‰Mœ"Ÿ-š-¨Ñ/Ô0àœ#˜f›+¨�/Ö*Ø—‘Ñ.¨s°JÓ?×@ñ1 *ô2 Ÿš vÑ.ˆŒà%¨¬c°&«k¸A­oÕ(>Õ>ˆÔÙ˜t×0Ñ0Ó1ˆŒ	Ü×+Ò+¨AÓ.ˆŒÜ—I’I˜d×/Ñ/°Ó=ˆŒ	à—‘–ˆAÜ˜!œRŸY™Y×'Ô'Ü—‘×%Ñ% a§h¡h°DÐ%Ô9Ø—6‘6Ô%Ü—G‘G—N‘N 1§6¡6Ö*ó	  ùò+ $\s   Ä"K)c                ó&   <€ V ^8„  d   QhRS[ RS[ /# r©   rª   )r    r|   s   "€r!   r"   rØ   Î  rÊ   r#   c                ó*  € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pVP	                  ^ ^^^^4      pV P                  V4      p\        P                  ! V^4      pV P                  V4      pV# )r   )	r×   rÞ   rá   rÆ   r3   rä   r.   r1   rå   )rˆ   r?   s   &&r!   r®   ÚSwinTransformer3d.forwardÎ  s€   € à×Ñ˜QÓˆØ�M‰M˜!ÓˆØ�M‰M˜!ÓˆØ�I‰I�a‹LˆØ�I‰I�a˜˜A˜q !Ó$ˆØ�L‰L˜‹OˆÜ�MŠM˜!˜QÓˆØ�I‰I�a‹LˆØˆr#   )rä   rá   rå   rÆ   rÔ   râ   r×   rÞ   )r°   r±   r²   r³   r´   r   r�   r®   rµ   r¶   r·   r¸   s   @@r!   r   r   n  sC   ù‡ € ñð4 ØØ#&Ø'*ØØ9=Ø48Ø5AØ:>÷J+õ J+÷X
÷ 
ð 
r#   c                óþ   € V ^8„  d   QhR\         \        ,          R\        R\         \        ,          R\         \        ,          R\         \        ,          R\        R\        \        ,          R\
        R	\        R
\        /
# )r   r:   r½   rÑ   rR   r   rÓ   ÚweightsÚprogressÚkwargsr   )r   r   rE   r   r   rX   r   r   )r    s   "r!   r"   r"   Û  s�   € ÷ ñ Ü”S•	ðäðô ”�Iðô ”C�yð	ô
 ”c•ðô !ðô ”kÕ"ðô ðô ðô ñr#   c                 óÖ   € Ve(   \        VR\        VP                  R,          4      4       \        RRV RVRVRVRVRV/VB p	Ve#   V	P	                  VP                  VR	R
7      4       V	# )NrÔ   Ú
categoriesr:   r½   rÑ   rR   r   rÓ   T)r÷   Ú
check_hashr�   )r   r‚   Úmetar   Úload_state_dictÚget_state_dict)
r:   r½   rÑ   rR   r   rÓ   rö   r÷   rø   Úmodels
   &&&&&&&&, r!   Ú_swin_transformer3dr   Û  sœ   € ð ÒÜ˜f m´S¸¿¹ÀlÕ9SÓ5TÔUäñ Øðàðð ðð ð	ð
  ðð 4ðð ñ€Eð ÒØ×Ñ˜g×4Ñ4¸hÐSWÐ4ÓXÔYà€Lr#   rú   Úmin_sizeÚmin_temporal_sizec                   ór   € ] tR tRt]! R]! ]RRRRR7      / ]CRRRRRR	R
RRRRR//RRRR/CR7      t]t	Rt
R# )r   i   z9https://download.pytorch.org/models/swin3d_t-7615ae03.pth©Ú	crop_sizeÚresize_sizeÚmeanr‘   ÚrecipeúFhttps://github.com/SwinTransformer/Video-Swin-Transformer#kinetics-400Ú_docsúœThe weights were ported from the paper. The accuracies are estimated on video-level with parameters `frame_rate=15`, `clips_per_video=12`, and `clip_len=32`Ú
num_paramsiv¨­Ú_metricsúKinetics-400úacc@1gö(\�ÂmS@úacc@5gð§ÆK7aW@Ú_opsg7‰A`åðE@Ú
_file_sizeg˜nƒÀb^@©ÚurlÚ
transformsrü   r�   N©éà   r  ©é   ©g
×£p=
ß?gÉv¾Ÿ/Ý?g–C‹lçûÙ?©gZd;ßOÍ?gyé&1¬Ì?gÍÌÌÌÌÌÌ?©r°   r±   r²   r³   r   r   r	   Ú_COMMON_METAÚKINETICS400_V1ÚDEFAULTrµ   r�   r#   r!   r   r      ó{   † ÙØGÙØØ ØØ)Ø(ô
ð
Øð
àÐ^Øð[ð ˜(ØØØ˜VØ˜Vð!ðð �FØ˜'ñ
ô€Nð6 „Gr#   c                   ór   € ] tR tRt]! R]! ]RRRRR7      / ]CRRRRRR	R
RRRRR//RRRR/CR7      t]t	Rt
R# )r   i  z9https://download.pytorch.org/models/swin3d_s-da41c237.pthr  r  r	  r
  r  r  if$ør  r  r  gÓMbXáS@r  g'1¬ŠW@r  gçû©ñÒµT@r  gð§ÆK7Ik@r  r�   Nr  r  r  r  r  r�   r#   r!   r   r     r   r#   c                   óÊ   € ] tR tRt]! R]! ]RRRRR7      / ]CRRRRRR	R
RRRRR//RRRR/CR7      t]! R]! ]RRRRR7      / ]CRRRRRR	R
RRRRR//RRRR/CR7      t	]t
RtR# )r   i>  z<https://download.pytorch.org/models/swin3d_b_1k-24f7c7c6.pthr  r  r	  r
  r  r  iX…?r  r  r  gÙÎ÷SÛS@r  gbX9´˜W@r  gÓMbX•a@r  g /Ý$Âv@r  z=https://download.pytorch.org/models/swin3d_b_22k-7c6ae6fa.pthgþÔxé&iT@gÛù~j¼äW@r�   Nr  r  r  r  )r°   r±   r²   r³   r   r   r	   r  r  ÚKINETICS400_IMAGENET22K_V1r  rµ   r�   r#   r!   r   r   >  sð   † ÙØJÙØØ ØØ)Ø(ô
ð
Øð
àÐ^Øð[ð ˜(ØØØ˜VØ˜Vð!ðð �GØ˜'ñ
ô€Nñ6 ")ØKÙØØ ØØ)Ø(ô
ð
Øð
àÐ^Øð[ð ˜(ØØØ˜VØ˜Vð!ðð �GØ˜'ñ
ô"Ðð6 „Gr#   Ú
pretrained)rö   rö   r÷   c                ó^   € V ^8„  d   QhR\         \        ,          R\        R\        R\        /# ©r   rö   r÷   rø   r   )r   r   rX   r   r   )r    s   "r!   r"   r"   z  ó0   € ÷ !ñ !œÔ"2Õ3ð !Ädð !Ô]`ð !Ôevñ !r#   c                ór   € \         P                  V 4      p \        R
R. ROR^`R. ROR. ROR. RORRRV R	V/VB # )as  
Constructs a swin_tiny architecture from
`Video Swin Transformer <https://arxiv.org/abs/2106.13230>`_.

Args:
    weights (:class:`~torchvision.models.video.Swin3D_T_Weights`, optional): The
        pretrained weights to use. See
        :class:`~torchvision.models.video.Swin3D_T_Weights` below for
        more details, and possible values. By default, no pre-trained
        weights are used.
    progress (bool, optional): If True, displays a progress bar of the
        download to stderr. Default is True.
    **kwargs: parameters passed to the ``torchvision.models.video.swin_transformer.SwinTransformer``
        base class. Please refer to the `source code
        <https://github.com/pytorch/vision/blob/main/torchvision/models/video/swin_transformer.py>`_
        for more details about this class.

.. autoclass:: torchvision.models.video.Swin3D_T_Weights
    :members:
r:   r½   rÑ   rR   r   rÓ   rÏ   rö   r÷   r�   ©r   é   r*  )r   r   é   r   ©r&   r+  é   é   ©é   é   r1  )r   Úverifyr   ©rö   r÷   rø   s   $$,r!   Úswin3d_tr4  x  sw   € ô. ×%Ñ% gÓ.€Gäñ 
Úð
àð
ò ð
ò !ð	
ò
 ð
ð "ð
ð ð
ð ð
ð ñ
ð 
r#   c                ó^   € V ^8„  d   QhR\         \        ,          R\        R\        R\        /# r&  )r   r   rX   r   r   )r    s   "r!   r"   r"      r'  r#   c                ór   € \         P                  V 4      p \        R
R. ROR^`R. ROR. ROR. RORRRV R	V/VB # )at  
Constructs a swin_small architecture from
`Video Swin Transformer <https://arxiv.org/abs/2106.13230>`_.

Args:
    weights (:class:`~torchvision.models.video.Swin3D_S_Weights`, optional): The
        pretrained weights to use. See
        :class:`~torchvision.models.video.Swin3D_S_Weights` below for
        more details, and possible values. By default, no pre-trained
        weights are used.
    progress (bool, optional): If True, displays a progress bar of the
        download to stderr. Default is True.
    **kwargs: parameters passed to the ``torchvision.models.video.swin_transformer.SwinTransformer``
        base class. Please refer to the `source code
        <https://github.com/pytorch/vision/blob/main/torchvision/models/video/swin_transformer.py>`_
        for more details about this class.

.. autoclass:: torchvision.models.video.Swin3D_S_Weights
    :members:
r:   r½   rÑ   rR   r   rÓ   rÏ   rö   r÷   r�   r)  ©r   r   é   r   r,  r/  )r   r2  r   r3  s   $$,r!   Úswin3d_sr9  ž  sw   € ô. ×%Ñ% gÓ.€Gäñ 
Úð
àð
ò ð
ò !ð	
ò
 ð
ð "ð
ð ð
ð ð
ð ñ
ð 
r#   c                ó^   € V ^8„  d   QhR\         \        ,          R\        R\        R\        /# r&  )r   r   rX   r   r   )r    s   "r!   r"   r"   Æ  r'  r#   c                ór   € \         P                  V 4      p \        R
R. ROR^€R. ROR. ROR. RORRRV R	V/VB # )as  
Constructs a swin_base architecture from
`Video Swin Transformer <https://arxiv.org/abs/2106.13230>`_.

Args:
    weights (:class:`~torchvision.models.video.Swin3D_B_Weights`, optional): The
        pretrained weights to use. See
        :class:`~torchvision.models.video.Swin3D_B_Weights` below for
        more details, and possible values. By default, no pre-trained
        weights are used.
    progress (bool, optional): If True, displays a progress bar of the
        download to stderr. Default is True.
    **kwargs: parameters passed to the ``torchvision.models.video.swin_transformer.SwinTransformer``
        base class. Please refer to the `source code
        <https://github.com/pytorch/vision/blob/main/torchvision/models/video/swin_transformer.py>`_
        for more details about this class.

.. autoclass:: torchvision.models.video.Swin3D_B_Weights
    :members:
r:   r½   rÑ   rR   r   rÓ   rÏ   rö   r÷   r�   r)  r7  )r*  r0  é   é    r/  )r   r2  r   r3  s   $$,r!   Úswin3d_br>  Ä  sw   € ô. ×%Ñ% gÓ.€Gäñ 
Úð
àð
ò ð
ò !ð	
ò
 ð
ð "ð
ð ð
ð ð
ð ñ
ð 
r#   )r   r   r   r   r4  r9  r>  )rA   rA   NNT)r_   r_   )1Ú	functoolsr   Útypingr   r   r   r.   Útorch.nn.functionalr   Ú
functionalrb   r   Útransforms._presetsr	   Úutilsr
   Ú_apir   r   r   Ú_metar   Ú_utilsr   r   Úswin_transformerr   r   Ú__all__r*   ÚfxÚwrapr8   r=   rM   rx   r¿   rz   rº   r   r   r  r   r   r   r  r4  r9  r>  r�   r#   r!   Ú<module>rL     sì  ðõ ß *Ñ *ã ß Ð ß å 6å (ç 7Ñ 7å +ß Cß Aò€õ	#ð ‡�‡�Ð*Ô +õ
"ð ‡�‡�Ð+Ô ,õ1ð
 ‡�‡�Ð$Ô %õ&ðR ‡�‡�Ð*Ô +÷mð` ‡�‡�Ð+Ô ,ôV
˜rŸy™yô V
ôv*�2—9‘9ô *ôZj˜Ÿ	™	ô jõZð> Ð)Ø�Ø˜ð€ô�{ô ô>�{ô ô>7�{ô 7ñt ÓÙ ,Ð0@×0OÑ0OÐ!PÔQð!°dð !ÈTö !ó Ró ð!ñH ÓÙ ,Ð0@×0OÑ0OÐ!PÔQð!°dð !ÈTö !ó Ró ð!ñH ÓÙ ,Ð0@×0OÑ0OÐ!PÔQð!°dð !ÈTö !ó Ró ò!r#   