+
    õäûja¡  ã                  óà  € R t ^ RIHt ^ RIt^ RIt^ RIt^ RIHt ^ RI	Hu H
t ^ RIHt ^ RIHtHt R tR t ! R R]P&                  4      t ! R	 R
]P&                  4      tR tR t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t  ! R R ]PB                  4      t" ! R! R"]PB                  4      t#]! 4        ! R# R$]P&                  4      4       t$R# )%z¸
An official Pytorch impl of `Transcending the Limit of Local Window:
Advanced Super-Resolution Transformer with Adaptive Token Dictionary`.

Arxiv: 'https://arxiv.org/abs/2401.08209'
)ÚannotationsN)Ústore_hyperparameters)Ú	to_2tupleÚtrunc_normal_c                ó  € \         P                  ! V 4      p\         P                  ! ^ V P                  R,          4      P	                  V P
                  4      p\        V P                  ^ ,          4       F  pW!W0VR3,          3&   K  	  V# )é    ºNNNéÿÿÿÿ)ÚtorchÚ
zeros_likeÚarangeÚshapeÚtoÚdeviceÚrange)ÚindexÚindex_rÚindÚis   &   ÚfC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\spandrel\architectures\ATD\__arch\atd_arch.pyÚindex_reverser      sg   € Ü×Ò˜uÓ%€GÜ
�,Š,�q˜%Ÿ+™+ b�/Ó
*×
-Ñ
-¨e¯l©lÓ
;€CÜ�5—;‘;˜q•>Ö"ˆØ"%�˜˜A˜•;�Óñ #à€Nó    c                ó¦  € VP                  4       pV P                  R V VP                  8X  g#   Q RV P                   RVP                   R24       h\        V P                  4       VP                  4       ,
          4       F  pVP                  R4      pK  	  VP	                  V P                  4      p\
        P                  ! W^,
          VR7      pV# )Nzx (z) and index (z) shape incompatible)Údimr   r	   )r   r   r   Ú	unsqueezeÚexpandr
   Úgather)Úxr   r   Ú_Ú
shuffled_xs   &&   r   Úfeature_shuffler       s©   € Ø
�)‰)‹+€Cà	�‰��ˆ˜Ÿ™Ô$ðEà	ˆQ�W‰WˆI�] 5§;¡; -Ð/CÐDóEØ$ô �1—5‘5“7˜UŸY™Y›[Õ(Ö)ˆØ—‘ Ó#Šñ *à�L‰L˜Ÿ™Ó!€Eä—’˜a¨1¥W°EÔ:€JØÐr   c                  ó2   a € ] tR t^,tRV 3R lltR tRtV ;t# )Údwconvc                	óà   <€ \         SV `  4        \        P                  ! \        P                  ! VVV^V^,
          ^,          ^VR7      \        P
                  ! 4       4      V n        Wn        R# )é   )Úkernel_sizeÚstrideÚpaddingÚdilationÚgroupsN)ÚsuperÚ__init__ÚnnÚ
SequentialÚConv2dÚGELUÚdepthwise_convÚhidden_features)Úselfr1   r%   Ú	__class__s   &&&€r   r+   Údwconv.__init__-   s\   ø€ Ü‰ÑÔÜ ŸmšmÜ�IŠIØØØ'ØØ$ q�¨QÕ.ØØ&ôô �GŠG‹Ió
ˆÔð  /Ör   c                	ó<  € VP                  ^^4      P                  VP                  ^ ,          V P                  V^ ,          V^,          4      P	                  4       pV P                  V4      pVP                  ^4      P                  ^^4      P	                  4       pV# ©r$   )Ú	transposeÚviewr   r1   Ú
contiguousr0   Úflatten©r2   r   Úx_sizes   &&&r   ÚforwardÚdwconv.forward=   s|   € à�K‰K˜˜1Óß‰T�!—'‘'˜!•*˜d×2Ñ2°F¸1µI¸vÀa½yÓIß‰Z‹\ð 	
ð
 ×Ñ Ó"ˆØ�I‰I�a‹L×"Ñ" 1 aÓ(×3Ñ3Ó5ˆØˆr   )r0   r1   )é   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r+   r=   Ú__static_attributes__Ú__classcell__©r3   s   @r   r"   r"   ,   s   ø† ÷/÷ ð r   r"   c                  óN   a € ] tR t^HtRR^]P
                  3V 3R lltR tRtV ;t	# )ÚConvFFNNc                	óú   <€ \         SV `  4        T;'       g    TpT;'       g    Tp\        P                  ! W4      V n        V! 4       V n        \        W$R 7      V n        \        P                  ! W#4      V n        R# ))r1   r%   N)r*   r+   r,   ÚLinearÚfc1Úactr"   Úfc2)r2   Úin_featuresr1   Úout_featuresr%   Ú	act_layerr3   s   &&&&&&€r   r+   ÚConvFFN.__init__I   s_   ø€ ô 	‰ÑÔØ#×2Ð2 {ˆØ)×8Ð8¨[ˆÜ—9’9˜[Ó:ˆŒÙ“;ˆŒÜ¨_ÔVˆŒÜ—9’9˜_Ó;ˆŽr   c                	óš   € V P                  V4      pV P                  V4      pWP                  W4      ,           pV P                  V4      pV# ©N)rK   rL   r"   rM   r;   s   &&&r   r=   ÚConvFFN.forwardY   s?   € Ø�H‰H�Q‹KˆØ�H‰H�Q‹KˆØ—‘˜AÓ&Õ&ˆØ�H‰H�Q‹KˆØˆr   )rL   r"   rK   rM   )
r@   rA   rB   rC   r,   r/   r+   r=   rD   rE   rF   s   @r   rH   rH   H   s$   ø† ð ØØØ—'‘'÷<÷ ð r   rH   c                óÐ   € V P                   w  r#rEV P                  W#V,          WV,          W4      p V P                  ^ ^^^^^4      P                  4       P                  RWV4      pV# )z‚
Args:
    x: (b, h, w, c)
    window_size (int): window size

Returns:
    windows: (num_windows*b, window_size, window_size, c)
r	   )r   r8   Úpermuter9   )r   Úwindow_sizeÚbÚhÚwÚcÚwindowss   &&     r   Úwindow_partitionr]   a   sh   € ð —‘�J€Aˆ!Ø	�‰ˆq�{Õ" K°kÕ1AÀ;ÓR€Aà	�	‰	�!�Q˜˜1˜a Ó#×.Ñ.Ó0×5Ñ5°b¸+ÐTUÓVð ð €Nr   c                ó   € \        V P                  ^ ,          W#,          V,          V,          ,          4      pV P                  WBV,          W1,          WR4      pVP                  ^ ^^^^^4      P	                  4       P                  WBVR4      pV# )z»
Args:
    windows: (num_windows*b, window_size, window_size, c)
    window_size (int): Window size
    h (int): Height of image
    w (int): Width of image

Returns:
    x: (b, h, w, c)
r	   )Úintr   r8   rV   r9   )r\   rW   rY   rZ   rX   r   s   &&&&  r   Úwindow_reverser`   r   s~   € ô 	ˆG�M‰M˜!Õ ¥¨Õ 3°kÕ AÕBÓC€AØ�‰Ø	�Õ˜QÕ-¨{Èó	€Að 	
�	‰	�!�Q˜˜1˜a Ó#×.Ñ.Ó0×5Ñ5°a¸A¸rÓB€AØ€Hr   c                  óF   a € ] tR t^…tRtRV 3R lltRR ltR R ltRtV ;t	# )	ÚWindowAttentiona4  
Shifted Window-based Multi-head Self-Attention

Args:
    dim (int): Number of input channels.
    window_size (tuple[int]): The height and width of the window.
    num_heads (int): Number of attention heads.
    qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
c                	óÞ  <€ \         SV `  4        Wn        W n        W0n        W@n        W,          pVR,          V n        \        P                  ! \        P                  ! ^V^ ,          ,          ^,
          ^V^,          ,          ^,
          ,          V4      4      V n        \        P                  ! W4      V n        \        V P                  RR7       \        P                  ! RR7      V n        R# )ç      à?ç{®Gáz”?©Ústd©r   Ng      à¿r	   )r*   r+   r   rW   Ú	num_headsÚqkv_biasÚscaler,   Ú	Parameterr
   ÚzerosÚrelative_position_bias_tablerJ   Úprojr   ÚSoftmaxÚsoftmax)r2   r   rW   ri   rj   Úhead_dimr3   s   &&&&& €r   r+   ÚWindowAttention.__init__�   sª   ø€ Ü‰ÑÔØŒØ&ÔØ"ŒØ ŒØÕ#ˆØ˜t•^ˆŒ
ô -/¯LªLÜ�KŠK˜˜[¨�^Õ+¨aÕ/°A¸ÀA½Õ4FÈÕ4JÕKÈYÓWó-
ˆÔ)ô —I’I˜cÓ'ˆŒ	ä�d×7Ñ7¸TÕBÜ—z’z bÔ)ˆŽr   c                ój  € VP                   w  rEpV^,          pVP                  WE^V P                  WpP                  ,          4      P                  ^^ ^^^4      pV^ ,          V^,          V^,          r©pW€P                  ,          pW‰P                  RR4      ,          pV P                  VP                  R4      ,          P                  V P                  ^ ,          V P                  ^,          ,          V P                  ^ ,          V P                  ^,          ,          R4      pVP                  ^^ ^4      P                  4       pW¼P                  ^ 4      ,           pVe‹   VP                   ^ ,          pVP                  WM,          WÐP                  WU4      VP                  ^4      P                  ^ 4      ,           pVP                  RV P                  WU4      pV P                  V4      pMV P                  V4      pWº,          P                  ^^4      P                  WEV4      pV P                  V4      pV# )zÅ
Args:
    qkv: Input query, key, and value tokens with shape of (num_windows*b, n, c*3)
    rpi: Relative position index
    mask (0/-inf):  Mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None
éþÿÿÿr	   )r   Úreshaperi   rV   rk   r7   rn   r8   rW   r9   r   rq   ro   )r2   ÚqkvÚrpiÚmaskÚb_ÚnÚc3r[   ÚqÚkÚvÚattnÚrelative_position_biasÚnwr   s   &&&&           r   r=   ÚWindowAttention.forward£   s×  € ð —I‘I‰	ˆˆrØ�!�GˆØ�k‰k˜"  D§N¡N°A¿¹Õ4GÓH×PÑPØˆq�!�Q˜ó
ˆð ��FØ��FØ��Fð ˆð —
‘
�NˆØ—;‘;˜r 2Ó&Õ&ˆà!%×!BÑ!BÀ3Ç8Á8ÈBÃ<Õ!P×!UÑ!UØ×Ñ˜QÕ $×"2Ñ"2°1Õ"5Õ5Ø×Ñ˜QÕ $×"2Ñ"2°1Õ"5Õ5Øó"
Ðð
 "8×!?Ñ!?Øˆq�!ó"
ç
‰*‹,ð 	ð ×6Ñ6°qÓ9Õ9ˆàÒØ—‘˜A•ˆBØ—9‘9˜R�X r¯>©>¸1Ó@À4Ç>Á>ØóDç‰i˜‹lõˆDð —9‘9˜R §¡°Ó6ˆDØ—<‘< Ó%‰Dà—<‘< Ó%ˆDà�X× Ñ   AÓ&×.Ñ.¨r°aÓ8ˆØ�I‰I�a‹LˆØˆr   c               ó   € V ^8„  d   QhRR/# ©é   ÚreturnÚstr© )Úformats   "r   Ú__annotate__ÚWindowAttention.__annotate__Ð   s   € ÷ wñ w˜Cñ wr   c                	ón   € R V P                    RV P                   RV P                   RV P                   2# )údim=z, window_size=z, num_heads=z, qkv_bias=)r   rW   ri   rj   ©r2   s   &r   Ú
extra_reprÚWindowAttention.extra_reprÐ   s:   € Ø�d—h‘h�Z˜~¨d×.>Ñ.>Ð-?¸|ÈDÏNÉNÐK[Ð[fÐgk×gtÑgtÐfuÐvÐvr   )r   ri   ro   rj   rn   rk   rq   rW   )TrS   )
r@   rA   rB   rC   Ú__doc__r+   r=   r�   rD   rE   rF   s   @r   rb   rb   …   s   ø† ñ÷*ô&+÷Zwó wr   rb   c                  ó6   a € ] tR t^ÔtRtRV 3R lltR tRtV ;t# )ÚATD_CAa¥  
Adaptive Token Dictionary Cross-Attention.

Args:
    dim (int): Number of input channels.
    input_resolution (tuple[int]): Input resolution.
    num_tokens (int): Number of tokens in external token dictionary. Default: 64
    reducted_dim (int, optional): Reducted dimension number for query and key matrix. Default: 4
    qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
c                	óÌ  <€ \         SV `  4        Wn        W n        W0n        W@n        WPn        \        P                  ! WVR 7      V n	        \        P                  ! WVR 7      V n
        \        P                  ! WVR 7      V n        \        P                  ! \        P                  ! V P                  .4      R,          RR7      V n        \        P                   ! RR7      V n        R# )©Úbiasrd   T©Úrequires_gradrh   Nr	   )r*   r+   r   Úinput_resolutionÚ
num_tokensÚrcrj   r,   rJ   ÚwqÚwkÚwvrl   r
   Úonesrk   rp   rq   )r2   r   rš   r›   Úreducted_dimrj   r3   s   &&&&&&€r   r+   ÚATD_CA.__init__à   sœ   ø€ ô 	‰ÑÔØŒØ 0ÔØ$ŒØŒØ Œä—)’)˜C°HÔ=ˆŒÜ—)’)˜C°HÔ=ˆŒÜ—)’)˜C¨8Ô4ˆŒä—\’\Ü�JŠJ˜Ÿ™Ð(Ó)¨CÕ/¸tô
ˆŒ
ô —z’z bÔ)ˆŽr   c                ó4  € VP                   w  rEpVP                   w  rGpV P                  V4      pV P                  V4      p	V P                  V4      p
\        P
                  ! VRR7      \        P
                  ! V	RR7      P                  RR4      ,          p\        P                  ! V P                  ^ ^4      pV^V\        P                  ! V P                  4      ,          ,           ,          pV P                  V4      pWº,          P                  WEV4      pW3# )zŽ
Args:
    x: input features with shape of (b, n, c)
    td: token dicitionary with shape of (b, m, c)
    x_size: size of the input x (h, w)
rh   r	   ru   )r   r�   rž   rŸ   ÚFÚ	normalizer7   r
   Úclamprk   ÚnpÚlogr›   rq   rv   )r2   r   Útdr<   rX   r{   r[   Ú_mr}   r~   r   r€   rk   s   &&&&         r   r=   ÚATD_CA.forwardó   sß   € ð —'‘'‰ˆˆaØ—8‘8‰ˆˆqð �G‰G�A‹Jˆà�G‰G�B‹Kˆà�G‰G�B‹Kˆô �{Š{˜1 "Ô%¬¯ª°A¸2Ô(>×(HÑ(HØ�ó)
õ 
ˆô —’˜DŸJ™J¨¨1Ó-ˆØ�q˜5¤2§6¢6¨$¯/©/Ó#:Õ:Õ:Õ;ˆØ�|‰|˜DÓ!ˆð �X×Ñ˜q QÓ'ˆàˆwˆr   )
r   rš   r›   rj   rœ   rk   rq   rž   r�   rŸ   )é@   é
   T©	r@   rA   rB   rC   r’   r+   r=   rD   rE   rF   s   @r   r”   r”   Ô   s   ø† ñ	÷*÷&ð r   r”   c                  ó6   a € ] tR tRtRtRV 3R lltR tRtV ;t# )ÚAC_MSAi  aâ  
Adaptive Category-based Multihead Self-Attention.

Args:
    dim (int): Number of input channels.
    input_resolution (tuple[int]): Input resolution.
    num_tokens (int): Number of tokens in external dictionary. Default: 64
    num_heads (int): Number of attention heads. Default: 4
    category_size (int): Number of tokens in each group for global sparse attention. Default: 128
    qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
c                	ój  <€ \         SV `  4        Wn        W n        W0n        W@n        WPn        \        P                  ! WVR 7      V n	        \        P                  ! \        P                  ! ^
\        P                  ! R4      ,          4      RR7      V n        \        P                  ! RR7      V n        R# )r–   Tr˜   rh   N)r$   r$   r	   )r*   r+   r   rš   r›   ri   Úcategory_sizer,   rJ   ro   rl   r
   r¨   r    Úlogit_scalerp   rq   )r2   r   rš   r›   ri   r²   rj   r3   s   &&&&&&&€r   r+   ÚAC_MSA.__init__  s~   ø€ ô 	‰ÑÔØŒØ 0ÔØ$ŒØ"ŒØ*Ôô —I’I˜c¨XÔ6ˆŒ	äŸ<š<Ü�IŠI�bœ5Ÿ:š: fÓ-Õ-Ó.¸dô
ˆÔô —z’z bÔ)ˆŽr   c           	     óÆ  € VP                   w  rEpV^,          pVP                   w  rEp\        WPP                  4      p	WY,           ^,
          V	,          p
\        P                  ! VR	RR7      p\        P
                  ! VR	RR7      w  rÍ\        V4      p\        W4      pW©,          V,
          p\        P                  ! V\        P                  ! VRVV,
          V1R3,          ^.R7      3^R7      pVP                  VR	W–4      pVP                  WJV	^V P                  WpP                  ,          4      P                  ^^ ^^^^4      pV^ ,          V^,          V^,          pppVVP                  R
R	4      ,          p\        P                  ! V P                  \        P                   ! \        P"                  ! R4      4      P%                  VP&                  4      R7      P)                  4       pVV,          pV P+                  V4      pVV,          P                  ^ ^^^^4      P                  WEV,           V4      RRV1R3,          p\        VV4      pV P-                  V4      pV# )zˆ
Args:
    x: input features with shape of (b, HW, c)
    mask: similarity map with shape of (b, HW, m)
    x_size: size of the input x
F)r   Úkeepdim)r   Ústabler   )Údimsrh   )ÚmaxNr	   ru   g      Y@)r   Úminr²   r
   ÚargmaxÚsortr   r    ÚcatÚfliprv   ri   rV   r7   r¦   r³   r¨   Útensorr   r   Úexprq   ro   )r2   rw   Úsimr<   rX   r{   r|   r[   rª   ÚgsÚngÚtk_idÚ_x_sort_valuesÚx_sort_indicesÚx_sort_indices_reverseÚshuffled_qkvÚpad_nÚ	paded_qkvÚyr}   r~   r   r€   r³   r   s   &&&&                     r   r=   ÚAC_MSA.forward7  s  € ð —9‘9‰ˆˆbØ�!�GˆØ—9‘9‰ˆˆbÜ�×&Ñ&Ó'ˆØ�f�q�j˜RÕˆô —’˜S b°%Ô8ˆä).¯ª°E¸rÈ%Ô)PÑ&ˆÜ!.¨~Ó!>ÐÜ& sÓ;ˆØ•˜!•ˆÜ—I’IØœ5Ÿ:š: l°1°a¸%µiÀ!°mÀQÐ3FÕ&GÈqÈcÔRÐSØô
ˆ	ð ×Ñ˜a  RÓ,ˆà�i‰i˜˜r 1 d§n¡n°a¿>¹>Õ6IÓJ×RÑRØˆq�!�Q˜˜1ó
ˆð �a•&˜#˜a�& # a¥&ˆaˆ1ˆð �1—;‘;˜r 2Ó&Õ&ˆä—k’kØ×Ñ¤%§)¢)¬E¯LªL¸Ó,DÓ"E×"HÑ"HÈÏÉÓ"Tô
ç
‰#‹%ð 	ð �kÕ!ˆð �|‰|˜DÓ!ˆð �A�X×Ñ˜q ! Q¨¨1Ó-×5Ñ5°a¸U½ÀAÓFÀqÈ"È1È"ÈaÀxÕPˆä˜AÐ5Ó6ˆØ�I‰I�a‹Lˆàˆr   )r²   r   rš   r³   ri   r›   ro   rq   )r¬   é   é€   Tr®   rF   s   @r   r°   r°     s   ø† ñ
÷*÷01ð 1r   r°   c                  óf   a € ] tR tRtRtR]P                  ]P                  R3V 3R lltR t	Rt
V ;t# )ÚATDTransformerLayerik  aª  
ATD Transformer Layer

Args:
    dim (int): Number of input channels.
    idx (int): Layer index.
    input_resolution (tuple[int]): Input resolution.
    num_heads (int): Number of attention heads.
    window_size (int): Window size.
    shift_size (int): Shift size for SW-MSA.
    category_size (int): Category size for AC-MSA.
    num_tokens (int): Token number for each token dictionary.
    reducted_dim (int): Reducted dimension number for query and key matrix.
    convffn_kernel_size (int): Convolutional kernel size for ConvFFN.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
    qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
    act_layer (nn.Module, optional): Activation layer. Default: nn.GELU
    norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
    is_last (bool): True if this layer is the last of a ATD Block. Default: False
TFc           	     	ó”  <€ \         SV `  4        Wn        W0n        W@n        WPn        W`n        W°n        W n        W€n	        \        P                  ! RR7      V n        \        P                  ! 4       V n        \        P                  ! 4       V n        W�n        Wðn        V! V4      V n        V! V4      V n        V'       gQ   \        P*                  ! VRR7      V n        \        P.                  ! \0        P2                  ! V^.4      RR7      V n        \        P6                  ! V^V,          VR7      V n        \;        V P                  \=        V P
                  4      VVR7      V n        \A        V P                  VVVV	R7      V n!        \E        V P                  VVVVVR7      V n#        \I        W,          4      p\K        VVV
VR	7      V n&        R
# )r$   rh   T)Úaffiner˜   r–   )rW   ri   rj   )rš   rj   r›   r¡   )rš   r›   ri   r²   rj   )rN   r1   r%   rP   Nr	   )'r*   r+   r   rš   ri   rW   Ú
shift_sizeÚ	mlp_ratioÚconvffn_kernel_sizer›   r,   rp   rq   Ú	LeakyReLUÚlreluÚSigmoidÚsigmoidr¡   Úis_lastÚnorm1Únorm2ÚInstanceNorm1dÚnorm3rl   r
   rm   ÚsigmarJ   Úwqkvrb   r   Úattn_winr”   Úattn_atdr°   Úattn_acar_   rH   Úconvffn)r2   r   Úidxrš   ri   rW   rÓ   r²   r›   r¡   rÕ   rÔ   rj   rP   Ú
norm_layerrÚ   Úmlp_hidden_dimr3   s   &&&&&&&&&&&&&&&& €r   r+   ÚATDTransformerLayer.__init__�  sg  ø€ ô$ 	‰ÑÔàŒØ 0ÔØ"ŒØ&ÔØ$ŒØ"ŒØ#6Ô Ø$ŒÜ—z’z bÔ)ˆŒÜ—\’\“^ˆŒ
Ü—z’z“|ˆŒØ(ÔØŒá “_ˆŒ
Ù “_ˆŒ
ßÜ×*Ò*¨:¸dÔCˆDŒJÜŸš¤e§k¢k°:¸q°/Ó&BÐRVÔWˆDŒJä—I’I˜c 1 s¥7°Ô:ˆŒ	ä'Ø�H‰HÜ! $×"2Ñ"2Ó3ØØô	
ˆŒô Ø�H‰HØ-ØØ!Ø%ô
ˆŒô Ø�H‰HØ-Ø!ØØ'Øô
ˆŒô ˜S�_Ó-ˆÜØØ*Ø+Øô	
ˆŽr   c                	ó~  € Vw  rVVP                   w  rxp	^V	,          p
TpV P                  V4      pV P                  V4      pV P                  WV4      w  rÞV P	                  WÎV4      pVP                  WuWj4      pV P                  ^ 8”  d;   \        P                  ! WÀP                  ) V P                  ) 3RR7      pVR,          pMTpRp\        VV P                  4      pVP                  RV P                  V P                  ,          V
4      pV P                  VVR,          VR7      pVP                  RV P                  V P                  V	4      p\        VV P                  WV4      pV P                  ^ 8”  d1   \        P                  ! VV P                  V P                  3RR7      pMTpTpVVP                  WxV	4      ,           V,           V,           pWP                  V P                  V4      V4      ,           pVP                   w  ppp	VP                   w  rxp	V P                   '       g“   V P#                  V P%                  VP'                  RR	4      4      4      pVP                  VVV	4      pV P)                  V P*                  4      pVV,          ^V,
          \        P,                  ! RVV4      ,          ,           pW3# )
é   )Úshiftsr¸   Ú	attn_maskNÚrpi_sa)rx   ry   zbtn,bnc->btc)r$   r†   r	   ru   )r   rÛ   rà   râ   rã   rv   rÓ   r
   Úrollr]   rW   r8   rá   r`   rä   rÜ   rÚ   rq   rÞ   r7   rÙ   rß   Úeinsum)r2   r   r©   r<   ÚparamsrY   rZ   rX   r{   r[   r|   Úshortcutrw   Úx_atdÚsim_atdÚx_acaÚshifted_qkvrì   Ú	x_windowsÚattn_windowsÚ	shifted_xÚattn_xÚx_winÚNÚ	mask_softÚmask_xÚss   &&&&&                      r   r=   ÚATDTransformerLayer.forwardÉ  sh  € Ø‰ˆØ—'‘'‰ˆˆaØ��UˆàˆØ�J‰J�q‹MˆØ�i‰i˜‹lˆð Ÿ™Ø�6ó
‰ˆð
 —‘˜c¨FÓ3ˆð �k‰k˜! Ó&ˆð �?‰?˜QÔÜŸ*š*ØŸo™oÐ-°·±Ð/?Ð@ÀvôˆKð ˜{Õ+‰IàˆKØˆIô %Ø˜×)Ñ)ó
ˆ	ð —N‘NØ�× Ñ  4×#3Ñ#3Õ3°Ró
ˆ	ð
 —}‘} Y°F¸8Õ4DÈ9�}ÓUˆð $×(Ñ(¨¨T×-=Ñ-=¸t×?OÑ?OÐQRÓSˆÜ" <°×1AÑ1AÀ1ÓHˆ	ð �?‰?˜QÔÜ—Z’ZØ 4§?¡?°D·O±OÐ"DÈ6ô‰Fð ˆFØˆØ�u—z‘z !¨Ó*Õ*¨UÕ2°UÕ:ˆð —‘˜TŸZ™Z¨›]¨FÓ3Õ3ˆà—'‘'‰ˆˆ1ˆaØ—(‘(‰ˆˆað �|�|ˆ|ØŸ™ T§Z¡Z°×0AÑ0AÀ"ÀbÓ0IÓ%JÓKˆIØ—Y‘Y˜q ! QÓ'ˆFØ—‘˜TŸZ™ZÓ(ˆAØ�R•˜1˜q�5¤E§L¢L°ÀÈFÓ$SÕSÕSˆBàˆuˆr   )rã   râ   rá   rä   rÕ   r   rš   rÚ   r×   rÔ   rÛ   rÜ   rÞ   ri   r›   r¡   rÓ   rß   rÙ   rq   rW   rà   )r@   rA   rB   rC   r’   r,   r/   Ú	LayerNormr+   r=   rD   rE   rF   s   @r   rÐ   rÐ   k  s2   ø† ñðD Ø—'‘'Ø—<‘<Ø÷!F
÷PDð Dr   rÐ   c                  óX   a € ] tR tRtRt]P                  3V 3R lltR tR R lt	Rt
V ;t# )ÚPatchMergingi  z×Patch Merging Layer.

Args:
    input_resolution (tuple[int]): Resolution of input feature.
    dim (int): Number of input channels.
    norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
c                	ó¼   <€ \         SV `  4        Wn        W n        \        P
                  ! ^V,          ^V,          RR7      V n        V! ^V,          4      V n        R# )rÍ   Fr–   N)r*   r+   rš   r   r,   rJ   Ú	reductionÚnorm)r2   rš   r   ræ   r3   s   &&&&€r   r+   ÚPatchMerging.__init__  sE   ø€ Ü‰ÑÔØ 0ÔØŒÜŸš 1 s¥7¨A°­G¸%Ô@ˆŒÙ˜q 3�wÓ'ˆŽ	r   c                óÒ  € V P                   w  r#VP                  w  rEpWRV,          8X  g   Q R4       hV^,          ^ 8X  d   V^,          ^ 8X  g   Q RV RV R24       hVP                  WBW64      pVR,          pVR,          pVR,          p	VR,          p
\        P                  ! WxWš.R	4      pVP                  VR	^V,          4      pV P                  V4      pV P                  V4      pV# )
z
x: b, h*w, c
zinput feature has wrong sizezx size (Ú*z) are not even.)r   ºr   Nr†   r	  r   )r   ºr$   Nr†   r	  r   )r   r	  r
  r   )r   r
  r
  r   r	   )rš   r   r8   r
   r½   r  r  )r2   r   rY   rZ   rX   Úseq_lenr[   Úx0Úx1Úx2Úx3s   &&         r   r=   ÚPatchMerging.forward   sê   € ð ×$Ñ$‰ˆØŸ™‰ˆ�AØ˜a�%ÔÐ?Ð!?Ó?ÐØ�1�u˜Œz˜a !�e qœjÐK¨H°Q°C°q¸¸¸?Ð*KÓKÐ(à�F‰F�1˜ÓˆàÐÕ ˆØÐÕ ˆØÐÕ ˆØÐÕ ˆÜ�IŠI�r˜rÐ&¨Ó+ˆØ�F‰F�1�b˜!˜a�%Ó ˆà�I‰I�a‹LˆØ�N‰N˜1Óˆàˆr   c               ó   € V ^8„  d   QhRR/# r…   r‰   )rŠ   s   "r   r‹   ÚPatchMerging.__annotate__7  s   € ÷ Kñ K˜Cñ Kr   c                	ó:   € R V P                    RV P                   2# )zinput_resolution=z, dim=)rš   r   r�   s   &r   r�   ÚPatchMerging.extra_repr7  s    € Ø" 4×#8Ñ#8Ð"9¸ÀÇÁ¸zÐJÐJr   )r   rš   r  r  ©r@   rA   rB   rC   r’   r,   r   r+   r=   r�   rD   rE   rF   s   @r   r  r    s'   ø† ñð :<¿¹÷ (ò÷.Kó Kr   r  c                  ó^   a € ] tR tRtRtRR]P                  R3V 3R lltR tR R	 lt	R
t
V ;t# )Ú
BasicBlocki;  aÒ  A basic ATD Block for one stage.

Args:
    dim (int): Number of input channels.
    input_resolution (tuple[int]): Input resolution.
    idx (int): Block index.
    depth (int): Number of blocks.
    num_heads (int): Number of attention heads.
    window_size (int): Local window size.
    category_size (int): Category size for AC-MSA.
    num_tokens (int): Token number for each token dictionary.
    reducted_dim (int): Reducted dimension number for query and key matrix.
    convffn_kernel_size (int): Convolutional kernel size for ConvFFN.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
    qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
    norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
    downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
    use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
g      @TNc                	óê  <€ \         SV `  4        Wn        W n        W@n        W0n        \        P                  ! 4       V n        \        V4       FU  pV P                  P                  \        TTTTTV^,          ^ 8X  d   ^ MV^,          VVV	V
VVVWô^,
          8H  R7      4       KW  	  Ve   V! W!VR7      V n        MRV n        \        P                  ! \        P                  ! W�.4      RR7      V n        R# )r†   )r   rå   rš   ri   rW   rÓ   r²   r›   rÕ   r¡   rÔ   rj   ræ   rÚ   N)r   ræ   Tr˜   )r*   r+   r   rš   Údepthrå   r,   Ú
ModuleListÚlayersr   ÚappendrÐ   Ú
downsamplerl   r
   Úrandnr©   )r2   r   rš   rå   r  ri   rW   r²   r›   rÕ   r¡   rÔ   rj   ræ   r  r   r3   s   &&&&&&&&&&&&&&& €r   r+   ÚBasicBlock.__init__P  sÝ   ø€ ô" 	‰ÑÔØŒØ 0ÔØŒ
ØŒä—m’m“oˆŒÜ�u–ˆAØ�K‰K×ÑÜ#ØØØ%5Ø'Ø +Ø%&¨¥U¨a¤Z™q°kÀQÕ6FØ"/Ø)Ø(;Ø!-Ø'Ø%Ø)Ø¨¥™Nôöñ ð* Ò!Ù(Ø °jôˆD�Oð #ˆDŒOô —,’,œuŸ{š{¨JÐ+<Ó=ÈTÔRˆŽr   c                	óÚ   € VP                   w  rEpV P                  P                  V^^.4      pV P                   F  pV! WW#4      w  rK  	  V P                  e   V P	                  V4      pV# r6   )r   r©   Úrepeatr  r  )	r2   r   r<   rð   rX   Ú_nÚ_cr©   Úlayers	   &&&&     r   r=   ÚBasicBlock.forward‡  sb   € Ø—G‘G‰	ˆˆrØ�W‰W�^‰^˜Q  1˜IÓ&ˆØ—[”[ˆEÙ˜! Ó0‰EˆA‰rñ !à�?‰?Ò&Ø—‘ Ó"ˆAØˆr   c               ó   € V ^8„  d   QhRR/# r…   r‰   )rŠ   s   "r   r‹   ÚBasicBlock.__annotate__�  s   € ÷ _ñ _˜Cñ _r   c                	óT   € R V P                    RV P                   RV P                   2# )rŽ   z, input_resolution=z, depth=)r   rš   r  r�   s   &r   r�   ÚBasicBlock.extra_repr�  s/   € Ø�d—h‘h�ZÐ2°4×3HÑ3HÐ2IÈÐRV×R\ÑR\ÐQ]Ð^Ð^r   )r  r   r  rå   rš   r  r©   r  rF   s   @r   r  r  ;  s3   ø† ñð@ ØØ—<‘<Ø÷5Sòn÷_ó _r   r  c                  óV   a € ] tR tRtRtR]P                  R^à^R3V 3R lltR tRt	V ;t
# )	ÚATDBi”  a?  Adaptive Token Dictionary Block (ATDB).

Args:
    dim (int): Number of input channels.
    input_resolution (tuple[int]): Input resolution.
    depth (int): Number of blocks.
    num_heads (int): Number of attention heads.
    window_size (int): Local window size.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
    qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
    norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
    downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
    use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
    img_size: Input image size.
    patch_size: Patch size.
    resi_connection: The convolutional block before residual connection.
TNÚ1convc                	óˆ  <€ \         SV `  4        Wn        W0n        \	        VV^ VRR7      V n        \        VV^ VRR7      V n        \        VVVVVVVVV	V
VVVVR7      V n	        VR8X  d!   \        P                  ! W^^^4      V n        R# VR8X  d±   \        P                  ! \        P                  ! W^,          ^^^4      \        P                  ! RRR7      \        P                  ! V^,          V^,          ^^^ 4      \        P                  ! RRR7      \        P                  ! V^,          V^^^4      4      V n        R# R# )	r   N©Úimg_sizeÚ
patch_sizeÚin_chansÚ	embed_dimræ   )r   rš   rå   r  ri   rW   r›   r²   r¡   rÕ   rÔ   rj   ræ   r  r,  Ú3convçš™™™™™É?T©Únegative_slopeÚinplace)r*   r+   r   rš   Ú
PatchEmbedÚpatch_embedÚPatchUnEmbedÚpatch_unembedr  Úresidual_groupr,   r.   Úconvr-   rÖ   )r2   r   rå   rš   r  ri   rW   r²   r›   r¡   rÕ   rÔ   rj   ræ   r  r/  r0  Úresi_connectionr3   s   &&&&&&&&&&&&&&&&&&€r   r+   ÚATDB.__init__§  s1  ø€ ô( 	‰ÑÔàŒØ 0Ôä%ØØ!ØØØô
ˆÔô *ØØ!ØØØô
ˆÔô )ØØ-ØØØØ#Ø!Ø'Ø%Ø 3ØØØ!Ø!ô
ˆÔð" ˜gÔ%ÜŸ	š	 #¨A¨q°!Ó4ˆDŽIØ Ô'äŸšÜ—	’	˜# a�x¨¨A¨qÓ1Ü—’¨C¸Ô>Ü—	’	˜# �( C¨1¥H¨a°°AÓ6Ü—’¨C¸Ô>Ü—	’	˜# �( C¨¨A¨qÓ1óˆDŽIñ (r   c                	ó�   € V P                  V P                  V P                  V P                  WV4      V4      4      4      V,           # rS   )r9  r=  r;  r<  )r2   r   r<   rð   s   &&&&r   r=   ÚATDB.forwardí  sH   € à×ÑØ—	‘	Ø×&Ñ& t×':Ñ':¸1ÀfÓ'MÈvÓVóóð
 õð	
r   )r=  r   rš   r9  r;  r<  )r@   rA   rB   rC   r’   r,   r   r+   r=   rD   rE   rF   s   @r   r+  r+  ”  s1   ø† ñð> Ø—<‘<ØØØØ÷%D÷L
ð 
r   r+  c                  ó6   a € ] tR tRtRtRV 3R lltR tRtV ;t# )r8  iø  a[  Image to Patch Embedding

Args:
    img_size (int): Image size.  Default: 224.
    patch_size (int): Patch token size. Default: 4.
    in_chans (int): Number of input image channels. Default: 3.
    embed_dim (int): Number of linear projection output channels. Default: 96.
    norm_layer (nn.Module, optional): Normalization layer. Default: None
c                	óT  <€ \         SV `  4        \        V4      p\        V4      pV^ ,          V^ ,          ,          V^,          V^,          ,          .pWn        W n        W`n        V^ ,          V^,          ,          V n        W0n        W@n        Ve   V! V4      V n	        R# RV n	        R# ©r   N)
r*   r+   r   r/  r0  Úpatches_resolutionÚnum_patchesr1  r2  r  ©r2   r/  r0  r1  r2  ræ   rE  r3   s   &&&&&& €r   r+   ÚPatchEmbed.__init__  s™   ø€ ô 	‰ÑÔÜ˜XÓ&ˆÜ˜zÓ*ˆ
à�Q�K˜: a�=Õ(Ø�Q�K˜: a�=Õ(ð
Ðð !ŒØ$ŒØ"4ÔØ-¨aÕ0Ð3EÀaÕ3HÕHˆÔà ŒØ"ŒàÒ!Ù" 9Ó-ˆDŽIàˆDŽIr   c                	ó†   € VP                  ^4      P                  ^^4      pV P                  e   V P                  V4      pV# ©r†   )r:   r7   r  )r2   r   s   &&r   r=   ÚPatchEmbed.forward  s8   € Ø�I‰I�a‹L×"Ñ" 1 aÓ(ˆØ�9‰9Ò Ø—	‘	˜!“ˆAØˆr   )r2  r/  r1  r  rF  r0  rE  ©éà   rÍ   rê   é`   Nr®   rF   s   @r   r8  r8  ø  s   ø† ñ÷÷.ð r   r8  c                  ó6   a € ] tR tRtRtRV 3R lltR tRtV ;t# )r:  i!  a]  Image to Patch Unembedding

Args:
    img_size (int): Image size.  Default: 224.
    patch_size (int): Patch token size. Default: 4.
    in_chans (int): Number of input image channels. Default: 3.
    embed_dim (int): Number of linear projection output channels. Default: 96.
    norm_layer (nn.Module, optional): Normalization layer. Default: None
c                	ó   <€ \         SV `  4        \        V4      p\        V4      pV^ ,          V^ ,          ,          V^,          V^,          ,          .pWn        W n        W`n        V^ ,          V^,          ,          V n        W0n        W@n        R# rD  )	r*   r+   r   r/  r0  rE  rF  r1  r2  rG  s   &&&&&& €r   r+   ÚPatchUnEmbed.__init__,  s€   ø€ ô 	‰ÑÔÜ˜XÓ&ˆÜ˜zÓ*ˆ
à�Q�K˜: a�=Õ(Ø�Q�K˜: a�=Õ(ð
Ðð !ŒØ$ŒØ"4ÔØ-¨aÕ0Ð3EÀaÕ3HÕHˆÔà ŒØ"Žr   c                	ó    € VP                  ^^4      P                  VP                  ^ ,          V P                  V^ ,          V^,          4      pV# r6   )r7   r8   r   r2  r;   s   &&&r   r=   ÚPatchUnEmbed.forward>  sA   € Ø�K‰K˜˜1Ó×"Ñ"Ø�G‰G�A�J˜Ÿ™¨¨q­	°6¸!µ9ó
ˆð ˆr   )r2  r/  r1  rF  r0  rE  rL  r®   rF   s   @r   r:  r:  !  s   ø† ñ÷#÷$ð r   r:  c                  ó,   a € ] tR tRtRtV 3R ltRtV ;t# )ÚUpsampleiE  z‘Upsample module.

Args:
    scale (int): Scale factor. Supported scales: 2^n and 3.
    num_feat (int): Channel number of intermediate features.
c           
     	óL  <€ . pWn         W n        W^,
          ,          ^ 8X  d‡   \        \        \        P
                  ! V^4      4      4       FX  pVP                  \        P                  ! V^V,          ^^^4      4       VP                  \        P                  ! ^4      4       KZ  	  MlV^8X  dW   VP                  \        P                  ! V^	V,          ^^^4      4       VP                  \        P                  ! ^4      4       M\        RV R24      h\        SV `0  ! V!   R# )r$   zscale z/ is not supported. Supported scales: 2^n and 3.N)rk   Únum_featr   r_   Úmathr¨   r  r,   r.   ÚPixelShuffleÚ
ValueErrorr*   r+   )r2   rk   rW  Úmr   r3   s   &&&  €r   r+   ÚUpsample.__init__M  sÞ   ø€ ØˆØŒ
Ø ŒØ˜Q•YÕ AÔ%Üœ3œtŸxšx¨¨qÓ1Ó2Ö3�Ø—‘œŸš 8¨Q°­\¸1¸aÀÓCÔDØ—‘œŸš¨Ó+Ö,ò 4ð �aŒZØ�H‰H”R—Y’Y˜x¨¨X­°q¸!¸QÓ?Ô@Ø�H‰H”R—_’_ QÓ'Õ(äØ˜˜ÐNÐOóð ô 	‰Ò˜!Ôr   )rW  rk   ©r@   rA   rB   rC   r’   r+   rD   rE   rF   s   @r   rU  rU  E  s   ø† ñ÷ó r   rU  c                  ó0   a € ] tR tRtRtRV 3R lltRtV ;t# )ÚUpsampleOneStepi_  a  UpsampleOneStep module (the difference with Upsample is that it always only has 1conv + 1pixelshuffle)
   Used in lightweight SR to save parameters.

Args:
    scale (int): Scale factor. Supported scales: 2^n and 3.
    num_feat (int): Channel number of intermediate features.

c           	     	óö   <€ W n         W@n        . pVP                  \        P                  ! W!^,          V,          ^^^4      4       VP                  \        P
                  ! V4      4       \        SV `  ! V!   R# )r†   N)rW  rš   r  r,   r.   rY  r*   r+   )r2   rk   rW  Ú
num_out_chrš   r[  r3   s   &&&&& €r   r+   ÚUpsampleOneStep.__init__i  sY   ø€ Ø ŒØ 0ÔØˆØ	�‰”—’˜8¨Q¥h°*Õ%<¸aÀÀAÓFÔGØ	�‰”—’ Ó'Ô(Ü‰Ò˜!Ôr   )rš   rW  rS   r]  rF   s   @r   r_  r_  _  s   ø† ñ÷õ r   r_  c            *      ó:  a € ] tR tRtRt/ tR^@R^R^R^ZRR)RR)R	^R
RR^@R^R^RRRRR]P                  RRRRR^RRRRRRRR/V 3R lltR  t	]
P                  P                  R! 4       t]
P                  P                  R" 4       tR# tR$ tR% t]R& 4       tR' tR(tV ;t# )*ÚATDir  aŠ  
ATD

A PyTorch impl of : `Transcending the Limit of Local Window: Advanced Super-Resolution Transformer with Adaptive Token Dictionary`.

Args:
    img_size (int | tuple(int)): Input image size. Default 64
    patch_size (int | tuple(int)): Patch size. Default: 1
    in_chans (int): Number of input image channels. Default: 3
    embed_dim (int): Patch embedding dimension. Default: 96
    depths (tuple(int)): Depth of each Swin Transformer layer.
    num_heads (tuple(int)): Number of attention heads in different layers.
    window_size (int): Window size. Default: 7
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 2
    qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
    norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
    ape (bool): If True, add absolute position embedding to the patch embedding. Default: False
    patch_norm (bool): If True, add normalization after patch embedding. Default: True
    use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False
    upscale: Upscale factor. 2/3/4/8 for image SR, 1 for denoising and compress artifact reduction
    img_range: Image range. 1. or 255.
    upsampler: The reconstruction reconstruction module. 'pixelshuffle'/'pixelshuffledirect'/'nearest+conv'/None
    resi_connection: The convolutional block before residual connection. '1conv'/'3conv'
r/  r0  r1  r2  Údepthsri   rW   r²   é   r›   r¡   rÕ   rÔ   g       @rj   Træ   ÚapeFÚ
patch_normÚupscaleÚ	img_rangeç      ð?Ú	upsamplerÚ r>  r,  r  c               	ó|  <€ \         SV `  4        TpTp^@pVV n        V  V'       g(   V P                  R\        P
                  ! ^4      4       MRV n        V^8X  d1   R"p\        P                  ! V4      P                  ^^^^4      V n	        M\        P
                  ! ^^^^4      V n	        VV n
        VV n        \        P                  ! VV^^^4      V n        \        V4      V n        W@n        Wðn        VV n        W@n        WÀn        Wpn        \/        TTTTV P&                  '       d   TMRR7      V n        V P0                  P2                  pV P0                  P4                  pVV n        \7        TTTTV P&                  '       d   TMRR7      V n        V P$                  '       dI   \        P:                  ! \        P
                  ! ^VV4      4      V n        \?        V P<                  RR7       V PA                  4       pV P                  RV4       \        PB                  ! 4       V n"        \G        V P                   4       F‚  p\I        R#/ RVbRVbR	V^ ,          V^,          3bR
VV,          bRVV,          bRVbRVbRV	bRV
bRVbRV P*                  bRVbRVbRRbRVbRVbRVb pV PD                  PK                  V4       K„  	  V! V P(                  4      V n&        VR8X  d    \        P                  ! WD^^^4      V n'        MµVR8X  d¯   \        PP                  ! \        P                  ! WD^,          ^^^4      \        PR                  ! RRR7      \        P                  ! V^,          V^,          ^^^ 4      \        PR                  ! RRR7      \        P                  ! V^,          V^^^4      4      V n'        V P                  R8X  d|   \        PP                  ! \        P                  ! VV^^^4      \        PR                  ! RR7      4      V n*        \W        VV4      V n,        \        P                  ! VV^^^4      V n-        EMaV P                  R8X  d&   \]        VVVV^ ,          V^,          34      V n,        EM+V P                  R 8X  dü   V P                  ^8X  g   Q R!4       h\        PP                  ! \        P                  ! VV^^^4      \        PR                  ! RR7      4      V n*        \        P                  ! VV^^^4      V n/        \        P                  ! VV^^^4      V n0        \        P                  ! VV^^^4      V n1        \        P                  ! VV^^^4      V n-        \        PR                  ! RRR7      V n2        M\        P                  ! VV^^^4      V n-        V Pg                  V Ph                  4       R# )$r¬   Úno_normNr.  re   rf   Úrelative_position_index_SAr   rå   rš   r  ri   rW   r²   r›   r¡   rÕ   rÔ   rj   ræ   r  r/  r0  r>  r,  r3  r4  Tr5  Úpixelshuffle)r7  Úpixelshuffledirectúnearest+convzonly support x4 now.)gšwœ¢#¹Ü?gï8EGrùÛ?gB`åÐ"ÛÙ?r‰   )5r*   r+   rj  Úregister_bufferr
   rm   ro  ÚTensorr8   Úmeanri  rl  r,   r.   Ú
conv_firstÚlenÚ
num_layersr2  rg  rh  Únum_featuresrÔ   rW   r8  r9  rF  rE  r:  r;  rl   Úabsolute_pos_embedr   Úcalculate_rpi_sar  r  r   r+  r  r  Úconv_after_bodyr-   rÖ   Úconv_before_upsamplerU  ÚupsampleÚ	conv_lastr_  Úconv_up1Úconv_up2Úconv_hrr×   ÚapplyÚ_init_weights) r2   r/  r0  r1  r2  re  ri   rW   r²   r›   r¡   rÕ   rÔ   rj   ræ   rg  rh  ri  rj  rl  r>  r  Ú	num_in_chra  rW  Úrgb_meanrF  rE  rp  Úi_layerr$  r3   s    &$$$$$$$$$$$$$$$$$$$$$         €r   r+   ÚATD.__init__�  s  ø€ ô2 	‰ÑÔØˆ	Øˆ
ØˆØ"ˆŒáßØ× Ñ  ¬E¯KªK¸«NÕ;àˆDŒLà�qŒ=Ø/ˆHÜŸš XÓ.×3Ñ3°A°q¸!¸QÓ?ˆD�IäŸš A q¨!¨QÓ/ˆDŒIØˆŒØ"ˆŒô Ÿ)š) I¨y¸!¸QÀÓBˆŒô ˜f›+ˆŒØ"ŒØŒØ$ˆŒØ%ÔØ"ŒØ&Ôô &ØØ!ØØØ%)§_§_ _‘z¸$ô
ˆÔð ×&Ñ&×2Ñ2ˆØ!×-Ñ-×@Ñ@ÐØ"4ˆÔô *ØØ!ØØØ%)§_§_ _‘z¸$ô
ˆÔð �8�8ˆ8Ü&(§l¢lÜ—’˜A˜{¨IÓ6ó'ˆDÔ#ô ˜$×1Ñ1°tÕ<ð &*×%:Ñ%:Ó%<Ð"Ø×ÑÐ9Ð;UÔVô —m’m“oˆŒÜ˜TŸ_™_Ö-ˆGÜò Ùðáðð #5°QÕ"7Ð9KÈAÕ9NÑ!Oðð ˜W–oð	ð
 $ GÖ,ðñ (ðñ ,ðñ &ðñ *ðñ %8ðð Ÿ.š.ðñ "ðñ &ðñ  ðñ "ðñ  &ð!ñ" !0ð#ˆEð& �K‰K×Ñ˜uÖ%ñ) .ñ* ˜t×0Ñ0Ó1ˆŒ	ð ˜gÔ%Ü#%§9¢9¨YÀ1ÀaÈÓ#KˆDÕ Ø Ô'ä#%§=¢=Ü—	’	˜)°!¥^°Q¸¸1Ó=Ü—’¨C¸Ô>Ü—	’	˜) q�.¨)°q­.¸!¸QÀÓBÜ—’¨C¸Ô>Ü—	’	˜) q�.¨)°Q¸¸1Ó=ó$ˆDÔ ð �>‰>˜^Ô+ä(*¯ªÜ—	’	˜) X¨q°!°QÓ7¼¿ºÈdÔ9Só)ˆDÔ%ô % W¨hÓ7ˆDŒMÜŸYšY x°¸QÀÀ1ÓEˆDŽNØ�^‰^Ð3Ô3ä+ØØØØ# AÕ&Ð(:¸1Õ(=Ð>ó	ˆDŽMð �^‰^˜~Ô-à—<‘< 1Ô$Ð<Ð&<Ó<Ð$Ü(*¯ªÜ—	’	˜) X¨q°!°QÓ7¼¿ºÈdÔ9Só)ˆDÔ%ô ŸIšI h°¸!¸QÀÓBˆDŒMÜŸIšI h°¸!¸QÀÓBˆDŒMÜŸ9š9 X¨x¸¸A¸qÓAˆDŒLÜŸYšY x°¸QÀÀ1ÓEˆDŒNÜŸš°SÀ$ÔGˆD�Jô  ŸYšY y°*¸aÀÀAÓFˆDŒNà�
‰
�4×%Ñ%Ö&r   c                	ó  € \        V\        P                  4      '       dv   \        VP                  R R7       \        V\        P                  4      '       d=   VP
                  e-   \        P                  P                  VP
                  ^ 4       R# R# R# \        V\        P                  4      '       dW   \        P                  P                  VP
                  ^ 4       \        P                  P                  VP                  R4       R# R# )re   rf   Nrk  )	Ú
isinstancer,   rJ   r   Úweightr—   ÚinitÚ	constant_r   )r2   r[  s   &&r   r…  ÚATD._init_weights/  s¡   € Ü�aœŸ™×#Ò#Ü˜!Ÿ(™(¨Õ-Ü˜!œRŸY™Y×'Ò'¨A¯F©FÒ,>Ü—‘×!Ñ! !§&¡&¨!Ö,ñ -?Ñ'ä˜œ2Ÿ<™<×(Ò(Ü�G‰G×Ñ˜aŸf™f aÔ(Ü�G‰G×Ñ˜aŸh™h¨Ö,ñ )r   c                	ó   € R 0# )r{  r‰   r�   s   &r   Úno_weight_decayÚATD.no_weight_decay8  s   € à$Ð%Ð%r   c                	ó   € R 0# )rn   r‰   r�   s   &r   Úno_weight_decay_keywordsÚATD.no_weight_decay_keywords<  s   € à.Ð/Ð/r   c                	ó8  € VP                   ^,          VP                   ^,          3pV P                  V4      pV P                  '       d   WP                  ,           pV P                   F  pV! WV4      pK  	  V P                  V4      pV P                  W4      pV# rJ  )r   r9  rg  r{  r  r  r;  )r2   r   rð   r<   r$  s   &&&  r   Úforward_featuresÚATD.forward_features@  s�   € Ø—'‘'˜!•*˜aŸg™g a�jÐ)ˆØ×Ñ˜QÓˆØ�8�8ˆ8Ø×+Ñ+Õ+ˆAà—[”[ˆEÙ�a Ó(ŠAñ !ð �I‰I�a‹LˆØ×Ñ˜qÓ)ˆàˆr   c                	óŽ  € \         P                  ! V P                  4      p\         P                  ! V P                  4      p\         P                  ! \         P                  ! W.4      4      p\         P
                  ! V^4      pVR,          VR,          ,
          pVP                  ^^^ 4      P                  4       pVR;;,          V P                  ^,
          ,          uu&   VR;;,          V P                  ^,
          ,          uu&   VR;;,          ^V P                  ,          ^,
          ,          uu&   VP                  R4      pV# )r$   )r   r   N)r   Nr   )r   r   r   )r   r   r$   r	   )	r
   r   rW   ÚstackÚmeshgridr:   rV   r9   Úsum)r2   Úcoords_hÚcoords_wÚcoordsÚcoords_flattenÚrelative_coordsÚrelative_position_indexs   &      r   r|  ÚATD.calculate_rpi_saN  sý   € ä—<’< × 0Ñ 0Ó1ˆÜ—<’< × 0Ñ 0Ó1ˆÜ—’œUŸ^š^¨XÐ,@ÓAÓBˆÜŸš v¨qÓ1ˆà˜:Õ&¨¸
Õ)CÕCð 	ð *×1Ñ1Øˆq�!ó
ç
‰*‹,ð 	ð 	˜×  D×$4Ñ$4°qÕ$8Õ8Ó Ø˜×  D×$4Ñ$4°qÕ$8Õ8Ó Ø˜×  A¨×(8Ñ(8Õ$8¸1Õ$<Õ<Ó Ø"1×"5Ñ"5°bÓ"9ÐØ&Ð&r   c                	ó  € Vw  r#\         P                  ! ^W#^34      p\        ^ V P                  ) 4      \        V P                  ) V P                  ^,          ) 4      \        V P                  ^,          ) R4      3p\        ^ V P                  ) 4      \        V P                  ) V P                  ^,          ) 4      \        V P                  ^,          ) R4      3p^ pV F  pV F  pWtRW#R3&   V^,          pK  	  K  	  \	        W@P                  4      pVP                  RV P                  V P                  ,          4      pVP                  ^4      VP                  ^4      ,
          p	V	P                  V	^ 8g  R4      P                  V	^ 8H  R4      p	V	# )r$   Nr   g        r	   g      YÀ)r
   rm   ÚslicerW   r]   r8   r   Úmasked_fill)
r2   r<   rY   rZ   Úimg_maskÚh_slicesÚw_slicesÚcntÚmask_windowsrì   s
   &&        r   Úcalculate_maskÚATD.calculate_mask`  sƒ  € à‰ˆÜ—;’;  1¨˜|Ó,ˆä�!�d×&Ñ&Ð&Ó'Ü�4×#Ñ#Ð# t×'7Ñ'7¸1Õ'<Ð%=Ó>Ü�D×$Ñ$¨Õ)Ð*¨DÓ1ð
ˆô �!�d×&Ñ&Ð&Ó'Ü�4×#Ñ#Ð# t×'7Ñ'7¸1Õ'<Ð%=Ó>Ü�D×$Ñ$¨Õ)Ð*¨DÓ1ð
ˆð
 ˆÛˆAÛ�Ø'*˜˜A !˜Ñ$Ø�q•’ó ñ ô
 (Ø×&Ñ&ó
ˆð $×(Ñ(¨¨T×-=Ñ-=À×@PÑ@PÕ-PÓQˆØ ×*Ñ*¨1Ó-°×0FÑ0FÀqÓ0IÕIˆ	Ø×)Ñ)¨)°q©.¸&ÓA×MÑMØ˜‰N˜Có
ˆ	ð Ðr   c                	ó   € V P                   R J # rS   )ro  r�   s   &r   Úis_normÚATD.is_norm  s   € à�|‰|˜tÐ#Ð#r   c           
     	ó>  € VP                  4       R,          VP                  4       R,          r2V P                  pW$,           ^,
          V,          V,          V,
          pW4,           ^,
          V,          V,          V,
          pW%,           W6,           r‡\        P                  ! V\        P                  ! V^.4      .^4      RRRV1R3,          p\        P                  ! V\        P                  ! V^.4      .^4      RRRRV13,          pV P
                  '       dD   V P                  P                  V4      V n        WP                  ,
          V P                  ,          pV P                  Wx.4      P                  VP                  4      p	RV	RV P                  /p
V P                  R8X  dl   V P                  V4      pV P                  V P!                  W4      4      V,           pV P#                  V4      pV P%                  V P'                  V4      4      pEM£V P                  R8X  dL   V P                  V4      pV P                  V P!                  W4      4      V,           pV P'                  V4      pEMGV P                  R8X  dè   V P                  V4      pV P                  V P!                  W4      4      V,           pV P#                  V4      pV P)                  V P+                  \,        P.                  ! V^RR	7      4      4      pV P)                  V P1                  \,        P.                  ! V^RR	7      4      4      pV P%                  V P)                  V P3                  V4      4      4      pMOV P                  V4      pV P                  V P!                  Wº4      4      V,           pWP%                  V4      ,           pV P
                  '       d$   WP                  ,          V P                  ,           pVR
RW P4                  ,          1RW0P4                  ,          13,          pV# )r†   r   Nrì   rí   rq  rr  rs  Únearest)Úscale_factorÚmode.ru   r	   )ÚsizerW   r
   r½   r¾   r¯  rv  Útype_asrj  r¬  r   r   rp  rl  rw  r}  r—  r~  r€  r  r×   r�  r¤   Úinterpolater‚  rƒ  ri  )r2   r   Úh_oriÚw_oriÚmodÚh_padÚw_padrY   rZ   rì   rð   Úx_firstÚress   &&           r   r=   ÚATD.forwardƒ  s.  € à—v‘v“x •| Q§V¡V£X¨b¥\ˆuØ×ÑˆØ•+ •/ cÕ)¨SÕ0°5Õ8ˆØ•+ •/ cÕ)¨SÕ0°5Õ8ˆØ�}˜e�mˆ1Ü�IŠI�qœ%Ÿ*š* Q¨¨Ó,Ð-¨qÓ1°!°Q¸¸¸¸A°+Õ>ˆÜ�IŠI�qœ%Ÿ*š* Q¨¨Ó,Ð-¨qÓ1°!°Q¸¸2¸A¸2°+Õ>ˆð �<�<ˆ<ØŸ	™	×)Ñ)¨!Ó,ˆDŒIØ—Y‘Y• $§.¡.Õ0ˆAà×'Ñ'¨¨Ó/×2Ñ2°1·8±8Ó<ˆ	Ø˜y¨(°D×4SÑ4SÐTˆà�>‰>˜^Ô+à—‘ Ó"ˆAØ×$Ñ$ T×%:Ñ%:¸1Ó%EÓFÈÕJˆAØ×)Ñ)¨!Ó,ˆAØ—‘˜tŸ}™}¨QÓ/Ó0ŠAØ�^‰^Ð3Ô3à—‘ Ó"ˆAØ×$Ñ$ T×%:Ñ%:¸1Ó%EÓFÈÕJˆAØ—‘˜aÓ ŠAØ�^‰^˜~Ô-à—‘ Ó"ˆAØ×$Ñ$ T×%:Ñ%:¸1Ó%EÓFÈÕJˆAØ×)Ñ)¨!Ó,ˆAØ—
‘
Ø—‘œaŸmšm¨A¸AÀIÔNÓOóˆAð —
‘
Ø—‘œaŸmšm¨A¸AÀIÔNÓOóˆAð —‘˜tŸz™z¨$¯,©,°q«/Ó:Ó;‰Að —o‘o aÓ(ˆGØ×&Ñ& t×'<Ñ'<¸WÓ'MÓNÐQXÕXˆCØ—N‘N 3Ó'Õ'ˆAà�<�<ˆ<Ø—N‘NÕ" T§Y¡YÕ.ˆAð ˆcÐ)�UŸ\™\Õ)Ð)Ð+A¨U·\±\Õ-AÐ+AÐAÕBˆàˆr   )r{  rg  r}  r~  rw  rƒ  r€  r�  r‚  r2  rj  r  r×   rv  rÔ   ro  r  rz  ry  r9  rh  r;  rE  r  rl  ri  rW   )é   rÀ  rÀ  rÀ  )r@   rA   rB   rC   r’   Úhyperparametersr,   r   r+   r…  r
   ÚjitÚignorer‘  r”  r—  r|  r¬  Úpropertyr¯  r=   rD   rE   rF   s   @r   rd  rd  r  so  ø† ñð2 €Oð^'ð ð^'ð ð	^'ð
 ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð  —<‘<ð!^'ð" ð#^'ð$ ð%^'ð& ð'^'ð( ð)^'ð* ð+^'ð,  ð-^'ð. ÷/^'ò@-ð ‡Y�Y×Ññ&ó ð&ð ‡Y�Y×Ññ0ó ð0òò'ò$ð> ñ$ó ð$÷5ð 5r   rd  )%r’   Ú
__future__r   rX  Únumpyr§   r
   Útorch.nnr,   Útorch.nn.functionalÚ
functionalr¤   Úspandrel.utilr   Úspandrel.util.timmr   r   r   r    ÚModuler"   rH   r]   r`   rb   r”   r°   rÐ   r  r  r+  r8  r:  r-   rU  r_  rd  r‰   r   r   Ú<module>rÍ     sI  ðñõ #ã ã Û Ý ß Ð å /ß 7òòôˆR�Y‰Yô ô8ˆb�i‰iô ò2ò"ô&Lw�b—i‘iô Lwô^;ˆR�Y‰Yô ;ô|VˆR�Y‰Yô Vôrb˜"Ÿ)™)ô bôJ(K�2—9‘9ô (KôVV_�—‘ô V_ôra
ˆ2�9‰9ô a
ôH&�—‘ô &ôR!�2—9‘9ô !ôHˆr�}‰}ô ô4�b—m‘mô ñ& ÓôEˆ"�)‰)ó Eó òEr   