+
    öäûjpx  ã                   óú  € ^ RI t ^ RIt^ RIHt ^ RIHu Ht ^ RIHt ^ RI	H
t
 ^ RIHt R tR t ! R R]P                  4      t ! R	 R
]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tR tR t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R ]P                  4      t ! R! R"]P                  4      t ! R# R$]P>                  4      t ]! 4        ! R% R&]P                  4      4       t!R# )'é    N©Ú	rearrange)Útrunc_normal_)Ústore_hyperparametersc                óÜ   € V P                   w  r4rVV P                  W4WQ,          WV,          V4      pVP                  ^ ^^^^^4      P                  4       P	                  RW,          V4      pV# )z?
Input: Image (B, C, H, W)
Output: Window Partition (B', N, C)
éÿÿÿÿ)ÚshapeÚviewÚpermuteÚ
contiguousÚreshape)	ÚimgÚH_spÚW_spÚBÚCÚHÚWÚimg_reshapeÚimg_perms	   &&&      ÚeC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\spandrel\architectures\CRAFT\__arch\CRAFT.pyÚimg2windowsr      sj   € ð
 —‘�J€Aˆ!Ø—(‘(˜1 ¥¨D°tµ)¸TÓB€Kà×Ñ˜A˜q ! Q¨¨1Ó-×8Ñ8Ó:×BÑBÀ2ÀtÅ{ÐTUÓVð ð €Oó    c                ó   € \        V P                  ^ ,          W4,          V,          V,          ,          4      pV P                  WSV,          WB,          WR4      pVP                  ^ ^^^^^4      P	                  4       P                  WSVR4      pV# )z?
Input: Window Partition (B', N, C)
Output: Image (B, H, W, C)
r   )Úintr	   r
   r   r   )Úimg_splits_hwr   r   r   r   r   r   s   &&&&&  r   Úwindows2imgr      sz   € ô
 	ˆM×Ñ Õ" a¥e¨d¥l°TÕ&9Õ:Ó;€Aà
×
Ñ
˜Q T¥	¨1­9°dÀ"Ó
E€CØ
�+‰+�a˜˜A˜q ! QÓ
'×
2Ñ
2Ó
4×
9Ñ
9¸!ÀÀ2Ó
F€CØ€Jr   c                   óV   a a€ ] tR t^(t oRR]P
                  3V 3R lltR tRtVt	V ;t
# )ÚMlpNc                óð   <€ \         SV `  4        T;'       g    TpT;'       g    TpW n        Wn        \        P
                  ! W4      V n        V! 4       V n        \        P
                  ! W#4      V n        R # ©N)	ÚsuperÚ__init__Úhidden_dimsÚin_dimsÚnnÚLinearÚfc1ÚactÚfc2)ÚselfÚin_featuresÚhidden_featuresÚout_featuresÚ	act_layerÚ	__class__s   &&&&&€r   r#   ÚMlp.__init__)   s^   ø€ ô 	‰ÑÔØ#×2Ð2 {ˆØ)×8Ð8¨[ˆØ*ÔØ"ŒÜ—9’9˜[Ó:ˆŒÙ“;ˆŒÜ—9’9˜_Ó;ˆŽr   c                ó–   € VP                   w  r#pW0n        V P                  V4      pV P                  V4      pV P	                  V4      pV# r!   )r	   ÚNr(   r)   r*   )r+   ÚxÚ_Úns   &&  r   ÚforwardÚMlp.forward5   s@   € Ø—'‘'‰ˆˆaØŒØ�H‰H�Q‹KˆØ�H‰H�Q‹KˆØ�H‰H�Q‹KˆØˆr   )r3   r)   r(   r*   r$   r%   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r&   ÚGELUr#   r7   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r0   Ú__classdict__s   @@r   r   r   (   s   ù‡ € à+/¸dÈbÏgÉg÷
<÷ò r   r   c                   ó<   a a€ ] tR t^>t oRtV 3R ltR tRtVtV ;t	# )ÚDynamicPosBiaszÃDynamic Relative Position Bias.
Args:
    dim (int): Number of input channels.
    num_heads (int): Number of attention heads.
    residual (bool):  If True, use residual strage to connect conv.
c                óz  <€ \         SV `  4        W0n        W n        V^,          V n        \
        P                  ! ^V P                  4      V n        \
        P                  ! \
        P                  ! V P                  4      \
        P                  ! RR7      \
        P                  ! V P                  V P                  4      4      V n        \
        P                  ! \
        P                  ! V P                  4      \
        P                  ! RR7      \
        P                  ! V P                  V P                  4      4      V n        \
        P                  ! \
        P                  ! V P                  4      \
        P                  ! RR7      \
        P                  ! V P                  V P                  4      4      V n        R# )é   T)ÚinplaceN)r"   r#   ÚresidualÚ	num_headsÚpos_dimr&   r'   Úpos_projÚ
SequentialÚ	LayerNormÚReLUÚpos1Úpos2Úpos3)r+   ÚdimrI   rH   r0   s   &&&&€r   r#   ÚDynamicPosBias.__init__G   s  ø€ Ü‰ÑÔØ ŒØ"ŒØ˜a•xˆŒÜŸ	š	 ! T§\¡\Ó2ˆŒÜ—M’MÜ�LŠL˜Ÿ™Ó&Ü�GŠG˜DÔ!Ü�IŠI�d—l‘l D§L¡LÓ1ó
ˆŒ	ô
 —M’MÜ�LŠL˜Ÿ™Ó&Ü�GŠG˜DÔ!Ü�IŠI�d—l‘l D§L¡LÓ1ó
ˆŒ	ô
 —M’MÜ�LŠL˜Ÿ™Ó&Ü�GŠG˜DÔ!Ü�IŠI�d—l‘l D§N¡NÓ3ó
ˆŽ	r   c           	     ó|  € VP                   w  V n        V n        V P                  '       dS   V P	                  V4      pW P                  V4      ,           pW P                  V4      ,           pV P                  V4      pV# V P                  V P                  V P                  V P	                  V4      4      4      4      pV# r!   )r	   ÚlÚcrH   rK   rO   rP   rQ   )r+   ÚbiasesÚposs   && r   r7   ÚDynamicPosBias.forward]   sŽ   € ØŸ™‰ˆŒ�”Ø�=�=ˆ=Ø—-‘- Ó'ˆCØŸ	™	 #›Õ&ˆCØŸ	™	 #›Õ&ˆCØ—)‘)˜C“.ˆCð ˆ
ð —)‘)˜DŸI™I d§i¡i°·±¸fÓ0EÓ&FÓGÓHˆCØˆ
r   )	rV   rU   rI   rO   rP   rQ   rJ   rK   rH   ©
r9   r:   r;   r<   Ú__doc__r#   r7   r>   r?   r@   rA   s   @@r   rD   rD   >   s   ù‡ € ñõ
÷,	ò 	r   rD   c                   óX   a a€ ] tR t^it oRt^^.R^RR3V 3R lltR tRR ltRtVt	V ;t
# )	ÚAttention_regularaâ  
Regular Rectangle-Window (regular-Rwin) self-attention with dynamic relative position bias.
It supports both of shifted and non-shifted window.

Args:
    dim (int): Number of input channels.
    resolution (int): Input resolution.
    idx (int): The identix of V-Rwin and H-Rwin, 0 is H-Rwin, 1 is Vs-Rwin. (different order from Attention_axial)
    split_size (tuple(int)): Height and Width of the regular rectangle window (regular-Rwin).
    dim_out (int | None): The dimension of the attention output. Default: None
    num_heads (int): Number of attention heads. Default: 6
    qk_scale (float | None): Override default qk scale of head_dim ** -0.5 if set
    position_bias (bool): The dynamic relative position bias. Default: True
NTc                ó>  <€ \         SV `  4        Wn        T;'       g    TV n        W0n        WPn        W n        Wpn        W,          pT;'       g
    VR,          V n        V^ 8X  d'   V P                  ^ ,          V P                  ^,          r©M;V^8X  d'   V P                  ^ ,          V P                  ^,          ršM\        RV 24      hW�n
        W n        \        V P                  ^,          V P
                  RR7      V n        \        P                  ! RR7      V n        R# )g      à?zERROR MODE: F)rH   ©rR   Ng      à¿r   )r"   r#   rR   Údim_outÚ
split_sizerI   ÚidxÚposition_biasÚscaleÚ
ValueErrorr   r   rD   rX   r&   ÚSoftmaxÚsoftmax)r+   rR   rb   ra   r`   rI   Úqk_scalerc   Úhead_dimr   r   r0   s   &&&&&&&&   €r   r#   ÚAttention_regular.__init__y   sÚ   ø€ ô 	‰ÑÔØŒØ—~�~ #ˆŒØ$ŒØ"ŒØŒØ*ÔàÕ#ˆØ×/Ð/ ¨4¥ˆŒ
Ø�!Œ8ØŸ™¨Õ+¨T¯_©_¸QÕ-?‘$Ø�AŒXØŸ™¨Õ+¨T¯_©_¸QÕ-?‘$ä˜|¨C¨5Ð1Ó2Ð2ØŒ	ØŒ	ä! $§(¡(¨a¥-°·±È%ÔPˆŒÜ—z’z bÔ)ˆŽr   c                ó–  € VP                   w  rEpVP                  RR4      P                  4       P                  WFW#4      p\	        WP
                  V P                  4      pVP                  RV P
                  V P                  ,          V P                  W`P                  ,          4      P                  ^ ^^^4      P                  4       pV# )é   éþÿÿÿr   )
r	   Ú	transposer   r
   r   r   r   r   rI   r   )r+   r4   r   r   r   r3   r   s   &&&&   r   Úim2winÚAttention_regular.im2win™   s–   € Ø—'‘'‰ˆˆaØ�K‰K˜˜BÓ×*Ñ*Ó,×1Ñ1°!¸Ó=ˆÜ˜Ÿ9™9 d§i¡iÓ0ˆà�I‰I�b˜$Ÿ)™) d§i¡iÕ/°·±ÀÇnÁnÕATÓUß‰W�Q˜˜1˜aÓ ß‰Z‹\ð 	
ð
 ˆr   c                óè  € V^ ,          V^,          V^,          r˜pVP                   w  r«pW²V,          8X  g   Q R4       hW°P                  V P                  ,          ,          V n        V P	                  WrV4      pV P	                  W‚V4      pV P	                  W’V4      p	WpP
                  ,          pWxP                  RR4      ,          pV P                  V4      pWåP                  R4      ,          P                  V P                  V P                  ,          V P                  V P                  ,          R4      pVP                  ^^ ^4      P                  4       pWßP                  ^ 4      ,           pVP                   ^,          pVev   VP                   ^ ,          pVP                  V
VV P                  VV4      VP                  ^4      P                  ^ 4      ,           pVP                  RV P                  VV4      pV P                  V4      pWÙ,          pVP                  ^^4      P                  RV P                  V P                  ,          V4      p\        VV P                  V P                  W#4      pV# )z]
Input: qkv: (B, 3*L, C), H, W, mask: (B, N, N), N is the window size
Output: x (B, H, W, C)
z!flatten img_tokens has wrong sizerm   r   )r	   r   r   r3   ro   rd   rn   rX   r
   r   r   Ú	unsqueezerI   rg   r   r   )r+   Úqkvr   r   ÚmaskÚrpiÚ
rpe_biasesÚqÚkÚvr   ÚLr   ÚattnrX   Úrelative_position_biasr3   ÚnWr4   s   &&&&&&&            r   r7   ÚAttention_regular.forward¤   sø  € ð
 �a•&˜#˜a�& # a¥&ˆaˆà—'‘'‰ˆˆaØ˜•EŒzÐ>Ð>Ó>ˆzà—y‘y 4§9¡9Õ,Õ-ˆŒà�K‰K˜˜aÓ ˆØ�K‰K˜˜aÓ ˆØ�K‰K˜˜aÓ ˆà—
‘
�NˆØ—;‘;˜r 2Ó&Õ&ˆð �h‰h�zÓ"ˆà!$§X¡X¨b£\Õ!2×!7Ñ!7Ø�I‰I˜Ÿ	™	Õ! 4§9¡9¨t¯y©yÕ#8¸"ó"
Ðð "8×!?Ñ!?ÀÀ1ÀaÓ!H×!SÑ!SÓ!UÐØ×6Ñ6°qÓ9Õ9ˆà�J‰J�q�Mˆð ÒØ—‘˜A•ˆBØ—9‘9˜Q  D§N¡N°A°qÓ9¸D¿N¹NÈ1Ó<M×<WÑ<WØó=õ ˆDð —9‘9˜R §¡°°AÓ6ˆDØ�|‰|˜DÓ!ˆà�HˆØ�K‰K˜˜1Ó×%Ñ%Ø�—	‘	˜DŸI™IÕ% qó
ˆô
 ˜˜4Ÿ9™9 d§i¡i°Ó6ˆàˆr   )r   r3   r   rR   r`   rb   rI   rX   rc   rd   rg   ra   )NNN)r9   r:   r;   r<   r[   r#   ro   r7   r>   r?   r@   rA   s   @@r   r]   r]   i   s4   ù‡ € ñð& �q�6ØØØØ÷*ò@	÷/ô /r   r]   c                   ó|   a a€ ] tR t^Öt oRtRR	RRR]P                  ]P                  3V 3R llt]	3R lt
RtVtV ;t# )
ÚSRWABa|  Shift Rectangle Window Attention Block.

Args:
    dim (int): Number of input channels.
    num_heads (int): Number of attention heads.
    split_size (int): Define the window size.
    shift_size (int): Shift size for SW-MSA.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
    qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
    qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
    act_layer (nn.Module, optional): Activation layer. Default: nn.GELU
    norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
ç       @TNc
                ó^  <€ \         SV `  4        Wn        W@n        WPn        V	! V4      V n        \        P                  ! W^,          VR7      V n        \        P                  ! W4      V n	        ^V n
        \        P                  ! W^^^VR7      V n        \        P                  ! \        V P                  4       U
u. uF*  p
\        V^,          V
VV^,          V^,          VRR7      NK,  	  up
4      V n        V	! V4      V n        \%        W,          4      p\'        WVR7      V n        R# u up
i )é   )Úbias)Úkernel_sizeÚstrideÚpaddingÚgroupsT)rb   ra   rI   r`   rh   rc   )r,   r-   r/   N)r"   r#   rR   Ú
shift_sizeÚ	mlp_ratioÚnorm1r&   r'   rs   ÚprojÚ
branch_numÚConv2dÚget_vÚ
ModuleListÚranger]   ÚattnsÚnorm2r   r   Úmlp)r+   rR   rI   ra   r‰   rŠ   Úqkv_biasrh   r/   Ú
norm_layerÚiÚmlp_hidden_dimr0   s   &&&&&&&&&&  €r   r#   ÚSRWAB.__init__å   s  ø€ ô 	‰ÑÔØŒØ$ŒØ"ŒÙ “_ˆŒ
Ü—9’9˜S¨¥'°Ô9ˆŒÜ—I’I˜cÓ'ˆŒ	ØˆŒÜ—Y’YØ !¨A°qÀô
ˆŒ
ô —]’]ô ˜tŸ™Ô/óñ 0�Aô "Ø˜1•HØØ)Ø'¨1�nØ 1�HØ%Ø"&÷ñ 0ñó
ˆŒ
ñ   “_ˆŒ
Ü˜S�_Ó-ˆÜØÀyô
ˆŽùò!s   Â<0D*c           
     ó,  € Vw  rVVw  V n         V n        VP                  w  rxp	Tp
V P                  V4      pV P	                  V4      P                  VR^V	4      P                  ^^ ^^4      pV^,          P                  RR4      P                  4       P                  WyWV4      pV P                  ^ ,          ^ 8”  g   V P                  ^,          ^ 8”  Edú   VP                  ^WuWi4      p\        P                  ! VRRRRRV	^,          13,          V P                  ^ ,          ) V P                  ^,          ) 3RR7      pVP                  ^WuV,          V	^,          4      p\        P                  ! VRRRRV	^,          R13,          V P                  ^,          ) V P                  ^ ,          ) 3RR7      pVP                  ^WuV,          V	^,          4      pV P                  ^ ,          ! VVVV^ ,          VR,          VR,          R7      pV P                  ^,          ! VVVV^,          VR,          VR,          R7      p\        P                  ! WðP                  ^ ,          V P                  ^,          3RR7      p\        P                  ! VV P                  ^,          V P                  ^ ,          3RR7      p\        P                  ! VV.RR	7      pMšV P                  ^ ,          ! VRRRRV	^,          13,          VVVR,          VR,          R
7      pV P                  ^,          ! VRRRV	^,          R13,          VVVR,          VR,          R
7      p\        P                  ! VV.RR	7      pVP                  VRV	4      P                  4       pV P                  V4      pVP                  ^ ^^^4      P                  4       P                  VRV	4      pVV,           pV P!                  V4      pV
V,           pWP#                  V P%                  V4      4      ,           pV# )é   ºNNNN)ÚshiftsÚdimsÚrpi_sa_hÚbiases_h)rt   ru   rv   Úrpi_sa_vÚbiases_vr_   )ru   rv   r   rm   )rl   rƒ   )r›   rl   )ÚhÚwr	   r‹   rs   r   r   rn   r   r
   r‰   ÚtorchÚrollr’   Úcatr�   rŒ   r”   r“   )r+   r4   Úx_sizeÚparamsÚ	attn_maskr£   r¤   Úbr5   rV   Úshortcutrs   ry   Úqkv_0Úqkv_1Úx1_shiftÚx2_shiftÚx1Úx2Ú	attened_xÚlcms   &&&&&                r   r7   ÚSRWAB.forward  sž  € Ø‰ˆØ‰ˆŒ�”à—'‘'‰ˆˆaØˆØ�J‰J�q‹MˆØ�h‰h�q‹k×!Ñ! ! R¨¨AÓ.×6Ñ6°q¸!¸QÀÓBˆØ��F×Ñ˜R Ó$×/Ñ/Ó1×6Ñ6°q¸QÓBˆð �?‰?˜1Õ Ô! T§_¡_°QÕ%7¸!Õ%;Ø—(‘(˜1˜a AÓ)ˆCä—J’JØ�A�q˜!˜Q  ! q¥& Ð(Õ)ØŸ™¨Õ+Ð+¨d¯o©o¸aÕ.@Ð-@ÐAØôˆEð
 —J‘J˜q !¨¥U¨A°­FÓ3ˆEä—J’JØ�A�q˜!˜Q  Q¥¡Ð(Õ)ØŸ™¨Õ+Ð+¨d¯o©o¸aÕ.@Ð-@ÐAØôˆEð
 —J‘J˜q !¨¥U¨A°­FÓ3ˆEð —z‘z !–}ØØØØ˜q•\Ø˜:Õ&Ø! *Õ-ôˆHð —z‘z !–}ØØØØ˜q•\Ø˜:Õ&Ø! *Õ-ôˆHô —’Ø§/¡/°!Õ"4°d·o±oÀaÕ6HÐ!IÐPVôˆBô —’Ø $§/¡/°!Õ"4°d·o±oÀaÕ6HÐ!IÐPVôˆBô Ÿ	š	 2 r (°Ô3‰Ið —‘˜A–Ø�A�q˜!˜X˜q A�v˜XÐ%Õ&ØØØ˜:Õ&Ø! *Õ-ôˆBð —‘˜A–Ø�A�q˜!˜Q !�V™XÐ%Õ&ØØØ˜:Õ&Ø! *Õ-ôˆBô Ÿ	š	 2 r (°Ô3ˆIà—N‘N 1 b¨!Ó,×7Ñ7Ó9ˆ	ð �j‰j˜‹mˆØ�k‰k˜!˜Q  1Ó%×0Ñ0Ó2×7Ñ7¸¸2¸qÓAˆà •Oˆ	à—I‘I˜iÓ(ˆ	ð �yÕ ˆØ—‘˜Ÿ™ A›Ó'Õ'ˆØˆr   )r’   r�   rR   r�   r£   r”   rŠ   r‹   r“   rŒ   rs   r‰   r¤   )rl   rl   )r   r   )r9   r:   r;   r<   r[   r&   r=   rM   r#   ÚNotImplementedErrorr7   r>   r?   r@   rA   s   @@r   r€   r€   Ö   sA   ù‡ € ñð$ ØØØØØ—'‘'Ø—<‘<÷+
ðZ 4G÷ Xô Xr   r€   c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚHFERBim  c                ó   <€ V ^8„  d   QhRR/# )rl   ÚreturnN© )ÚformatrB   s   "€r   Ú__annotate__ÚHFERB.__annotate__n  s   ø€ ÷ Cñ C˜tñ Cr   c                óä  <€ \         SV `  4        V^,          V n        Wn        \        P
                  ! 4       V n        \        P                  ! V P                  V P                  ^4      V n        \        P                  ! V P                  V P                  ^4      V n	        \        P                  ! ^^^4      V n        \        P                  ! V P                  V P                  ^^^4      V n        R# ©rl   N)r"   r#   Úmid_dimrR   r&   r=   r)   rŽ   Úlast_fcÚfcÚ	MaxPool2dÚmax_poolÚconv)r+   rR   r0   s   &&€r   r#   ÚHFERB.__init__n  s›   ø€ Ü‰ÑÔØ˜a•xˆŒØŒÜ—7’7“9ˆŒÜ—y’y §¡¨4¯8©8°QÓ7ˆŒô —)’)˜DŸL™L¨$¯,©,¸Ó:ˆŒÜŸš Q¨¨1Ó-ˆŒô —I’I˜dŸl™l¨D¯L©L¸!¸QÀÓBˆŽ	r   c                ó¤  € VP                   R ,          w  V n        V n        TpV P                  V P	                  VRRV P
                  1RR3,          4      4      pV P                  V P                  V P                  VRV P
                  R1RR3,          4      4      4      p\        P                  ! W4.^R7      pW P                  V4      ,           pV# )ºrl   NNrœ   Nr_   )r	   r£   r¤   r)   rÆ   rÁ   rÃ   rÅ   r¥   r§   rÂ   )r+   r4   ÚshortÚlfeÚhfes   &&   r   r7   ÚHFERB.forward|  s¨   € ØŸ™ �‰ˆŒ�”Øˆð �h‰h�t—y‘y  1 n¨¯© n°a¸Ð#:Õ!;Ó<Ó=ˆð �h‰h�t—w‘w˜tŸ}™}¨Q¨q°$·,±,±.À!ÀQÐ/FÕ-GÓHÓIÓJˆä�IŠI�s�j aÔ(ˆØ—L‘L “OÕ#ˆØˆr   )	r)   rÆ   rR   rÃ   r£   rÂ   rÅ   rÁ   r¤   ©	r9   r:   r;   r<   r#   r7   r>   r?   r@   rA   s   @@r   r¸   r¸   m  s   ù‡ € ÷Có C÷ò r   r¸   c                   ób   a a€ ] tR tRt oR]! R4      ]! R4      33V 3R lltR tR tRtVt	V ;t
# )Ú	Attentioni�  c           
     ó^  <€ \         SV `  4        W n        W@n        WPn        \
        P                  ! \        P                  ! V^^4      4      V n	        Wn
        \
        P                  ! RR7      V n        \
        P                  ! W^VR7      V n        \
        P                  ! W^^^WR7      V n        \
        P                  ! W^,          ^VR7      V n        \
        P                  ! V^,          V^,          ^^^V^,          VR7      V n        \
        P                  ! W^VR7      V n        R# )r›   r_   ©r…   r„   ©r…   r†   r‡   rˆ   r„   Nr   )r"   r#   rI   Ú
train_sizeÚ	base_sizer&   Ú	Parameterr¥   ÚonesÚtemperaturerR   rf   rg   rŽ   rw   Úq_dwconvÚkvÚ	kv_dwconvÚproject_out)r+   rR   rI   r„   rÔ   rÕ   r0   s   &&&&&&€r   r#   ÚAttention.__init__Ž  sã   ø€ ô 	‰ÑÔØ"ŒØ$ŒØ"ŒÜŸ<š<¬¯
ª
°9¸aÀÓ(CÓDˆÔØŒÜ—z’z bÔ)ˆŒä—’˜3°¸Ô>ˆŒÜŸ	š	Ø !¨A°qÀô
ˆŒô —)’)˜C q¥°a¸dÔCˆŒÜŸšØ�!�GØ�!�GØØØØ˜•7Øô
ˆŒô Ÿ9š9 S¸1À4ÔHˆÖr   c                óð  € VP                  ^^R7      w  r4\        VRV P                  R7      p\        VRV P                  R7      p\        VRV P                  R7      p\        P                  P
                  P                  VRR7      p\        P                  P
                  P                  VRR7      pWP                  RR4      ,          V P                  ,          pV P                  V4      pWT,          pV# )rl   r_   z b (head c) h w -> b head c (h w))Úheadr   rm   )
Úchunkr   rI   r¥   r&   Ú
functionalÚ	normalizern   rØ   rg   )r+   rw   rÚ   rx   ry   r{   Úouts   &&&    r   Ú_forwardÚAttention._forward®  sÅ   € Ø�x‰x˜˜qˆxÓ!‰ˆÜ�aÐ;À$Ç.Á.ÔQˆÜ�aÐ;À$Ç.Á.ÔQˆÜ�aÐ;À$Ç.Á.ÔQˆä�H‰H×Ñ×)Ñ)¨!°Ð)Ó4ˆÜ�H‰H×Ñ×)Ñ)¨!°Ð)Ó4ˆà—K‘K  BÓ'Õ'¨4×+;Ñ+;Õ;ˆØ�|‰|˜DÓ!ˆØ�hˆØˆ
r   c                ó‚  € VP                   R ,          w  V n        V n        V P                  V P	                  V4      4      pV P                  V P                  V4      4      pV P                  W44      p\        VRV P                  VP                   R,          VP                   R,          R7      pV P                  V4      pV# )rÉ   z b head c (h w) -> b (head c) h w)rß   r£   r¤   rm   r   )r	   r£   r¤   rÙ   rw   rÛ   rÚ   rä   r   rI   rÜ   )r+   ÚlowÚhighrw   rÚ   rã   s   &&&   r   r7   ÚAttention.forward¼  s•   € ØŸ™ 2�‰ˆŒ�”à�M‰M˜$Ÿ&™& ›,Ó'ˆØ�^‰^˜DŸG™G C›LÓ)ˆØ�m‰m˜AÓ"ˆÜØØ.Ø—‘Ø�h‰h�r�lØ�h‰h�r�lô
ˆð ×Ñ˜sÓ#ˆØˆ
r   )rÕ   rR   r£   rÚ   rÛ   rI   rÜ   rw   rÙ   rg   rØ   rÔ   r¤   )r›   rƒ   é0   rê   g      R@)r9   r:   r;   r<   r   r#   rä   r7   r>   r?   r@   rA   s   @@r   rÐ   rÐ   �  s0   ù‡ € ð "Ù�x“=¡# h£-Ð0÷Iò@÷ò r   rÐ   c                 ó   € \        V R 4      # )zb c h w -> b (h w) cr   )r4   s   &r   Úto_3drì   Í  s   € Ü�QÐ.Ó/Ð/r   c                 ó   € \        V R WR7      # )zb (h w) c -> b c h w)r£   r¤   r   )r4   r£   r¤   s   &&&r   Úto_4drî   Ñ  s   € Ü�QÐ.°!Ô9Ð9r   c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚBiasFree_LayerNormiÕ  c                ó$  <€ \         SV `  4        \        V\        P                  4      '       d   V3p\
        P                  ! V4      p\        V4      ^8X  g   Q h\        P                  ! \
        P                  ! V4      4      V n        Wn        R# ©r›   N)r"   r#   Ú
isinstanceÚnumbersÚIntegralr¥   ÚSizeÚlenr&   rÖ   r×   ÚweightÚnormalized_shape©r+   rù   r0   s   &&€r   r#   ÚBiasFree_LayerNorm.__init__Ö  so   ø€ Ü‰ÑÔÜÐ&¬×(8Ñ(8×9Ò9Ø 0Ð2ÐÜ Ÿ:š:Ð&6Ó7ÐäÐ#Ó$¨Ô)Ð)Ð)ä—l’l¤5§:¢:Ð.>Ó#?Ó@ˆŒØ 0Ör   c                ó”   € VP                  RRRR7      pV\        P                  ! VR,           4      ,          V P                  ,          # )r›   TF©ÚkeepdimÚunbiasedçñhãˆµøä>r   )Úvarr¥   Úsqrtrø   )r+   r4   Úsigmas   && r   r7   ÚBiasFree_LayerNorm.forwardá  s7   € Ø—‘�b $°�Ó7ˆØ”5—:’:˜e d�lÓ+Õ+¨d¯k©kÕ9Ð9r   )rù   rø   rÎ   rA   s   @@r   rð   rð   Õ  s   ù‡ € õ	1÷:ò :r   rð   c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚWithBias_LayerNormiæ  c                ó‚  <€ \         SV `  4        \        V\        P                  4      '       d   V3p\
        P                  ! V4      p\        V4      ^8X  g   Q h\        P                  ! \
        P                  ! V4      4      V n        \        P                  ! \
        P                  ! V4      4      V n        Wn        R# rò   )r"   r#   ró   rô   rõ   r¥   rö   r÷   r&   rÖ   r×   rø   Úzerosr„   rù   rú   s   &&€r   r#   ÚWithBias_LayerNorm.__init__ç  s‰   ø€ Ü‰ÑÔÜÐ&¬×(8Ñ(8×9Ò9Ø 0Ð2ÐÜ Ÿ:š:Ð&6Ó7ÐäÐ#Ó$¨Ô)Ð)Ð)ä—l’l¤5§:¢:Ð.>Ó#?Ó@ˆŒÜ—L’L¤§¢Ð-=Ó!>Ó?ˆŒ	Ø 0Ör   c                óè   € VP                  RRR7      pVP                  RRRR7      pW,
          \        P                  ! VR,           4      ,          V P                  ,          V P
                  ,           # )r›   T)rþ   Frý   r   r   )Úmeanr  r¥   r  rø   r„   )r+   r4   Úmur  s   &&  r   r7   ÚWithBias_LayerNorm.forwardó  sT   € Ø�V‰V�B ˆVÓ%ˆØ—‘�b $°�Ó7ˆØ•œ%Ÿ*š* U¨T¥\Ó2Õ2°T·[±[Õ@À4Ç9Á9ÕLÐLr   )r„   rù   rø   rÎ   rA   s   @@r   r  r  æ  s   ù‡ € õ
1÷Mò Mr   r  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )rM   iù  c                óv   <€ \         SV `  4        VR 8X  d   \        V4      V n        R# \	        V4      V n        R# )ÚBiasFreeN)r"   r#   rð   Úbodyr  )r+   rR   ÚLayerNorm_typer0   s   &&&€r   r#   ÚLayerNorm.__init__ú  s.   ø€ Ü‰ÑÔØ˜ZÔ'Ü*¨3Ó/ˆDŽIä*¨3Ó/ˆDŽIr   c                ól   € VP                   RR w  r#\        V P                  \        V4      4      W#4      # )rl   Nrm   )r	   rî   r  rì   )r+   r4   r£   r¤   s   &&  r   r7   ÚLayerNorm.forward  s-   € Ø�w‰w�r�sˆ|‰ˆÜ�T—Y‘Yœu Q›xÓ(¨!Ó/Ð/r   )r  rÎ   rA   s   @@r   rM   rM   ù  s   ù‡ € õ0÷0ò 0r   rM   c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚFeedForwardi  c           
     óR  <€ \         SV `  4        \        W,          4      pW@n        Wn        \
        P                  ! W^,          ^VR7      V n        \
        P                  ! V^,          V^,          ^^^V^,          VR7      V n        \
        P                  ! WA^VR7      V n	        R# )rl   rÒ   rÓ   N)
r"   r#   r   Úhid_fearR   r&   rŽ   Ú
project_inÚdwconvrÜ   )r+   rR   Úffn_expansion_factorr„   r-   r0   s   &&&& €r   r#   ÚFeedForward.__init__	  sŠ   ø€ Ü‰ÑÔä˜cÕ8Ó9ˆØ&ŒØŒäŸ)š) C¸1Õ)<È!ÐRVÔWˆŒä—i’iØ˜aÕØ˜aÕØØØØ" QÕ&Øô
ˆŒô Ÿ9š9 _ÀqÈtÔTˆÖr   c                ó  € VP                   R ,          w  V n        V n        V P                  V4      pV P	                  V4      P                  ^^R7      w  r#\        P                  ! V4      V,          pV P                  V4      pV# )rÉ   r_   )	r	   r£   r¤   r  r  rà   ÚFÚgelurÜ   )r+   r4   r±   r²   s   &&  r   r7   ÚFeedForward.forward  sj   € ØŸ™ �‰ˆŒ�”Ø�O‰O˜AÓˆØ—‘˜Q“×%Ñ% a¨QÐ%Ó/‰ˆÜ�FŠF�2‹J˜�OˆØ×Ñ˜QÓˆØˆr   )rR   r  r£   r  r  rÜ   r¤   rÎ   rA   s   @@r   r  r    s   ù‡ € õU÷*ò r   r  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )ÚHFBi(  a  Hybrid Fusion Block.

Args:
    dim (int): Number of input channels.
    num_heads (int): Number of attention heads.
    ffn_expansion_factor (int): Define the window size.
    bias (int): Shift size for SW-MSA.
    LayerNorm_type (float): Ratio of mlp hidden dim to embedding dim.
c                ó´   <€ \         SV `  4        \        W4      V n        \	        WV4      V n        \        W4      V n        \        WV4      V n        Wn	        R # r!   )
r"   r#   rM   r‹   rÐ   r{   r“   r  ÚffnrR   )r+   rR   rI   r  r„   r  r0   s   &&&&&&€r   r#   ÚHFB.__init__3  sG   ø€ Ü‰ÑÔä˜sÓ3ˆŒ
Ü˜c¨dÓ3ˆŒ	Ü˜sÓ3ˆŒ
Ü˜s¸$Ó?ˆŒØŽr   c                óà   € VP                   R ,          w  V n        V n        WP                  V P	                  V4      V4      ,           pW0P                  V P                  V4      4      ,           pV# )rÉ   )r	   r£   r¤   r{   r‹   r%  r“   )r+   rç   rè   r4   s   &&& r   r7   ÚHFB.forward<  sP   € ØŸ™ 2�‰ˆŒ�”Ø—)‘)˜DŸJ™J s›O¨TÓ2Õ2ˆØ—‘˜Ÿ™ A›Ó'Õ'ˆàˆr   )r{   rR   r%  r£   r‹   r“   r¤   rZ   rA   s   @@r   r#  r#  (  s   ù‡ € ñõ÷ò r   r#  c                   ó`   a a€ ] tR tRt oRt^^RRR]P                  3V 3R lltR tRt	Vt
V ;t# )	ÚCRFBiD  aò  Cross-Refinement Fusion Block.

Args:
    dim (int): Number of input channels.
    depth (int): Number of blocks.
    num_heads (int): Number of attention heads.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
    qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
    qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
    norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
r�   TNc
                ó*  <€ \         SV `  4        W n        \        P                  ! \        ^V,          4       U
u. uF9  p
\        TTWE.V
^,          ^ 8X  d   ^ ^ .MV^,          V^,          .VVVV	R7      NK;  	  up
4      V n        \        P                  ! \        V4       Uu. uF  p\        V4      NK  	  up4      V n	        \        P                  ! \        V4       U
u. uF  p
\        VVRRRR7      NK  	  up
4      V n        R# u up
i u upi u up
i )rl   )rR   rI   ra   r‰   rŠ   r•   rh   r–   gHáz®G@FÚWithBias)rR   rI   r  r„   r  N)r"   r#   Údepthr&   r�   r‘   r€   Úsrwa_blocksr¸   Úhfer_blocksr#  Ú	hf_blocks)r+   rR   r-  rI   Úsplit_size_0Úsplit_size_1rŠ   r•   rh   r–   r—   r5   r0   s   &&&&&&&&&&  €r   r#   ÚCRFB.__init__Q  s  ø€ ô 	‰ÑÔØŒ
ô Ÿ=š=ô ˜q 5�yÔ)óñ *�Aô ØØ'Ø ,Ð;à˜A� œ
ð !" 1™và&¨!Õ+¨\¸QÕ->Ð?Ø'Ø%Ø%Ø)÷ñ *ñó
ˆÔô& Ÿ=š=¼eÀE¼lÓ)K¹l¸¬%°®*¹lÑ)KÓLˆÔô Ÿšô ˜uœó	ñ &�Aô ØØ'Ø)-ØØ#-÷ñ &ñ	ó
ˆŽùò+ùò$ *Lùò	s   º?DÂ"DÃDc           
     ó  € VP                   w  rErg\        V P                  4       Fß  pVP                  ^ ^^^4      p	V	P	                  WFV,          V4      p	V P
                  ^V,          ^,           ,          ! V P
                  ^V,          ,          ! W’W3R,          4      VVVR,          4      p	V	P	                  WFWu4      p	V	P                  ^ ^^^4      p	V P                  V,          ! V4      p
V P                  V,          ! Wš4      pKá  	  V# )r   rª   )r	   r‘   r-  r   r   r.  r/  r0  )r+   r4   r¨   r©   r«   rV   r£   r¤   r—   rç   rè   s   &&&&       r   r7   ÚCRFB.forward„  sè   € Ø—W‘W‰
ˆˆaÜ�t—z‘zÖ"ˆAØ—)‘)˜A˜q ! QÓ'ˆCØ—+‘+˜a Q¥¨Ó*ˆCØ×"Ñ" 1 q¥5¨1¥9Ö-Ø× Ñ   Q¥Ö'¨°VÀKÕ=PÓQØØØ�{Õ#ó	ˆCð —+‘+˜a AÓ)ˆCØ—+‘+˜a  A qÓ)ˆCØ×#Ñ# AÖ& qÓ)ˆDØ—‘˜qÖ! #Ó,ŠAñ #ð ˆr   )r-  r0  r/  r.  ©r9   r:   r;   r<   r[   r&   rM   r#   r7   r>   r?   r@   rA   s   @@r   r*  r*  D  s2   ù‡ € ñ
ð" ØØØØØ—<‘<÷1
÷fò r   r*  c                   ó`   a a€ ] tR tRt oRtRRR^^]P                  3V 3R lltR tRt	Vt
V ;t# )	ÚRCRFGi–  aL  Residual Cross-Refinement Fusion Group (RCRFG).

Args:
    dim (int): Number of input channels.
    depth (int): Number of blocks.
    num_heads (int): Number of attention heads.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
    qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
    qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
    norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
    resi_connection: The convolutional block before residual connection.
r�   TNc
                óž   <€ \         S
V `  4        Wn        \        VVVVVVVVV	R 7	      V n        \
        P                  ! W^^^4      V n        R# )©	rR   r-  rI   rŠ   r•   rh   r1  r2  r–   N)r"   r#   rR   r*  Úresidual_groupr&   rŽ   rÆ   )r+   rR   r-  rI   rŠ   r•   rh   r1  r2  r–   r0   s   &&&&&&&&&&€r   r#   ÚRCRFG.__init__¤  sU   ø€ ô 	‰ÑÔàŒä"ØØØØØØØ%Ø%Ø!ô

ˆÔô —I’I˜c¨¨1¨aÓ0ˆŽ	r   c                óp   € Vw  V n         V n        V P                  V P                  WV4      4      V,           # r!   )r£   r¤   rÆ   r;  )r+   r4   r¨   r©   s   &&&&r   r7   ÚRCRFG.forwardÂ  s0   € Ø‰ˆŒ�”Ø�y‰y˜×,Ñ,¨Q¸Ó?Ó@À1ÕDÐDr   )rÆ   rR   r£   r;  r¤   r6  rA   s   @@r   r8  r8  –  s3   ù‡ € ñð$ ØØØØØ—<‘<÷1÷<Eò Er   r8  c                   ó:   a a€ ] tR tRt oRtRV 3R lltRtVtV ;t# )ÚUpsampleOneStepiÇ  a  UpsampleOneStep module (the difference with Upsample is that it always only has 1conv + 1pixelshuffle)
   Used in lightweight SR to save parameters.

Args:
    scale (int): Scale factor. Supported scales: 2^n and 3.
    num_feat (int): Channel number of intermediate features.

c           	     ó  <€ W n         W@n        Wn        . pVP                  \        P
                  ! W!^,          V,          ^^^4      4       VP                  \        P                  ! V4      4       \        SV `   ! V!   R# rÀ   )	Únum_featÚinput_resolutionrd   Úappendr&   rŽ   ÚPixelShuffler"   r#   )r+   rd   rB  Ú
num_out_chrC  Úmr0   s   &&&&& €r   r#   ÚUpsampleOneStep.__init__Ñ  s^   ø€ Ø ŒØ 0ÔØŒ
ØˆØ	�‰”—’˜8¨Q¥h°*Õ%<¸aÀÀAÓFÔGØ	�‰”—’ Ó'Ô(Ü‰Ò˜!Ôr   )rC  rB  rd   r!   )	r9   r:   r;   r<   r[   r#   r>   r?   r@   rA   s   @@r   r@  r@  Ç  s   ù‡ € ñ÷÷ r   r@  c                   ó  a a€ ] tR tRt oRt/ tR^R^R^0R. ROR. ROR^R	^R
RRRRRR]P                  R^RRRR/V 3R lltR t	R t
]P                  P                  R 4       t]P                  P                  R 4       tR tR tRtVtV ;t# ) ÚCRAFTiÛ  aE  Cross-Refinement Adaptive Fusion Transformer
    Some codes are based on SwinIR.
Args:
    in_chans (int): Number of input image channels. Default: 3
    embed_dim (int): Patch embedding dimension. Default: 96
    depths (tuple(int)): Depth of each Swin Transformer layer.
    num_heads (tuple(int)): Number of attention heads in different layers.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 2
    qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
    qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. Default: None
    norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
    upscale: Upscale factor. 2/3/4/
    img_range: Image range. 1. or 255.
    resi_connection: The convolutional block before residual connection. '1conv'/'3conv'
Úin_chansÚwindow_sizeÚ	embed_dimÚdepthsrI   r1  r2  rŠ   r�   r•   Trh   Nr–   ÚupscaleÚ	img_rangeç      ð?Úresi_connectionÚ1convc               óà  <€ \         SV `  4        W n        Wg3V n        TpTp^@pWÐn        VV n        VV n        V^8X  d1   Rp\        P                  ! V4      P                  ^^^^4      V n
        M\        P                  ! ^^^^4      V n
        WÀn        V P                  4        \        P                  ! Wó^^^4      V n        \#        V4      V n        W0n        W0n        W€n        \        P,                  ! 4       V n        \1        V P$                  4       FJ  p\3        VVV,          VV,          V P*                  V	V
VVVR7	      pV P.                  P5                  V4       KL  	  \7        V P(                  R4      V n        VR8X  d    \        P                  ! W3^^^4      V n        M!VR8X  d   \        P<                  ! 4       V n        \?        WÃV4      V n         V PC                  V PD                  4       R# )é@   r:  Ú	with_biasrS  ÚidentityN)gšwœ¢#¹Ü?gï8EGrùÛ?gB`åÐ"ÛÙ?)#r"   r#   rL  ra   rP  rB  rF  r¥   ÚTensorr
   r  r  rO  Úcalculate_rpi_v_sar&   rŽ   Ú
conv_firstr÷   Ú
num_layersrM  Únum_featuresrŠ   r�   Úlayersr‘   r8  rD  rM   ÚnormÚconv_after_bodyÚIdentityr@  ÚupsampleÚapplyÚ_init_weights)r+   rK  rL  rM  rN  rI   r1  r2  rŠ   r•   rh   r–   rO  rP  rR  Ú	num_in_chrF  rB  Úrgb_meanÚi_layerÚlayerr0   s   &$$$$$$$$$$$$$$      €r   r#   ÚCRAFT.__init__ï  s¤  ø€ ô& 	‰ÑÔà&ÔØ'Ð6ˆŒàˆ	Øˆ
ØˆØ"ŒØ ˆŒØ$ˆŒØ�qŒ=Ø/ˆHÜŸš XÓ.×3Ñ3°A°q¸!¸QÓ?ˆD�IäŸš A q¨!¨QÓ/ˆDŒIØŒð 	×ÑÔ!ô Ÿ)š) I¸!¸QÀÓBˆŒô ˜f›+ˆŒØ"ŒØ%ÔØ"Œô —m’m“oˆŒÜ˜TŸ_™_Ö-ˆGÜØØ˜W•oØ# GÕ,ØŸ.™.Ø!Ø!Ø)Ø)Ø%ô
ˆEð �K‰K×Ñ˜uÖ%ñ .ô ˜d×/Ñ/°Ó=ˆŒ	ð ˜gÔ%Ü#%§9¢9¨YÀ1ÀaÈÓ#KˆDÕ Ø 
Ô*Ü#%§;¢;£=ˆDÔ ô (¨¸JÓGˆŒà�
‰
�4×%Ñ%Ö&r   c                ó  € \        V\        P                  4      '       dv   \        VP                  R R7       \        V\        P                  4      '       d=   VP
                  e-   \        P                  P                  VP
                  ^ 4       R# R# R# \        V\        P                  4      '       dW   \        P                  P                  VP
                  ^ 4       \        P                  P                  VP                  R4       R# R# )g{®Gáz”?)ÚstdNrQ  )	ró   r&   r'   r   rø   r„   ÚinitÚ	constant_rM   )r+   rG  s   &&r   rc  ÚCRAFT._init_weights=  s¡   € Ü�aœŸ™×#Ò#Ü˜!Ÿ(™(¨Õ-Ü˜!œRŸY™Y×'Ò'¨A¯F©FÒ,>Ü—‘×!Ñ! !§&¡&¨!Ö,ñ -?Ñ'ä˜œ2Ÿ<™<×(Ò(Ü�G‰G×Ñ˜aŸf™f aÔ(Ü�G‰G×Ñ˜aŸh™h¨Ö,ñ )r   c                ó  € V P                   ^ ,          V P                   ^,          r!\        P                  ! ^V,
          V4      p\        P                  ! ^V,
          V4      p\        P                  ! \        P                  ! W4.4      4      pVP                  ^4      P                  ^ ^4      P                  4       P                  4       p\        P                  ! V4      p\        P                  ! V4      p\        P                  ! \        P                  ! Wg.4      4      p\        P
                  ! V^4      p	V	R,          V	R,          ,
          p
V
P                  ^^^ 4      P                  4       p
V
R;;,          V^,
          ,          uu&   V
R;;,          V^,
          ,          uu&   V
R;;,          ^V,          ^,
          ,          uu&   V
P                  R	4      pV P                   ^,          V P                   ^ ,          r!\        P                  ! ^V,
          V4      p\        P                  ! ^V,
          V4      p\        P                  ! \        P                  ! W4.4      4      pVP                  ^4      P                  ^ ^4      P                  4       P                  4       p\        P                  ! V4      p\        P                  ! V4      p\        P                  ! \        P                  ! Wg.4      4      p\        P
                  ! V^4      p	V	R,          V	R,          ,
          p
V
P                  ^^^ 4      P                  4       p
V
R;;,          V^,
          ,          uu&   V
R;;,          V^,
          ,          uu&   V
R;;,          ^V,          ^,
          ,          uu&   V
P                  R	4      pV P                  RV4       V P                  RV4       V P                  RV4       V P                  RV4       WÅ3# )
r   Úrelative_position_index_hÚrelative_position_index_vr¢   r    )rœ   rœ   N)rœ   Nrœ   )rœ   rœ   r   )rœ   rœ   r›   r   )ra   r¥   ÚarangeÚstackÚmeshgridÚflattenrn   r   Úfloatr   ÚsumÚregister_buffer)r+   r   r   Úposition_bias_hÚposition_bias_wr    Úcoords_hÚcoords_wÚcoordsÚcoords_flattenÚrelative_coordsro  r¢   rp  s   &             r   rY  ÚCRAFT.calculate_rpi_v_saF  s÷  € à—_‘_ QÕ'¨¯©¸Õ);ˆdÜŸ,š, q¨4¥x°Ó6ˆÜŸ,š, q¨4¥x°Ó6ˆÜ—;’;œuŸ~š~¨Ð.PÓQÓRˆØ×#Ñ# AÓ&×0Ñ0°°AÓ6×AÑAÓC×IÑIÓKˆô —<’< Ó%ˆÜ—<’< Ó%ˆÜ—’œUŸ^š^¨XÐ,@ÓAÓBˆÜŸš v¨qÓ1ˆØ(¨Õ4°~ÀjÕ7QÕQˆØ)×1Ñ1°!°Q¸Ó:×EÑEÓGˆØ˜×  D¨1¥HÕ,Ó Ø˜×  D¨1¥HÕ,Ó Ø˜×  A¨¥H¨q¥LÕ0Ó Ø$3×$7Ñ$7¸Ó$;Ð!à—_‘_ QÕ'¨¯©¸Õ);ˆdÜŸ,š, q¨4¥x°Ó6ˆÜŸ,š, q¨4¥x°Ó6ˆÜ—;’;œuŸ~š~¨Ð.PÓQÓRˆØ×#Ñ# AÓ&×0Ñ0°°AÓ6×AÑAÓC×IÑIÓKˆô —<’< Ó%ˆÜ—<’< Ó%ˆÜ—’œUŸ^š^¨XÐ,@ÓAÓBˆÜŸš v¨qÓ1ˆØ(¨Õ4°~ÀjÕ7QÕQˆØ)×1Ñ1°!°Q¸Ó:×EÑEÓGˆØ˜×  D¨1¥HÕ,Ó Ø˜×  D¨1¥HÕ,Ó Ø˜×  A¨¥H¨q¥LÕ0Ó Ø$3×$7Ñ$7¸Ó$;Ð!Ø×ÑÐ8Ð:SÔTØ×ÑÐ8Ð:SÔTØ×Ñ˜Z¨Ô2Ø×Ñ˜Z¨Ô2àÐ!Ð!r   c                ó   € R 0# )Úabsolute_pos_embedr»   ©r+   s   &r   Úno_weight_decayÚCRAFT.no_weight_decayr  s   € à$Ð%Ð%r   c                ó   € R 0# )Úrelative_position_bias_tabler»   r‚  s   &r   Úno_weight_decay_keywordsÚCRAFT.no_weight_decay_keywordsv  s   € à.Ð/Ð/r   c           
     ó  € VP                   ^,          VP                   ^,          3pRRRV P                  RV P                  RV P                  RV P                  /pV P
                   F  pV! WV4      pK  	  V P                  V4      pV# )rl   rª   rŸ   r¡   r¢   r    )NN)r	   ro  rp  r¢   r    r]  r^  )r+   r4   r¨   r©   rg  s   &&   r   Úforward_featuresÚCRAFT.forward_featuresz  s€   € Ø—'‘'˜!•*˜aŸg™g a�jÐ)ˆà˜Ø˜×6Ñ6Ø˜×6Ñ6Ø˜Ÿ™Ø˜Ÿ™ð
ˆð —[”[ˆEÙ�a Ó(ŠAñ !ð �I‰I�a‹Lˆàˆr   c                ó<  € VP                  4       w   r#pW0P                  ,          ^,           V P                  ,          V,
          pW@P                  ,          ^,           V P                  ,          V,
          pV^ 8g  ;'       g    V^ 8g  pV'       d…   \        P                  ! V\        P                  ! V^.4      .^4      RRRW5,           1R3,          p\        P                  ! V\        P                  ! V^.4      .^4      RRRRWF,           13,          pVP
                  R,          w  V n        V n        V P                  P                  V4      V n        WP                  ,
          V P                  ,          pV P                  V4      pV P                  V P                  V4      4      V,           pV P                  V4      pWP                  ,          V P                  ,           pV'       d1   VRRW0P                  ,          1RW@P                  ,          13,          pV# )r›   rœ   NrÉ   .)ÚsizerL  r¥   r§   Úflipr	   r£   r¤   r  Útype_asrP  rZ  r_  rŠ  ra  rO  )r+   r4   r5   Úh_oldÚw_oldÚh_padÚw_padÚpads   &&      r   r7   ÚCRAFT.forward‹  s¡  € àŸV™V›XÑˆˆ1�UØ×*Ñ*Õ*¨QÕ.°$×2BÑ2BÕBÀUÕJˆØ×*Ñ*Õ*¨QÕ.°$×2BÑ2BÕBÀUÕJˆØ�q‰j×&Ð&˜E Q™JˆçÜ—	’	˜1œeŸjšj¨¨Q¨CÓ0Ð1°1Ó5°a¸¸O¸e½m¸OÈQÐ6NÕOˆAÜ—	’	˜1œeŸjšj¨¨Q¨CÓ0Ð1°1Ó5°a¸¸A¸ÀÅ¸Ð6NÕOˆAàŸ™ �‰ˆŒ�”Ø—I‘I×%Ñ% aÓ(ˆŒ	Ø—‘�]˜dŸn™nÕ,ˆà�O‰O˜AÓˆØ× Ñ  ×!6Ñ!6°qÓ!9Ó:¸QÕ>ˆà�M‰M˜!ÓˆØ—‘Õ §¡Õ*ˆçØ�#Ð-˜§¡Õ-Ð-Ð/E°¿¹Õ1EÐ/EÐEÕFˆAàˆr   )r_  rZ  rM  r£   rP  r]  r  rŠ   r^  rB  r\  r[  rF  ra   ra  rO  r¤   rL  )rl   rl   rl   rl   )é   r–  r–  r–  )r9   r:   r;   r<   r[   Úhyperparametersr&   rM   r#   rc  rY  r¥   ÚjitÚignorerƒ  r‡  rŠ  r7   r>   r?   r@   rA   s   @@r   rJ  rJ  Û  s  ù‡ € ñð  €OðL'ð ðL'ð
 ðL'ð ðL'ò ðL'ò ðL'ð ðL'ð ðL'ð ðL'ð ðL'ð ðL'ð —<‘<ðL'ð ðL'ð  ð!L'ð"  ÷#L'ò\-ò*"ðX ‡Y�Y×Ññ&ó ð&ð ‡Y�Y×Ññ0ó ð0ò÷"ò r   rJ  )"rô   r¥   Útorch.nnr&   Útorch.nn.functionalrá   r  Úeinopsr   Útorch.nn.initr   Úspandrel.utilr   r   r   ÚModuler   rD   r]   r€   r¸   rÐ   rì   rî   rð   r  rM   r  r#  r*  r8  rL   r@  rJ  r»   r   r   Ú<module>r      sJ  ðó ã Ý ß Ð Ý Ý 'å /òò	ôˆ"�)‰)ô ô,(�R—Y‘Yô (ôVj˜Ÿ	™	ô jôZTˆB�I‰Iô TônˆB�I‰Iô ô@=�—	‘	ô =ò@0ò:ô:˜Ÿ™ô :ô"M˜Ÿ™ô Mô&
0�—	‘	ô 
0ô�"—)‘)ô ô@ˆ"�)‰)ô ô8Oˆ2�9‰9ô Oôd.EˆB�I‰Iô .Eôb�b—m‘mô ñ( ÓôGˆB�I‰Ió Gó òGr   