+
    ùäûj|c  ã                  ó  € R t ^ RIHt ^ RIHt ^ RIt^ RIHt ^ RIH	t	 ^ RI
HtHt ^RIHt ^RIHt ^RIHtHt ^R	IHtHtHtHtHtHt ^R
IHt ^RIHtHt  ! R R]P@                  4      t!]	! 4        ! R R]P@                  4      4       t"R# )a
  
Efficient and Explicit Modelling of Image Hierarchies for Image Restoration
Image restoration transformers with global, regional, and local modelling
A clean version of the.
Shared buffers are used for relative_coords_table, relative_position_index, and attn_mask.
)Úannotations)ÚLiteralN)Ústore_hyperparameters)Ú	to_2tupleÚtrunc_normal_)Úpad_to_multiple)Ú	GRLConfig)Ú EfficientMixAttnTransformerBlockÚget_stripe_info)Úbchw_to_blcÚblc_to_bchwÚcalculate_maskÚcalculate_mask_allÚget_relative_coords_table_allÚ"get_relative_position_index_simple)Úbuild_last_conv)ÚUpsampleÚUpsampleOneStepc                  ó†   a € ] tR t^$tRtRRRRR^RRR]P                  ^ ^ .^ ^ .RRR	R	R
3R V 3R llltR tR t	R t
RtV ;t# )ÚTransformerStagea„  
Transformer stage.

Args:
    dim (int): Number of input channels.
    input_resolution (tuple[int]): Input resolution.
    depth (int): Number of blocks.
    num_heads_window (list[int]): Number of window attention heads in different layers.
    num_heads_stripe (list[int]): Number of stripe attention heads in different layers.
    stripe_size (list[int]): Stripe size. Default: [8, 8]
    stripe_groups (list[int]): Number of stripe groups. Default: [None, None].
    stripe_shift (bool): whether to shift the stripes. This is used as an ablation study.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
    qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
    qkv_proj_type (str): QKV projection type. Default: linear. Choices: linear, separable_conv.
    anchor_proj_type (str): Anchor projection type. Default: avgpool. Choices: avgpool, maxpool, conv2d, separable_conv, patchmerging.
    anchor_one_stage (bool): Whether to use one operator or multiple progressive operators to reduce feature map resolution. Default: True.
    anchor_window_down_factor (int): The downscale factor used to get the anchors.
    drop (float, optional): Dropout rate. Default: 0.0
    attn_drop (float, optional): Attention dropout rate. Default: 0.0
    drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
    norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
    pretrained_window_size (list[int]): pretrained window size. This is actually not used. Default: [0, 0].
    pretrained_stripe_size (list[int]): pretrained stripe size. This is actually not used. Default: [0, 0].
    conv_type: The convolutional block before residual connection.
    init_method: initialization method of the weight parameters used to train large scale models.

        Choices:
            * n, normal -- Swin V1 init method.
            * l, layernorm -- Swin V2 init method. Zero the weight and bias in the post layer normalization layer.
            * r, res_rescale -- EDSR rescale method. Rescale the residual blocks with a scaling factor 0.1
            * w, weight_rescale -- MSRResNet rescale method. Rescale the weight parameter in residual blocks with a scaling factor 0.1
            * t, `trunc_normal_` -- nn.Linear, trunc_normal, nn.Conv2d, weight_rescale
    fairscale_checkpoint (bool): Whether to use fairscale checkpoint.
    offload_to_cpu (bool): used by fairscale_checkpoint
    args:
        out_proj_type (str): Type of the output projection in the self-attention modules. Default: linear. Choices: linear, conv2d.
        local_connection (bool): Whether to enable the local modelling module (two convs followed by Channel attention). For GRL base model, this is used.                "local_connection": local_connection,
        euclidean_dist (bool): use Euclidean distance or inner product as the similarity metric. An ablation study.
ç      @TÚlinearÚavgpoolç        Ú1convÚ FNc               ó4   € V ^8„  d   QhRRRRRRRRRRRRR	R
/# )é   ÚdimÚintÚinput_resolutionútuple[int, int]ÚdepthÚnum_heads_windowÚnum_heads_stripeÚwindow_sizeÚargsr   © )Úformats   "ÚaC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\spandrel\architectures\GRL\__arch\grl.pyÚ__annotate__ÚTransformerStage.__annotate__N   sY   € ÷ H4ñ H4àðH4ð *ðH4ð ð	H4ð
 ðH4ð ðH4ð %ðH4ð6 ñ7H4ó    c           
     	óx  <€ \         SV `  4        Wn        W n        VV n        \
        P                  ! 4       V n        \        V4       FÝ  p\        R!/ R VbRVbRVbRVbRVbRV^,          ^ 8H  bRVbRVbRV^,          ^ 8X  d   R	MR
bRV	'       d   V^,          R"9   MRbRV
bRVbRVbRVbRVbRVbRVbRVbR\        V\        4      '       d
   VV,          MTbRVbRVbRVbRVR8X  d   RMRbRVb pV'       d   ^ RIHp V! VVR7      pV P                  P                  V4       Kß  	  \        VV4      V n        R # )#r   r    Únum_heads_wÚnum_heads_sr%   Úwindow_shiftÚstripe_sizeÚstripe_groupsÚstripe_typeÚHÚWÚstripe_shiftFÚ	mlp_ratioÚqkv_biasÚqkv_proj_typeÚanchor_proj_typeÚanchor_one_stageÚanchor_window_down_factorÚdropÚ	attn_dropÚ	drop_pathÚ
norm_layerÚpretrained_window_sizeÚpretrained_stripe_sizeÚ	res_scaleÚrçš™™™™™¹?ç      ð?r&   )Úcheckpoint_wrapper)Úoffload_to_cpuNr'   )r   é   )ÚsuperÚ__init__r   r    Úinit_methodÚnnÚ
ModuleListÚblocksÚranger	   Ú
isinstanceÚlistÚfairscale.nnrG   Úappendr   Úconv)Úselfr   r    r"   r#   r$   r%   r1   r2   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   Ú	conv_typerL   Úfairscale_checkpointrH   r&   ÚiÚblockrG   Ú	__class__s   &&&&&&&&&&&&&&&&&&&&&&&&&&&   €r)   rK   ÚTransformerStage.__init__N   s¬  ø€ ô: 	‰ÑÔàŒØ 0ÔØ&ˆÔä—m’m“oˆŒÜ�u–ˆAÜ4ò Ùðá!1ðñ -ðñ -ð	ñ
 (ðð  �U ašZðñ (ðñ ,ðð $% q¥5¨A¤:™C°3øð÷ 1=˜Q �U fš_À%øðñ $ðñ "ðñ ,ðñ "2ðñ "2ðñ  +Dð!ñ" ð#ñ$ $ð%ô& +5°YÄ×*EÒ*E˜) Až,È9øð'ñ( &ð)ñ* (>ð+ñ, (>ð-ð. "-°Ô!3™#¸øð/ñ0 ð1ˆE÷8 $Ý;á*¨5ÀÔP�à�K‰K×Ñ˜uÖ%ñE ôH $ I¨sÓ3ˆŽ	r,   c                	óx  € V P                  4        EF$  w  rV P                  R 8X  dp   \        V\        P                  \        P
                  34      '       d>   VP                  R4      ^ 8  d&   VP                  ;P                  R,          un        K‚  K„  K†  V P                  R8X  d{   \        V\        P                  4      '       dX   \        P                  P                  VP                  ^ 4       \        P                  P                  VP                  ^ 4       EK  EK  V P                  P                  R4      ^ 8¼  dÞ   R\        V P                  4      ^,
          ,          \        V P                  R	,          4      ,          p\        V\        P                  4      '       d1   VP                  R4      ^ 8  d   \        VP                  VR7       EKÅ  \        V\        P
                  4      '       d'   VP                  ;P                  R,          un        EK  EK  \!        RV P                   R24      h	  R# )
ÚwÚcpb_mlprE   ÚlÚt©Ústdz Parameter initialization method z% not implemented in TransformerStage.Néÿÿÿÿ)Únamed_modulesrL   rQ   rM   ÚLinearÚConv2dÚfindÚweightÚdataÚ	LayerNormÚinitÚ	constant_ÚbiasÚlenr   r   ÚNotImplementedError)rV   ÚnÚmÚscales   &   r)   Ú_init_weightsÚTransformerStage._init_weights˜   s‹  € Ø×&Ñ&×(‰DˆAØ×Ñ 3Ô&Ü˜a¤"§)¡)¬R¯Y©YÐ!7×8Ò8¸Q¿V¹VÀIÓ=NÐQRÔ=Rà—H‘H—M’M SÕ(—Mñ >SÑ8ð ×!Ñ! SÔ(Ü˜a¤§¡×.Ò.ä—G‘G×%Ñ% a§f¡f¨aÔ0Ü—G‘G×%Ñ% a§h¡h°×2ò /ð ×!Ñ!×&Ñ& sÓ+¨qÔ0Ø¤ D×$4Ñ$4Ó 5¸Õ 9Õ:¼SÀ×AQÑAQÐRTÕAUÓ=VÕV�Ü˜a¤§¡×+Ò+°·±°yÓ0AÀAÔ0EÜ! !§(¡(°×6Ð6Ü ¤2§9¡9×-Ò-Ø—H‘H—M’M SÕ(—M�Mò .ô *Ø6°t×7GÑ7GÐ6HÐHmÐnóð ó) )r,   c                	ó”   € TpV P                    F  pV! WBV4      pK  	  \        V P                  \        WB4      4      4      pWA,           # ©N)rO   r   rU   r   )rV   ÚxÚx_sizeÚtable_index_maskÚresÚblks   &&&&  r)   ÚforwardÚTransformerStage.forward±   sA   € ØˆØ—;”;ˆCÙ�cÐ#3Ó4ŠCñ ä˜$Ÿ)™)¤K°Ó$<Ó=Ó>ˆà�wˆr,   c                	ó   € R # rw   r'   ©rV   s   &r)   ÚflopsÚTransformerStage.flops¹   s   € Ùr,   )rO   rU   r   rL   r    )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rM   rk   rK   rt   r}   r�   Ú__static_attributes__Ú__classcell__©r[   s   @r)   r   r   $   sk   ø† ñ'ðh ØØØ"ØØ"#ØØØØ—<‘<Ø ! 1˜vØ ! 1˜vØØØ"ØØ÷7H4ò H4òTò2÷ð r,   r   c            B      óŽ  a € ] tR t^½tRt/ tR^@R^R^R^`R^RRR	R
R. R<OR. R=OR. R=OR^R^^.RRR.RRRRRRRRRRRRR^RRRRR R!R"R!R#R$R%]P                  R&^ ^ .R'^ ^ .R(R)R*R+R,RR-RR.R/!R/ V 3R0 llltR1 R2 lt	R3 R4 lt
R5 t]P                  P                  R6 4       t]P                  P                  R7 4       tR8 tR9 tR: tR;tV ;t# )>ÚGRLa„  Image restoration transformer with global, non-local, and local connections

Args:
    img_size (int | list[int]): Input image size. Default 64
    in_channels (int): Number of input image channels. Default: 3
    out_channels (int): Number of output image channels. Default: None
    embed_dim (int): Patch embedding dimension. Default: 96
    upscale (int): Upscale factor. 2/3/4/8 for image SR, 1 for denoising and compress artifact reduction
    img_range (float): Image range. 1. or 255.
    upsampler (str): The reconstruction reconstruction module. 'pixelshuffle'/'pixelshuffledirect'/'nearest+conv'/None
    depths (list[int]): Depth of each Swin Transformer layer.
    num_heads_window (list[int]): Number of window attention heads in different layers.
    num_heads_stripe (list[int]): Number of stripe attention heads in different layers.
    window_size (int): Window size. Default: 8.
    stripe_size (list[int]): Stripe size. Default: [8, 8]
    stripe_groups (list[int]): Number of stripe groups. Default: [None, None].
    stripe_shift (bool): whether to shift the stripes. This is used as an ablation study.
    mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4
    qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
    qkv_proj_type (str): QKV projection type. Default: linear. Choices: linear, separable_conv.
    anchor_proj_type (str): Anchor projection type. Default: avgpool. Choices: avgpool, maxpool, conv2d, separable_conv, patchmerging.
    anchor_one_stage (bool): Whether to use one operator or multiple progressive operators to reduce feature map resolution. Default: True.
    anchor_window_down_factor (int): The downscale factor used to get the anchors.
    out_proj_type (str): Type of the output projection in the self-attention modules. Default: linear. Choices: linear, conv2d.
    local_connection (bool): Whether to enable the local modelling module (two convs followed by Channel attention). For GRL base model, this is used.
    drop_rate (float): Dropout rate. Default: 0
    attn_drop_rate (float): Attention dropout rate. Default: 0
    drop_path_rate (float): Stochastic depth rate. Default: 0.1
    pretrained_window_size (list[int]): pretrained window size. This is actually not used. Default: [0, 0].
    pretrained_stripe_size (list[int]): pretrained stripe size. This is actually not used. Default: [0, 0].
    norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
    conv_type (str): The convolutional block before residual connection. Default: 1conv. Choices: 1conv, 3conv, 1conv1x1, linear
    init_method: initialization method of the weight parameters used to train large scale models.

        Choices:
            * n, normal -- Swin V1 init method.
            * l, layernorm -- Swin V2 init method. Zero the weight and bias in the post layer normalization layer.
            * r, res_rescale -- EDSR rescale method. Rescale the residual blocks with a scaling factor 0.1
            * w, weight_rescale -- MSRResNet rescale method. Rescale the weight parameter in residual blocks with a scaling factor 0.1
            * t, `trunc_normal_` -- nn.Linear, trunc_normal, nn.Conv2d, weight_rescale
    fairscale_checkpoint (bool): Whether to use fairscale checkpoint.
    offload_to_cpu (bool): used by fairscale_checkpoint
    euclidean_dist (bool): use Euclidean distance or inner product as the similarity metric. An ablation study.

Úimg_sizeÚin_channelsÚout_channelsÚ	embed_dimÚupscaleÚ	img_rangerF   Ú	upsamplerr   Údepthsr#   r$   r%   r1   r2   Nr6   Fr7   r   r8   Tr9   r   r:   r   r;   r<   Úout_proj_typeÚlocal_connectionÚ	drop_rater   Úattn_drop_rateÚdrop_path_raterE   r@   rA   rB   rW   r   rL   rq   rX   rH   Úeuclidean_distc               ó@   € V ^8„  d   QhRRRRRRRRRRRRR	R
RRRRRR/
# )r   rŽ   r   r�   r”   z	list[int]r#   r$   r1   r2   zlist[int | None]r•   zLiteral['linear', 'conv2d']rA   rB   r'   )r(   s   "r)   r*   ÚGRL.__annotate__ï   s|   € ÷ c&ñ c&ð ð	c&ð
 ðc&ð ðc&ð $ðc&ð $ðc&ð ðc&ð (ðc&ð. 3ð/c&ð: !*ð;c&ð< !*ñ=c&r,   c       !        	ó  <€ \         S/V `  4        W n        W0n        ^@p"W@n        WPn        Wpn        W`n        V^8X  d1   R)p#\        P                  ! V#4      P                  ^^^^4      V n        M\        P                  ! ^^^^4      V n        \        V U$u. uF  p$V$f   ^ MT$NK  	  up$4      p%\        V U$u. uF  p$V$f   ^ MT$NK  	  up$4      p&V&V,          p&\        VV%V&4      V n        \        V4      V n        \        V4      V n        V P"                   U'u. uF  p'V'^,          NK  	  up'V n        WÀn        WÐn        VV n        VV n        VV n        \0        P2                  ! W$^^^4      V n        V! V4      V n        \0        P8                  ! VR7      V n        \        P<                  ! ^ V\?        V4      4       U(u. uF  p(V(PA                  4       NK  	  p)p(\C        VVV!R7      p*V PE                  V P                   4      PG                  4        F  w  p+p,V PI                  V+V,RR7       K  	  \0        PJ                  ! 4       V n&        \O        \Q        V4      4       FÀ  p-\S        R*/ RVbRV P                   bRVV-,          bR	V	V-,          bR
V
V-,          bRV P"                  bRVbRVbRVbRVbRVbRVbRVbRVbRVbRVbRVbRV)\?        VRV- 4      \?        VRV-^,            4       bRVbRVbRVbRVbRVbRVbRV bRV*b p.V PL                  PU                  V.4       KÂ  	  V! V4      V n+        \Y        VV4      V n-        V P                  R 8X  d|   \0        P\                  ! \0        P2                  ! VV"^^^4      \0        P^                  ! R!R"7      4      V n0        \c        VV"4      V n2        \0        P2                  ! V"V^^^4      V n3        EMOV P                  R#8X  d   \i        VVV4      V n2        EM*V P                  R$8X  dü   V P
                  ^8X  g   Q R%4       h\0        P\                  ! \0        P2                  ! VV"^^^4      \0        P^                  ! R!R"7      4      V n0        \0        P2                  ! V"V"^^^4      V n5        \0        P2                  ! V"V"^^^4      V n6        \0        P2                  ! V"V"^^^4      V n7        \0        P2                  ! V"V^^^4      V n3        \0        P^                  ! R&R!R'7      V n8        M\0        P2                  ! WC^^^4      V n3        V Ps                  V Pt                  4       VR+9   g   VPw                  R(4      ^ 8¼  d&   V PL                   F  p.V.Pu                  4        K  	  R# R# u up$i u up$i u up'i u up(i ),é@   N)Úp)r•   r–   rš   F)Ú
persistentr   r    r"   r#   r$   r%   r1   r2   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rW   rL   rX   rH   r&   ÚpixelshuffleT)ÚinplaceÚpixelshuffledirectúnearest+convzonly support x4 now.gš™™™™™É?)Únegative_sloper¢   ra   )gšwœ¢#¹Ü?gï8EGrùÛ?gB`åÐ"ÛÙ?r'   )r`   r^   )<rJ   rK   rŽ   r�   r�   r‘   r“   r’   ÚtorchÚTensorÚviewÚmeanÚzerosÚmaxÚpad_sizer   r    r%   Ú
shift_sizer1   r2   rA   rB   r<   rM   rg   Ú
conv_firstÚ
norm_startÚDropoutÚpos_dropÚlinspaceÚsumÚitemr   Úset_table_index_maskÚitemsÚregister_bufferrN   ÚlayersrP   ro   r   rT   Únorm_endr   Úconv_after_bodyÚ
SequentialÚ	LeakyReLUÚconv_before_upsampler   ÚupsampleÚ	conv_lastr   Úconv_up1Úconv_up2Úconv_hrÚlreluÚapplyrt   rh   )0rV   r�   rŽ   r�   r�   r‘   r’   r“   r”   r#   r$   r%   r1   r2   r6   r7   r8   r9   r:   r;   r<   r•   r–   r—   r˜   r™   r@   rA   rB   rW   rL   rX   rH   rš   Únum_out_featsÚrgb_meanÚsÚmax_stripe_sizeÚmax_stripe_groupsr^   rx   Údprr&   ÚkÚvrY   Úlayerr[   s0   &$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$             €r)   rK   ÚGRL.__init__ï   sh  ø€ ôJ 	‰ÑÔà&ÔØ(ÔØˆØ"ŒØŒØ"ŒØ"ŒØ˜!ÔØ/ˆHÜŸš XÓ.×3Ñ3°A°q¸!¸QÓ?ˆD�IäŸš A q¨!¨QÓ/ˆDŒIä¹kÓJ¹k¸ A¢I™q°1Ò4¹kÑJÓKˆÜÁÓ NÁ¸1 a¢i¡°QÒ!6ÁÑ NÓOÐØÐ6Õ6ÐÜ˜K¨Ð:KÓLˆŒô !*¨(Ó 3ˆÔÜ$ [Ó1ˆÔØ+/×+;Ò+;Ó<Ñ+; a˜1 Ÿ6˜6Ñ+;Ñ<ˆŒØ&ÔØ*ÔØ&<ˆÔ#Ø&<ˆÔ#Ø)BˆÔ&ô Ÿ)š) K¸A¸qÀ!ÓDˆŒñ % YÓ/ˆŒÜŸ
š
 YÔ/ˆŒô "'§¢°°>Ä3ÀvÃ;Ô!OÓPÑ!O˜Aˆq�v‰vŽxÑ!OˆÐPäØ'Ø-Ø)ô
ˆð
 ×-Ñ-¨d×.CÑ.CÓD×JÑJÖL‰DˆAˆqØ× Ñ   A°%Ð Ö8ñ Mô —m’m“oˆŒÜ”s˜6“{Ö#ˆAÜ$ò Ùðà!%×!6Ò!6ðð ˜Q–iðð "2°!Ö!4ð	ð
 "2°!Ö!4ðð !×,Ò,ðñ (ðñ ,ðñ *ðñ $ðñ "ðñ ,ðñ "2ðñ "2ðñ +Dðñ  ð!ñ" )ð#ð$ Ü˜˜r ˜
“O¤cØ˜w  Q¥˜ó'ñð%ñ. &ð/ñ0 (>ð1ñ2 (>ð3ñ4 $ð5ñ6 (ð7ñ8 &:ð9ñ:  .ð;ñ< ð=ˆEð@ �K‰K×Ñ˜uÖ%ñC $ñD # 9Ó-ˆŒô  /¨y¸)ÓDˆÔð �>‰>˜^Ô+ä(*¯ªÜ—	’	˜) ]°A°q¸!Ó<¼b¿lºlÐSWÔ>Xó)ˆDÔ%ô % W¨mÓ<ˆDŒMÜŸYšY }°lÀAÀqÈ!ÓLˆDŽNØ�^‰^Ð3Ô3ä+ØØØóˆDŽMð
 �^‰^˜~Ô-à—<‘< 1Ô$Ð<Ð&<Ó<Ð$Ü(*¯ªÜ—	’	˜) ]°A°q¸!Ó<¼b¿lºlÐSWÔ>Xó)ˆDÔ%ô ŸIšI m°]ÀAÀqÈ!ÓLˆDŒMÜŸIšI m°]ÀAÀqÈ!ÓLˆDŒMÜŸ9š9 ]°MÀ1ÀaÈÓKˆDŒLÜŸYšY }°lÀAÀqÈ!ÓLˆDŒNÜŸš°SÀ$ÔGˆD�Jô  ŸYšY yÀÀ1ÀaÓHˆDŒNà�
‰
�4×%Ñ%Ô&Ø˜*Ô$¨×(8Ñ(8¸Ó(=ÀÔ(BØŸœ�Ø×#Ñ#Ö%ó %ñ )Cùò[ KùÚ Nùò =ùò Qs   ÂU5Â5U:ÄU?Æ;Vc               ó   € V ^8„  d   QhRR/# )r   ry   r!   r'   )r(   s   "r)   r*   rœ   ”  s   € ÷ -
ñ -
¨?ñ -
r,   c                óú  € \        V P                  V P                  RV4      w  r#V P                  p\	        V P
                  V P                  4      p\	        W P                  V4      p\	        VRRR1,          V P                  V4      p\        V P
                  4      p\        W$R4      p	\        W$R4      p
\        VRRR1,          VR4      p\        VRRR1,          VR4      p\        WP
                  V P                  4      p\        WW4R4      p\        WW4R4      p\        WRRR1,          VRRR1,          VR4      p\        WRRR1,          VRRR1,          VR4      pRVRVRVRVRV	R	V
R
VRVRVRVRVRVRV/# )z’
Two used cases:
1) At initialization: set the shared buffers.
2) During forward pass: get the new buffers if the resolution of the input changes
TNFÚtable_wÚtable_shÚtable_svÚindex_wÚindex_sh_a2wÚindex_sh_w2aÚindex_sv_a2wÚindex_sv_w2aÚmask_wÚmask_sh_a2wÚmask_sh_w2aÚmask_sv_a2wÚmask_sv_w2ard   )r
   r1   r2   r<   r   r%   rA   rB   r   r   r­   r   )rV   ry   ÚssÚsssÚdfrÑ   rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   s   &&                r)   rµ   ÚGRL.set_table_index_mask”  s�  € ô " $×"2Ñ"2°D×4FÑ4FÈÈfÓU‰ˆØ×+Ñ+ˆä/Ø×Ñ˜d×9Ñ9ó
ˆô 1°×5PÑ5PÐRTÓUˆÜ0Ø‰t�ˆt�H�d×1Ñ1°2ó
ˆô 5°T×5EÑ5EÓFˆÜ9¸"À%ÓHˆÜ9¸"À$ÓGˆÜ9¸"¹T¸r¸T½(ÀBÈÓNˆÜ9¸"¹T¸r¸T½(ÀBÈÓMˆä ×(8Ñ(8¸$¿/¹/ÓJˆÜ(¨°S¸eÓDˆÜ(¨°S¸dÓCˆÜ(¨±D°b°Dµ¸3¹tÀ¸t½9ÀbÈ%ÓPˆÜ(¨±D°b°Dµ¸3¹tÀ¸t½9ÀbÈ$ÓOˆà�wØ˜Ø˜Ø�wØ˜LØ˜LØ˜LØ˜LØ�fØ˜;Ø˜;Ø˜;Ø˜;ð
ð 	
r,   c               ó   € V ^8„  d   QhRR/# )r   r    r!   r'   )r(   s   "r)   r*   rœ   Ã  s   € ÷ $ñ $¸_ñ $r,   c                	óÜ  € W P                   8X  dŸ   R V P                  RV P                  RV P                  RV P                  RV P
                  RV P                  RV P                  RV P                  RV P                  R	V P                  R
V P                  RV P                  RV P                  /# V P                  V4      pVP                  4        F  w  rEVP!                  V4      W4&   K  	  V# )rÑ   rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   )r    rÑ   rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rµ   r¶   Úto)rV   Údevicer    rz   rË   rÌ   s   &&&   r)   Úget_table_index_maskÚGRL.get_table_index_maskÃ  sÝ   € à×4Ñ4Ô4à˜4Ÿ<™<Ø˜DŸM™MØ˜DŸM™MØ˜4Ÿ<™<Ø × 1Ñ 1Ø × 1Ñ 1Ø × 1Ñ 1Ø × 1Ñ 1Ø˜$Ÿ+™+Ø˜t×/Ñ/Ø˜t×/Ñ/Ø˜t×/Ñ/Ø˜t×/Ñ/ðð ð   $×8Ñ8Ð9IÓJÐØ(×.Ñ.Ö0‘�Ø&'§d¡d¨6£lÐ Ó#ñ 1à#Ð#r,   c                	ó  € \        V\        P                  4      '       dv   \        VP                  R R7       \        V\        P                  4      '       d=   VP
                  e-   \        P                  P                  VP
                  ^ 4       R# R# R# \        V\        P                  4      '       dW   \        P                  P                  VP
                  ^ 4       \        P                  P                  VP                  R4       R# R# )g{®Gáz”?rb   NrF   )	rQ   rM   rf   r   ri   rn   rl   rm   rk   )rV   rr   s   &&r)   rt   ÚGRL._init_weightsÛ  s£   € Ü�aœŸ™×#Ò#ô ˜!Ÿ(™(¨Õ-Ü˜!œRŸY™Y×'Ò'¨A¯F©FÒ,>Ü—‘×!Ñ! !§&¡&¨!Ö,ñ -?Ñ'ä˜œ2Ÿ<™<×(Ò(Ü�G‰G×Ñ˜aŸf™f aÔ(Ü�G‰G×Ñ˜aŸh™h¨Ö,ñ )r,   c                	ó   € R 0# )Úabsolute_pos_embedr'   r€   s   &r)   Úno_weight_decayÚGRL.no_weight_decayë  s   € à$Ð%Ð%r,   c                	ó   € R 0# )Úrelative_position_bias_tabler'   r€   s   &r)   Úno_weight_decay_keywordsÚGRL.no_weight_decay_keywordsï  s   € à.Ð/Ð/r,   c                	ó‚   €  \        WP                  R R7      #   \         d    \        YP                  RR7      u # i ; i)Úreflect)ÚmodeÚconstant)r   r¬   ÚBaseException)rV   rx   s   &&r)   Úcheck_image_sizeÚGRL.check_image_sizeó  s;   € ð	FÜ" 1§m¡m¸)ÔDÐDøÜô 	FÜ" 1§m¡m¸*ÔEÒEð	Fús   ‚ ™">½>c                	óT  € VP                   ^,          VP                   ^,          3p\        V4      pV P                  V4      pV P                  V4      pV P	                  VP
                  V4      pV P                   F  pV! WV4      pK  	  V P                  V4      p\        W4      pV# )r   )	Úshaper   r¯   r±   ræ   rå   r¸   r¹   r   )rV   rx   ry   rz   rÍ   s   &&   r)   Úforward_featuresÚGRL.forward_featuresù  s’   € Ø—'‘'˜!•*˜aŸg™g a�jÐ)ˆÜ˜‹NˆØ�O‰O˜AÓˆØ�M‰M˜!Óˆà×4Ñ4°Q·X±X¸vÓFÐØ—[”[ˆEÙ�aÐ!1Ó2ŠAñ !ð �M‰M˜!ÓˆÜ˜Ó"ˆàˆr,   c           
     	ób  € VP                   R ,          w  r#V P                  V4      pV P                  P                  V4      V n        WP                  ,
          V P                  ,          pV P
                  R8X  dl   V P                  V4      pV P                  V P                  V4      4      V,           pV P                  V4      pV P                  V P                  V4      4      pEM÷V P
                  R8X  dL   V P                  V4      pV P                  V P                  V4      4      V,           pV P                  V4      pEM›V P
                  R8X  Ed   V P                  V4      pV P                  V P                  V4      4      V,           pV P                  V4      pV P                  V P                  \        P                  P                   P#                  V^RR7      4      4      pV P                  V P%                  \        P                  P                   P#                  V^RR7      4      4      pV P                  V P                  V P'                  V4      4      4      pM|V P                  V4      pV P                  V P                  V4      4      V,           pV P(                  V P*                  8X  d   WP                  V4      ,           pMV P                  V4      pWP                  ,          V P                  ,           pVRRRW P,                  ,          1RW0P,                  ,          13,          # ):r   NNr¡   r£   r¤   Únearest)Úscale_factorrô   :NNNN)rú   r÷   r©   Útype_asr’   r“   r®   rº   rû   r½   r¿   r¾   rÃ   rÀ   r¦   rM   Ú
functionalÚinterpolaterÁ   rÂ   rŽ   r�   r‘   )rV   rx   r4   r5   Úx_firstr{   s   &&    r)   r}   ÚGRL.forward  sz  € Ø�w‰w�r�{‰ˆØ×!Ñ! !Ó$ˆà—I‘I×%Ñ% aÓ(ˆŒ	Ø—‘�]˜dŸn™nÕ,ˆà�>‰>˜^Ô+à—‘ Ó"ˆAØ×$Ñ$ T×%:Ñ%:¸1Ó%=Ó>ÀÕBˆAØ×)Ñ)¨!Ó,ˆAØ—‘˜tŸ}™}¨QÓ/Ó0ŠAØ�^‰^Ð3Ô3à—‘ Ó"ˆAØ×$Ñ$ T×%:Ñ%:¸1Ó%=Ó>ÀÕBˆAØ—‘˜aÓ ŠAØ�^‰^˜~Õ-à—‘ Ó"ˆAØ×$Ñ$ T×%:Ñ%:¸1Ó%=Ó>ÀÕBˆAØ×)Ñ)¨!Ó,ˆAØ—
‘
Ø—‘Ü—H‘H×'Ñ'×3Ñ3°AÀAÈIÐ3ÓVóóˆAð
 —
‘
Ø—‘Ü—H‘H×'Ñ'×3Ñ3°AÀAÈIÐ3ÓVóóˆAð
 —‘˜tŸz™z¨$¯,©,°q«/Ó:Ó;‰Að —o‘o aÓ(ˆGØ×&Ñ& t×'<Ñ'<¸WÓ'EÓFÈÕPˆCØ×Ñ 4×#4Ñ#4Ô4ØŸ™ sÓ+Õ+‘à—N‘N 3Ó'�à—‘Õ §¡Õ*ˆà��AÐ)˜Ÿ\™\Õ)Ð)Ð+=¨Q·±Õ-=Ð+=Ð=Õ>Ð>r,   )r<   rº   r½   r®   rÂ   r¿   rÀ   rÁ   r�   r’   rŽ   r    r¸   rÃ   r©   r¹   r¯   r�   r¬   r±   rB   rA   r­   r2   r1   r¾   r“   r‘   r%   )é   r  r  r  r  r  )rI   rI   rI   rI   rI   rI   )rƒ   r„   r…   r†   r‡   ÚhyperparametersrM   rk   rK   rµ   ræ   rt   r¦   ÚjitÚignorerì   rð   r÷   rû   r}   rˆ   r‰   rŠ   s   @r)   rŒ   rŒ   ½   s  ø† ñ,ð\ €Oðc&ð ðc&ð ð	c&ð
 ðc&ð ðc&ð ðc&ð ðc&ð ðc&ò /ðc&ò '9ðc&ò '9ðc&ð ðc&ð #$ Q ðc&ð ,0°¨,ðc&ð  ð!c&ð" ð#c&ð$ ð%c&ð& ð'c&ð( #ð)c&ð* ð+c&ð, #$ð-c&ð. 6>ð/c&ð0 ð1c&ð2 ð3c&ð4 ð5c&ð6 ð7c&ð8 —<‘<ð9c&ð: ./°¨Fð;c&ð< ./°¨Fð=c&ð> ð?c&ð@ ðAc&ðB #ðCc&ðD ðEc&ðF ÷Gc&ò c&õJ-
õ^$ò0-ð  ‡Y�Y×Ññ&ó ð&ð ‡Y�Y×Ññ0ó ð0òFò÷-?ð -?r,   rŒ   )#r‡   Ú
__future__r   Útypingr   r¦   Útorch.nnrM   Úspandrel.utilr   Úspandrel.util.timmr   r   Ú__arch_helpers.paddingr   Úconfigr   Úmixed_attn_block_efficientr	   r
   Úopsr   r   r   r   r   r   Úswin_v1_blockr   r¾   r   r   ÚModuler   rŒ   r'   r,   r)   Ú<module>r     ss   ðñõ #å ã Ý å /ß 7å 5Ý ÷÷÷ õ +ß /ôV�r—y‘yô Vñr Óôw?ˆ"�)‰)ó w?ó òw?r,   