+
    þäûjÈ  ã                   ó´   € R t ^ RIt^ RIHt ^ RIHu Ht RR R lltRR R llt ! R R]P                  4      t
RR R	 llt ! R
 R]P                  4      tR# )a<  DropBlock, DropPath

PyTorch implementations of DropBlock and DropPath (Stochastic Depth) regularization layers.

Papers:
DropBlock: A regularization method for convolutional networks (https://arxiv.org/abs/1810.12890)

Deep Networks with Stochastic Depth (https://arxiv.org/abs/1603.09382)

Code:
DropBlock impl inspired by two Tensorflow impl that I liked:

- https://github.com/tensorflow/tpu/blob/master/models/official/resnet/resnet_model.py#L74
- https://github.com/clovaai/assembled-cnn/blob/master/nets/blocks.py

Hacked together by / Copyright 2020 Ross Wightman
Nc                ó€   € V ^8„  d   QhR\         P                  R\        R\        R\        R\        R\        R\        /# )é   ÚxÚ	drop_probÚ
block_sizeÚgamma_scaleÚ
with_noiseÚinplaceÚ	batchwise©ÚtorchÚTensorÚfloatÚintÚbool)Úformats   "ÚUC:\Ai_Coding\apps\studio\ComfyUI\.venv\Lib\site-packages\spandrel\util\timm\__drop.pyÚ__annotate__r      s[   € ÷ Bñ BÜ‡|�|ðBäðBô ðBô ð	Bô
 ðBô ðBô ñBó    c                óÊ  € V P                   w  rxršW©,          p\        V\        W©4      4      pVV,          V,          V^,          ,          W¢,
          ^,           W’,
          ^,           ,          ,          p\        P                  ! \        P                  ! V
4      P                  V P                  4      \        P                  ! V	4      P                  V P                  4      4      w  rïWì^,          8¬  WêV^,
          ^,          ,
          8  ,          Wü^,          8¬  WùV^,
          ^,          ,
          8  ,          ,          p\        P                  ! V^^Wš34      P                  V P                  R7      pV'       d2   \        P                  ! ^W‰V
3V P                  V P                  R7      pM\        P                  ! V 4      p^V,
          V,
          V,           ^8¬  P                  V P                  R7      p\        P                  ! V) V^V^,          R7      ) pV'       d¨   V'       d1   \        P                  ! ^W‰V
3V P                  V P                  R7      M\        P                  ! V 4      pV'       d1   V P                  V4      P!                  V^V,
          ,          4       V # V V,          V^V,
          ,          ,           p  V # VP#                  4       VP                  \        P$                  R7      P'                  4       P)                  R4      ,          P                  V P                  4      pV'       d   V P                  VV,          4       V # V V,          V,          p V # )züDropBlock. See https://arxiv.org/pdf/1810.12890.pdf

DropBlock with an experimental gaussian noise option. This layer has been tested on a few training
runs with success, but needs further validation and possibly optimization for lower runtime impact.
©Údtype)r   Údevice©Úkernel_sizeÚstrideÚpaddinggH¯¼šò×z>)ÚshapeÚminr   ÚmeshgridÚarangeÚtor   Úreshaper   ÚrandÚ	rand_likeÚFÚ
max_pool2dÚrandnÚ
randn_likeÚmul_Úadd_ÚnumelÚfloat32ÚsumÚadd)r   r   r   r   r   r	   r
   Ú_ÚCÚHÚWÚ
total_sizeÚclipped_block_sizeÚgammaÚw_iÚh_iÚvalid_blockÚuniform_noiseÚ
block_maskÚnormal_noiseÚnormalize_scales   &&&&&&&              r   Údrop_block_2dr=      s´  € ð —‘�J€Aˆ!Ø•€JÜ˜Z¬¨Q«Ó3Ðð 	Ø
õ	à
õ	ð ˜aÕ
õ	 ð �N˜QÕ 1¥>°AÕ#5Õ6õ		8ð 
ô �~Š~Ü�Š�Q‹×Ñ˜1Ÿ8™8Ó$¤e§l¢l°1£o×&8Ñ&8¸¿¹Ó&Bó�H€Cð 
 aÕ'Ñ	'¨CÐ7IÈAÕ7MÐRSÕ6SÕ2SÑ,SÕTØ
 qÕ(Ñ
(¨SÐ8JÈQÕ8NÐSTÕ7TÕ3TÑ-TÕ	UõW€Kô —-’- ¨a°°A¨\Ó:×=Ñ=ÀAÇGÁGÐ=ÓL€KçäŸ
š
 A q¨Q <°q·w±wÀqÇxÁxÔP‰äŸš¨Ó*ˆØ�u•9˜{Õ*¨]Õ:¸qÑ@×DÑDÈ1Ï7É7ÐDÓS€JÜ—,’,Ø	ˆØ&ØØ" aÕ'ô	ð €J÷ ÷ ô �KŠK˜˜A !˜¨A¯G©G¸A¿H¹HÕEä×!Ò! !Ó$ð 	÷
 Ø�F‰F�:Ó×#Ñ# L°A¸
µNÕ$CÔDð €Hð �J• °°ZµÕ!@Õ@‰Að €Hð ×ÑÓ §¡´U·]±] Ó!C×!GÑ!GÓ!I×!MÑ!MÈdÓ!SÕSß
‰"ˆQ�W‰W‹+ð 	÷ Ø�F‰F�: Õ/Ô0ð €Hð �J• Õ0ˆAØ€Hr   c                ót   € V ^8„  d   QhR\         P                  R\        R\        R\        R\        R\        /# )r   r   r   r   r   r   r	   r   )r   s   "r   r   r   ]   sH   € ÷ /ñ /Ü‡|�|ð/äð/ô ð/ô ð	/ô
 ð/ô ñ/r   c                óú  € V P                   w   rgpW‡,          p	\        V\        W‡4      4      p
VV,          V	,          V
^,          ,          W‚,
          ^,           Wr,
          ^,           ,          ,          p\        P                  ! V 4      P	                  V4      p\
        P                  ! VP                  V P                  4      V
^V
^,          R7      pV'       d|   \        P                  ! V 4      P                  4       pV'       d0   V P                  RV,
          4      P                  WÜ,          4       V # V RV,
          ,          WÜ,          ,           p  V # ^V,
          pVP                  4       VP                  \        P                  R7      P                  4       P                  R4      ,          P                  V P                  R7      pV'       d   V P                  WÎ,          4       V # W,          V,          p V # )z®DropBlock. See https://arxiv.org/pdf/1810.12890.pdf

DropBlock with an experimental gaussian noise option. Simplied from above without concern for valid
block mask at edges.
r   ç      ð?r   g�íµ ÷Æ°>)r   r   r   Ú
empty_likeÚ
bernoulli_r%   r&   r!   r   Únormal_r)   r*   r+   r,   r-   r.   )r   r   r   r   r   r	   r/   r1   r2   r3   r4   r5   r:   r;   r<   s   &&&&&&         r   Údrop_block_fast_2drD   ]   s¡  € ð —‘�J€A€qˆQØ•€JÜ˜Z¬¨Q«Ó3ÐàØ
õ	à
õ	ð ˜aÕ
õ	 ð �N˜QÕ 1¥>°AÕ#5Õ6õ		8ð 
ô ×!Ò! !Ó$×/Ñ/°Ó6€JÜ—’Ø�‰�a—g‘gÓØ&ØØ" aÕ'ô	€J÷ Ü×'Ò'¨Ó*×2Ñ2Ó4ˆßØ�F‰F�3˜Õ#Ó$×)Ñ)¨,Õ*CÔDð €Hð �S˜:Õ%Õ&¨Õ)BÕB‰Að €Hð ˜•^ˆ
à×ÑÓ §¡´U·]±] Ó!C×!GÑ!GÓ!I×!MÑ!MÈdÓ!SÕSß
‰"�1—7‘7ˆ"Ó
ð 	÷ Ø�F‰F�:Õ/Ô0ð €Hð • Õ0ˆAØ€Hr   c                   óX   a a€ ] tR t^�t oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚDropBlock2dz3DropBlock. See https://arxiv.org/pdf/1810.12890.pdfc                óD   <€ V ^8„  d   QhRS[ RS[RS[ RS[RS[RS[RS[/# )r   r   r   r   r   r	   r
   Úfast)r   r   r   )r   Ú__classdict__s   "€r   r   ÚDropBlock2d.__annotate__’   sQ   ø€ ÷ ñ áðñ ðñ ð	ñ
 ðñ ðñ ðñ ñr   c                óx   <€ \         SV `  4        Wn        W0n        W n        W@n        WPn        W`n        Wpn        R # ©N)	ÚsuperÚ__init__r   r   r   r   r	   r
   rH   )	Úselfr   r   r   r   r	   r
   rH   Ú	__class__s	   &&&&&&&&€r   rN   ÚDropBlock2d.__init__’   s4   ø€ ô 	‰ÑÔØ"ŒØ&ÔØ$ŒØ$ŒØŒØ"ŒØŽ	r   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   r   ©r   r   )r   rI   s   "€r   r   rJ   ¥   s   ø€ ÷ ñ ™Ÿ™ñ r   c           	     ó�  € V P                   '       d   V P                  '       g   V# V P                  '       dC   \        VV P                  V P                  V P
                  V P                  V P                  4      # \        VV P                  V P                  V P
                  V P                  V P                  V P                  4      # rL   )
Útrainingr   rH   rD   r   r   r   r	   r=   r
   ©rO   r   s   &&r   ÚforwardÚDropBlock2d.forward¥   s”   € Ø�}�}ˆ} D§N§N NØˆHØ�9�9ˆ9Ü%ØØ—‘Ø—‘Ø× Ñ Ø—‘Ø—‘óð ô !ØØ—‘Ø—‘Ø× Ñ Ø—‘Ø—‘Ø—‘óð r   )r
   r   r   rH   r   r	   r   )çš™™™™™¹?é   r@   FFFT)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rN   rX   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rP   rI   s   @@r   rF   rF   �   s!   ù‡ € Ù=÷õ ÷&÷ ð r   rF   c                ó\   € V ^8„  d   QhR\         P                  R\        R\        R\        /# )r   r   r   rV   Úscale_by_keep)r   r   r   r   )r   s   "r   r   r   ½   s4   € ÷ ñ Ü‡|�|ðäðô ðô ñ	r   c                ó,  € VR8X  g	   V'       g   V # ^V,
          pV P                   ^ ,          3RV P                  ^,
          ,          ,           pV P                  V4      P                  V4      pVR8”  d   V'       d   VP	                  V4       W,          # )a  Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,
the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for
changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use
'survival rate' as the argument.

ç        )é   )r   ÚndimÚ	new_emptyrB   Údiv_)r   r   rV   rf   Ú	keep_probr   Úrandom_tensors   &&&&   r   Ú	drop_pathro   ½   s}   € ð �CÔŸxØˆØ�I•€IØ�W‰W�Q�ZˆM˜DØ	�‰��
õõ €Eð —K‘K Ó&×1Ñ1°)Ó<€MØ�3„Ÿ=Ø×Ñ˜9Ô%ØÕÐr   c                   ó^   a a€ ] tR t^Øt oRtRV3R lV 3R llltV3R lR ltR tRtVt	V ;t
# )	ÚDropPathzYDrop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r   rf   )r   r   )r   rI   s   "€r   r   ÚDropPath.__annotate__Û   s   ø€ ÷ +ñ +¡%ð +¹dñ +r   c                ó<   <€ \         SV `  4        Wn        W n        R # rL   )rM   rN   r   rf   )rO   r   rf   rP   s   &&&€r   rN   ÚDropPath.__init__Û   s   ø€ Ü‰ÑÔØ"ŒØ*Ör   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rS   rT   )r   rI   s   "€r   r   rs   à   s   ø€ ÷ Oñ O™Ÿ™ñ Or   c                óX   € \        WP                  V P                  V P                  4      # rL   )ro   r   rV   rf   rW   s   &&r   rX   ÚDropPath.forwardà   s   € Ü˜ŸN™N¨D¯M©M¸4×;MÑ;MÓNÐNr   c                ó6   € R \        V P                  ^4      R 2# )z
drop_prob=z0.3f)Úroundr   )rO   s   &r   Ú
extra_reprÚDropPath.extra_reprã   s   € ØœE $§.¡.°Ó3°DÐ9Ð:Ð:r   )r   rf   )rh   T)r\   r]   r^   r_   r`   rN   rX   r{   ra   rb   rc   rd   s   @@r   rq   rq   Ø   s(   ù‡ € Ùc÷+õ +÷
Oð O÷;ò ;r   rq   )rZ   r[   r@   FFF)rZ   r[   r@   FF)rh   FT)r`   r   Útorch.nnÚnnÚtorch.nn.functionalÚ
functionalr%   r=   rD   ÚModulerF   ro   rq   © r   r   Ú<module>rƒ      sK   ðñó$ Ý ß Ð ÷B÷J/ôd+�"—)‘)ô +÷\ô6;ˆr�y‰yö ;r   