+
    ƒ°ñj
  ã                   óf  € R t ^ RIt^ RIHt ^ RIHtHtHt ^ RIt	 ! R R]P                  4      t ! R R]P                  4      tR t ! R R	]P                  4      t ! R
 R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tR# )a¥  
This file is part of ComfyUI.
Copyright (C) 2024 Stability AI

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.
N)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attentionc                   óB   a a€ ] tR t^t oRV 3R llt/ 3R ltRtVtV ;t# )ÚOptimizedAttentionc                ó  <€ \         SV `  4        \        4       V n        W n        VP                  WR WER7      V n        VP                  WR WER7      V n        VP                  WR WER7      V n        VP                  WR WER7      V n	        R# )T)ÚbiasÚdtypeÚdeviceN)
ÚsuperÚ__init__r   Úcomfy_attentionÚheadsÚLinearÚto_qÚto_kÚto_vÚout_proj©ÚselfÚcÚnheadÚdropoutr	   r
   Ú
operationsÚ	__class__s   &&&&&&&€Ú<C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\cascade\common.pyr   ÚOptimizedAttention.__init__   s‚   ø€ Ü‰ÑÔÜ-Ó/ˆÔØŒ
à×%Ñ% a°¸UÐ%ÓRˆŒ	Ø×%Ñ% a°¸UÐ%ÓRˆŒ	Ø×%Ñ% a°¸UÐ%ÓRˆŒ	à"×)Ñ)¨!°TÀÐ)ÓVˆŽó    c           	     ó  € V P                  V4      pV P                  V4      pV P                  V4      p\        V4      \        V4      \        V4      r2p\	        WW0P
                  V P                  VR 7      pV P                  V4      # ))Úpreferred_attentionÚtransformer_options)r   r   r   r   r   r   r   r   )r   ÚqÚkÚvr    Úouts   &&&&& r   ÚforwardÚOptimizedAttention.forward$   sw   € Ø�I‰I�a‹LˆØ�I‰I�a‹LˆØ�I‰I�a‹Lˆä*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆÜ! !¨¯:©:È4×K_ÑK_ð  vIô  Jˆà�}‰}˜SÓ!Ð!r   )r   r   r   r   r   r   ©ç        NNN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r%   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   Ú__classdict__s   @@r   r   r      s   ù‡ € ÷	Wð 46÷ "ô "r   r   c                   óD   a a€ ] tR t^.t oRV 3R lltR/ 3R ltRtVtV ;t# )ÚAttention2Dc                óJ   <€ \         SV `  4        \        WWEVR 7      V n        R# )©r	   r
   r   N)r   r   r   Úattnr   s   &&&&&&&€r   r   ÚAttention2D.__init__/   s   ø€ Ü‰ÑÔÜ& q°uÐXbÔcˆŽ	r   Fc                óL  € VP                   pVP                  VP                  ^ 4      VP                  ^4      R4      P                  ^ ^^4      pV'       d   \        P
                  ! W.^R7      pV P                  WW$R7      pVP                  ^ ^^4      P                  ! V!  pV# )é    ©Údim)r    éÿÿÿÿ)ÚshapeÚviewÚsizeÚpermuteÚtorchÚcatr7   )r   ÚxÚkvÚ	self_attnr    Ú
orig_shapes   &&&&& r   r%   ÚAttention2D.forward4   s…   € Ø—W‘Wˆ
Ø�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨Ó,×4Ñ4°Q¸¸1Ó=ˆßÜ—’˜A˜7¨Ô*ˆBà�I‰I�a˜RˆIÓIˆØ�I‰I�a˜˜AÓ×#Ò# ZÑ0ˆØˆr   )r7   r'   r)   r1   s   @@r   r4   r4   .   s   ù‡ € ÷dð
 (-À"÷ ô r   r4   c                 ó0   €  ! R  RV P                   4      pV# )c                   ó>   a a€ ] tR t^@t oV 3R ltV 3R ltRtVtV ;t# )Ú#LayerNorm2d_op.<locals>.LayerNorm2dc                ó(   <€ \         SV `  ! V/ VB  R # ©N)r   r   )r   ÚargsÚkwargsr   s   &*,€r   r   Ú,LayerNorm2d_op.<locals>.LayerNorm2d.__init__A   s   ø€ Ü‰GÒ˜dÐ- fÔ-r   c                ój   <€ \         SV `  VP                  ^ ^^^4      4      P                  ^ ^^^4      # ©r:   )r   r%   rA   )r   rD   r   s   &&€r   r%   Ú+LayerNorm2d_op.<locals>.LayerNorm2d.forwardD   s2   ø€ Ü‘7‘? 1§9¡9¨Q°°1°aÓ#8Ó9×AÑAÀ!ÀQÈÈ1ÓMÐMr   © r)   r1   s   @@r   ÚLayerNorm2drK   @   s   ù‡ € õ	.÷	Nõ 	Nr   rU   )Ú	LayerNorm)r   rU   s   & r   ÚLayerNorm2d_oprW   ?   s   € ôN�j×*Ñ*ô Nð Ðr   c                   ó@   a a€ ] tR t^Ht oRtRV 3R lltR tRtVtV ;t	# )ÚGlobalResponseNormzwfrom https://github.com/facebookresearch/ConvNeXt-V2/blob/3608f67cc1dae164790c5d0aead7bf2d73d9719b/models/utils.py#L105c                óô   <€ \         SV `  4        \        P                  ! \        P
                  ! ^^^WVR7      4      V n        \        P                  ! \        P
                  ! ^^^WVR7      4      V n        R# )é   ©r	   r
   N)r   r   ÚnnÚ	ParameterrB   ÚemptyÚgammaÚbeta)r   r<   r	   r
   r   s   &&&&€r   r   ÚGlobalResponseNorm.__init__J   sQ   ø€ Ü‰ÑÔÜ—\’\¤%§+¢+¨a°°A°sÐPVÔ"WÓXˆŒ
Ü—L’L¤§¢¨Q°°1°cÈvÔ!VÓWˆŽ	r   c                óP  € \         P                  ! V^RRR7      pW"P                  RRR7      R,           ,          p\        P                  P                  V P                  V4      W,          ,          \        P                  P                  V P                  V4      ,           V,           # )é   T)Úpr<   Úkeepdim)r<   rf   ç�íµ ÷Æ°>)r[   rd   r=   )rB   ÚnormÚmeanÚcomfyÚopsÚcast_to_inputr`   ra   )r   rD   ÚGxÚNxs   &&  r   r%   ÚGlobalResponseNorm.forwardO   sv   € Ü�ZŠZ˜˜Q F°DÔ9ˆØ—7‘7˜r¨4�7Ó0°4Õ7Õ8ˆÜ�y‰y×&Ñ& t§z¡z°1Ó5¸½Õ@Ä5Ç9Á9×CZÑCZÐ[_×[dÑ[dÐfgÓChÕhÐklÕlÐlr   )ra   r`   )NN)
r*   r+   r,   r-   Ú__doc__r   r%   r.   r/   r0   r1   s   @@r   rY   rY   H   s   ù‡ € Ù}÷X÷
mò mr   rY   c                   ó@   a a€ ] tR t^Ut oRV 3R lltRR ltRtVtV ;t# )ÚResBlockc                óº  <€ \         SV `  4        VP                  WW3^,          WVR7      V n        \	        V4      ! VRRWVR7      V n        \        P                  ! VP                  W,           V^,          WVR7      \        P                  ! 4       \        V^,          WVR7      \        P                  ! V4      VP                  V^,          WVR7      4      V n        R# )rd   )Úkernel_sizeÚpaddingÚgroupsr	   r
   Frg   ©Úelementwise_affineÚepsr	   r
   r\   N)r   r   ÚConv2dÚ	depthwiserW   rh   r]   Ú
Sequentialr   ÚGELUrY   ÚDropoutÚchannelwise)	r   r   Úc_skiprt   r   r	   r
   r   r   s	   &&&&&&&&€r   r   ÚResBlock.__init__V   s¹   ø€ Ü‰ÑÔØ#×*Ñ*¨1¸[ÐabÕRbÐklð  CIÐ*ó  JˆŒä" :Ô.¨qÀUÐPTÐ\aÔqˆŒ	ÜŸ=š=Ø×Ñ˜a�j¨!¨a­%°uÐÓLÜ�GŠG‹IÜ˜q 1�u¨EÔAÜ�JŠJ�wÓØ×Ñ˜a !�e Q¸FÐÓCó
ˆÖr   c                óú   € TpV P                  V P                  V4      4      pVe   \        P                  ! W.^R7      pV P	                  VP                  ^ ^^^4      4      P                  ^ ^^^4      pW,           # )Nr;   )rh   r{   rB   rC   r   rA   )r   rD   Úx_skipÚx_ress   &&& r   r%   ÚResBlock.forwardc   sn   € ØˆØ�I‰I�d—n‘n QÓ'Ó(ˆØÒÜ—	’	˜1˜+¨1Ô-ˆAØ×Ñ˜QŸY™Y q¨!¨Q°Ó2Ó3×;Ñ;¸A¸qÀ!ÀQÓGˆØ�yÐr   )r   r{   rh   )r:   é   r(   NNNrM   r)   r1   s   @@r   rr   rr   U   s   ù‡ € ÷
÷ô r   rr   c                   óB   a a€ ] tR t^lt oRV 3R llt/ 3R ltRtVtV ;t# )Ú	AttnBlockc	           
     ó
  <€ \         S	V `  4        W@n        \        V4      ! VR RWgR7      V n        \        WWVWxR7      V n        \        P                  ! \        P                  ! 4       VP                  W!WgR7      4      V n        R# )Frg   rw   r6   r\   N)r   r   rF   rW   rh   r4   Ú	attentionr]   r|   ÚSiLUr   Ú	kv_mapper)
r   r   Úc_condr   rF   r   r	   r
   r   r   s
   &&&&&&&&&€r   r   ÚAttnBlock.__init__m   sh   ø€ Ü‰ÑÔØ"ŒÜ" :Ô.¨qÀUÐPTÐ\aÔqˆŒ	Ü$ Q¨wÈFÔjˆŒÜŸšÜ�GŠG‹IØ×Ñ˜f¨uÐÓDó
ˆŽr   c                óŽ   € V P                  V4      pWP                  V P                  V4      W P                  VR 7      ,           pV# ))rF   r    )rŒ   rŠ   rh   rF   )r   rD   rE   r    s   &&&&r   r%   ÚAttnBlock.forwardw   s9   € Ø�^‰^˜BÓˆØ—‘˜tŸy™y¨›|¨R¿>¹>Ð_r�ÓsÕsˆØˆr   )rŠ   rŒ   rh   rF   )Tr(   NNNr)   r1   s   @@r   rˆ   rˆ   l   s   ù‡ € ÷
ð 24÷ ô r   rˆ   c                   ó<   a a€ ] tR t^}t oRV 3R lltR tRtVtV ;t# )ÚFeedForwardBlockc                ój  <€ \         SV `  4        \        V4      ! VR RW4R7      V n        \        P
                  ! VP                  W^,          W4R7      \        P                  ! 4       \        V^,          W4R7      \        P                  ! V4      VP                  V^,          WVR7      4      V n
        R# )Frg   rw   r\   N)r   r   rW   rh   r]   r|   r   r}   rY   r~   r   )r   r   r   r	   r
   r   r   s   &&&&&&€r   r   ÚFeedForwardBlock.__init__~   s‹   ø€ Ü‰ÑÔÜ" :Ô.¨qÀUÐPTÐ\aÔqˆŒ	ÜŸ=š=Ø×Ñ˜a Q¥¨eÐÓCÜ�GŠG‹IÜ˜q 1�u¨EÔAÜ�JŠJ�wÓØ×Ñ˜a !�e Q¸FÐÓCó
ˆÖr   c           	     óš   € WP                  V P                  V4      P                  ^ ^^^4      4      P                  ^ ^^^4      ,           pV# rR   )r   rh   rA   )r   rD   s   &&r   r%   ÚFeedForwardBlock.forward‰   sF   € Ø× Ñ  §¡¨1£×!5Ñ!5°a¸¸A¸qÓ!AÓB×JÑJÈ1ÈaÐQRÐTUÓVÕVˆØˆr   )r   rh   r'   r)   r1   s   @@r   r’   r’   }   s   ù‡ € ÷	
÷ò r   r’   c                   óF   a a€ ] tR t^Žt oR.RRR3V 3R lltR tRtVtV ;t# )ÚTimestepBlockÚscaNc                óÐ   <€ \         SV `  4        VP                  W!^,          WER7      V n        W0n        V F+  p\        V RV 2VP                  W!^,          WER7      4       K-  	  R# )rd   r\   Úmapper_N)r   r   r   ÚmapperÚcondsÚsetattr)	r   r   Ú
c_timestepr�   r	   r
   r   Úcnamer   s	   &&&&&&& €r   r   ÚTimestepBlock.__init__�   s`   ø€ Ü‰ÑÔØ ×'Ñ'¨
¸µEÀÐ'ÓVˆŒØŒ
ÛˆEÜ�D˜G E 7Ð+¨Z×->Ñ->¸zÈqÍ5ÐX]Ð->Ó-mÖnó r   c                óÀ  € VP                  \        V P                  4      ^,           ^R7      pV P                  V^ ,          4      R,          P                  ^^R7      w  r4\	        V P                  4       FP  w  rV\        V RV 24      ! W%^,           ,          4      R,          P                  ^^R7      w  rxW7,           WH,           rCKR  	  V^V,           ,          V,           # )r[   r;   r›   )ºNNNr£   NN)ÚchunkÚlenr�   rœ   Ú	enumerateÚgetattr)	r   rD   ÚtÚaÚbÚir   ÚacÚbcs	   &&&      r   r%   ÚTimestepBlock.forward–   s¶   € Ø�G‰G”C˜Ÿ
™
“O aÕ'¨QˆGÓ/ˆØ�{‰{˜1˜Q�4Ó Ð!1Õ2×8Ñ8¸ÀÐ8ÓB‰ˆÜ˜dŸj™jÖ)‰DˆAÜ˜T W¨Q¨C =Ô1°!¸µEµ(Ó;Ð<LÕM×SÑSÐTUÐ[\ÐSÓ]‰FˆBØ•6˜1�6Šqñ *ð �A˜•E�{˜Q�Ðr   )r�   rœ   r)   r1   s   @@r   r˜   r˜   Ž   s!   ù‡ € Ø-2¨G¸4ÈÐY]÷ o÷ò r   r˜   )rp   rB   Útorch.nnr]   Úcomfy.ldm.modules.attentionr   r   r   Ú	comfy.opsrj   ÚModuler   r4   rW   rY   rr   rˆ   r’   r˜   rT   r   r   Ú<module>r³      s’   ðñó$ Ý ß eÑ eÛ ô"˜Ÿ™ô "ô,�"—)‘)ô ò"ô
m˜Ÿ™ô 
môˆr�y‰yô ô.�—	‘	ô ô"�r—y‘yô ô"�B—I‘Iö r   