+
    ƒ°ñjE  ã                   óü  € ^ RI t ^ RIHt ^ RIHu Ht ^ RIHt ^ RIH	t	H
t
Ht ^ RIHtHtHt R tR t ! R R]P$                  4      t ! R	 R
]P$                  4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R]	4      t ! R R]P$                  4      t ! R R]P$                  4      t ! R R]P$                  4      tR# )é    N)Ú	rearrange)ÚTimestepEmbedderÚMlpÚtimestep_embedding)ÚAttentionTensorContainerÚComfyAttentionÚoptimized_attentionc                 ól   € V ^VP                  ^4      ,           ,          VP                  ^4      ,           # ©é   )Ú	unsqueeze©ÚxÚshiftÚscales   &&&Ú;C:\Ai_Coding\apps\studio\ComfyUI\comfy\ldm\pixart\blocks.pyÚmodulater      s)   € Ø��E—O‘O AÓ&Õ&Õ'¨%¯/©/¸!Ó*<Õ<Ð<ó    c                 ó0   € V ^V,           ,          V,           # r   © r   s   &&&r   Út2i_modulater      s   € Ø��E•	�?˜UÕ"Ð"r   c                   ó@   a a€ ] tR t^t oRV 3R lltRR ltRtVtV ;t# )ÚMultiHeadCrossAttentionc                ó®  <€ \         \        V `  4        \        4       V n        W,          ^ 8X  g   Q R4       hWn        W n        W,          V n        VP                  WWVR7      V n	        VP                  W^,          WVR7      V n
        \        P                  ! V4      V n        VP                  WWVR7      V n        \        P                  ! V4      V n        R# )r   z&d_model must be divisible by num_heads©ÚdtypeÚdeviceN)Úsuperr   Ú__init__r   Úcomfy_attentionÚd_modelÚ	num_headsÚhead_dimÚLinearÚq_linearÚ	kv_linearÚnnÚDropoutÚ	attn_dropÚprojÚ	proj_drop)
Úselfr!   r"   r)   r+   r   r   Ú
operationsÚkwargsÚ	__class__s
   &&&&&&&&,€r   r   Ú MultiHeadCrossAttention.__init__   s°   ø€ ÜÔ% tÑ5Ô7Ü-Ó/ˆÔØÕ" aÔ'ÐQÐ)QÓQÐ'àŒØ"ŒØÕ,ˆŒà"×)Ñ)¨'À%Ð)ÓWˆŒØ#×*Ñ*¨7¸AµIÀUÐ*ÓZˆŒÜŸš IÓ.ˆŒØ×%Ñ% g¸eÐ%ÓSˆŒ	ÜŸš IÓ.ˆŽr   c           	     óf  € VP                   w  rEpV P                  V4      P                  ^RV P                  V P                  4      pV P                  V4      P                  ^R^V P                  V P                  4      pVP                  ^4      w  ršVe   Q h\        VP                  VRV4      4      \        V	P                  VRV4      4      \        V
P                  VRV4      4      r©p?\        WyW P                  RV P                  R7      pV P                  V4      pV P                  V4      pV# )r   N)ÚmaskÚpreferred_attentionéÿÿÿÿ)Úshaper%   Úviewr"   r#   r&   Úunbindr   r	   r    r*   r+   )r,   r   Úcondr2   ÚBÚNÚCÚqÚkvÚkÚvs   &&&&       r   ÚforwardÚMultiHeadCrossAttention.forward)   s+  € à—'‘'‰ˆˆaà�M‰M˜!Ó×!Ñ! ! R¨¯©¸¿¹ÓGˆØ�^‰^˜DÓ!×&Ñ& q¨"¨a°·±ÀÇÁÓOˆØ�y‰y˜‹|‰ˆàŠ|Ðˆ|ô4 +¨1¯6©6°!°R¸Ó+;Ó<Ô>VÐWX×W]ÑW]Ð^_ÐacÐefÓWgÓ>hô  kCð  DE÷  DJñ  DJð  KLð  NPð  RSó  DTó  kUˆaˆØÜ  a¯©¸dÐX\×XlÑXlÔmˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr   )	r)   r    r!   r#   r&   r"   r*   r+   r%   )ç        rB   NNN©N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r@   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r/   Ú__classdict__s   @@r   r   r      s   ù‡ € ÷/÷'ô 'r   r   c                   óN   a a€ ] tR t^St oRtRV 3R lltRR ltRR ltRtVt	V ;t
# )	ÚAttentionKVCompresszAMulti-head Attention block with KV token compression and qk norm.c
           
     ó¤  <€ \         SV `  4        \        4       V n        W,          ^ 8X  g   Q R4       hW n        W,          V n        V P
                  R,          V n        V	P                  W^,          W7VR7      V n        V	P                  WWxR7      V n	        W@n
        WPn        V^8”  d:   VR8X  d3   V	P                  WWWWVR7      V n        V	P                  WVR7      V n        V'       d3   V	P                  WVR7      V n        V	P                  WVR7      V n        R# \$        P&                  ! 4       V n        \$        P&                  ! 4       V n        R# )z©
Args:
    dim (int): Number of input channels.
    num_heads (int): Number of attention heads.
    qkv_bias (bool:  If True, add a learnable bias to query, key, value.
z$dim should be divisible by num_heads©Úbiasr   r   r   Úconv)ÚgroupsÚkernel_sizeÚstrider   r   Ng      à¿)r   r   r   r    r"   r#   r   r$   Úqkvr*   ÚsamplingÚsr_ratioÚConv2dÚsrÚ	LayerNormÚnormÚq_normÚk_normr'   ÚIdentity)r,   Údimr"   Úqkv_biasrW   rX   Úqk_normr   r   r-   r.   r/   s   &&&&&&&&&&,€r   r   ÚAttentionKVCompress.__init__U   s!  ø€ ô 	‰ÑÔÜ-Ó/ˆÔØ� !Ô#ÐKÐ%KÓKÐ#Ø"ŒØÕ(ˆŒØ—]‘] dÕ*ˆŒ
à×$Ñ$ S°­'¸ÐV\Ð$Ó]ˆŒØ×%Ñ% c°eÐ%ÓKˆŒ	àŒØ ŒØ�aŒ<˜H¨Ô.à ×'Ñ'¨¸Ð[cÐyÐ'ó  AˆDŒGð #×,Ñ,¨SÀfÐ,ÓMˆDŒIßØ$×.Ñ.¨sÈÐ.ÓOˆDŒKØ$×.Ñ.¨sÈÐ.ÓOˆDŽKäŸ+š+›-ˆDŒKÜŸ+š+›-ˆDŽKr   c                óÔ  € Ve   V^8X  d   V# VP                   w  rgpVR8X  d    VRR R V13,          \        Wt,          4      3# VP                  WbW84      P                  ^ ^^^4      p\        W$,          4      \        W4,          4      r©Wš,          pVR8X  d4   \        P
                  ! V^V,          RR7      P                  ^ ^^^4      pM~VR8X  d'   VRRR R V1R R V13,          P                  ^ ^^^4      pMQVR8X  dE   V P                  V4      P                  WhR4      P                  ^ ^^4      pV P                  V4      pM\        hVP                  WkV4      P                  4       V3# )	NÚuniform_everyºNNNÚaveÚnearest)Úscale_factorÚmodeÚuniformrR   r4   )
r5   ÚintÚreshapeÚpermuteÚFÚinterpolaterZ   r\   Ú
ValueErrorÚ
contiguous)r,   ÚtensorÚHÚWri   rW   r9   r:   r;   Únew_HÚnew_WÚnew_Ns   &&&&&&      r   Údownsample_2dÚ!AttentionKVCompress.downsample_2du   sZ  € ØÒ˜|¨qÔ0ØˆMØ—,‘,‰ˆˆaà�Ô&Ø˜!™^˜|˜^Ð+Õ,¬c°!Õ2CÓ.DÐDÐDà—‘  aÓ+×3Ñ3°A°q¸!¸QÓ?ˆÜ˜1Õ+Ó,¬c°!Õ2BÓ.CˆuØ•ˆà�uÔÜ—]’]Ø Q¨Õ%5¸Iôç‰g�a˜˜A˜qÓ!ñ ð ˜Ô"Ø˜A˜q¡. L .±.°L°.Ð@ÕA×IÑIÈ!ÈQÐPQÐSTÓU‰FØ˜ÔØ—W‘W˜V“_×,Ñ,¨Q°2Ó6×>Ñ>¸qÀ!ÀQÓGˆFØ—Y‘Y˜vÓ&‰FäÐà�~‰~˜a¨Ó*×5Ñ5Ó7¸Ð>Ð>r   c           
     óú  € VP                   w  rVpTpVf   \        VR,          4      ;ršMVw  ršV P                  V4      P                  WV^V4      pVP	                  ^4      w  rÍpV P                  V4      pV P                  V4      pV P                  ^8”  dU   V P                  WÙW P                  V P                  R7      w  rØV P                  WéW P                  V P                  R7      w  rèVP                  WVV P                  WpP                  ,          4      pVP                  WXV P                  WpP                  ,          4      pVP                  WXV P                  WpP                  ,          4      pVe   \        R4      h\        R WÍV34      w  rÍp\        V4      \        V4      \        V4      ríp?\        WÍWàP                  R RV P                  R7      pVP!                  WVV4      pV P#                  V4      pV# )Nç      à?)rW   z,Attn mask logic not added for self attentionc                 ó&   € V P                  ^^4      # r   )Ú	transpose)Úts   &r   Ú<lambda>Ú-AttentionKVCompress.forward.<locals>.<lambda>¯   s   €  §¡¨A¨qÔ 1r   T)r2   Úskip_reshaper3   )r5   rl   rV   rm   r7   r]   r^   rX   ry   rW   r"   ÚNotImplementedErrorÚmapr   r	   r    r6   r*   )r,   r   r2   ÚHWÚblock_idr9   r:   r;   rx   rt   ru   rV   r<   r>   r?   s   &&&&&          r   r@   ÚAttentionKVCompress.forward�   s«  € Ø—'‘'‰ˆˆaØˆØŠ:Ü˜˜S�“MÐ!ˆA�à‰DˆAØ�h‰h�q‹k×!Ñ! !¨¨1Ó-ˆà—*‘*˜Q“-‰ˆˆaØ�K‰K˜‹NˆØ�K‰K˜‹Nˆð �=‰=˜1ÔØ×)Ñ)¨!°·=±=È4Ï=É=Ð)ÓY‰HˆAØ×)Ñ)¨!°·=±=È4Ï=É=Ð)ÓY‰HˆAà�I‰I�a˜DŸN™N¨A·±Õ,?Ó@ˆØ�I‰I�a §¡°·^±^Õ0CÓDˆØ�I‰I�a §¡°·^±^Õ0CÓDˆàÒÜ%Ð&TÓUÐUô Ñ1°A¸!°9Ó>‰ˆˆaÜ*¨1Ó-Ô/GÈÓ/JÔLdÐefÓLgˆaˆØÜ  a¯©¸dÐQUÐko×kÑkô  Aˆà�F‰F�1˜‹OˆØ�I‰I�a‹LˆØˆr   )r    r#   r^   r\   r"   r*   r]   rV   rW   r   rZ   rX   )é   TrR   r   FNNNrC   ©NNN)rD   rE   rF   rG   Ú__doc__r   ry   r@   rH   rI   rJ   rK   s   @@r   rN   rN   S   s   ù‡ € ÙK÷(ô@?÷4'ô 'r   rN   c                   ó@   a a€ ] tR t^¹t oRtRV 3R lltR tRtVtV ;t	# )Ú
FinalLayerú
The final layer of PixArt.
c                ó8  <€ \         SV `  4        VP                  VR RWER7      V n        VP	                  WV,          V,          RWER7      V n        \        P                  ! \        P                  ! 4       VP	                  V^V,          RWER7      4      V n	        R# ©Fç�íµ ÷Æ°>©Úelementwise_affineÚepsr   r   TrP   N©
r   r   r[   Ú
norm_finalr$   Úlinearr'   Ú
SequentialÚSiLUÚadaLN_modulation©r,   Úhidden_sizeÚ
patch_sizeÚout_channelsr   r   r-   r/   s   &&&&&&&€r   r   ÚFinalLayer.__init__½   sˆ   ø€ Ü‰ÑÔØ$×.Ñ.¨{ÈuÐZ^ÐfkÐ.Ó{ˆŒØ ×'Ñ'¨À*Õ5LÈ|Õ5[ÐbfÐnsÐ'ó  DˆŒÜ "§¢Ü�GŠG‹IØ×Ñ˜k¨1¨{­?ÀÈUÐÓbó!
ˆÖr   c                ó¦   € V P                  V4      P                  ^^R7      w  r4\        V P                  V4      W44      pV P	                  V4      pV# ©é   ©r`   ©r™   Úchunkr   r•   r–   )r,   r   Úcr   r   s   &&&  r   r@   ÚFinalLayer.forwardÆ   óK   € Ø×,Ñ,¨QÓ/×5Ñ5°a¸QÐ5Ó?‰ˆÜ�T—_‘_ QÓ'¨Ó6ˆØ�K‰K˜‹NˆØˆr   ©r™   r–   r•   r‰   ©
rD   rE   rF   rG   rŠ   r   r@   rH   rI   rJ   rK   s   @@r   rŒ   rŒ   ¹   s   ù‡ € ñ÷
÷ò r   rŒ   c                   ó@   a a€ ] tR t^Ìt oRtRV 3R lltR tRtVtV ;t	# )ÚT2IFinalLayerr�   c                ó.  <€ \         SV `  4        VP                  VR RWER7      V n        VP	                  WV,          V,          RWER7      V n        \        P                  ! \        P                  ! ^V4      VR,          ,          4      V n
        W0n        R# )Fr�   r‘   TrP   r|   N)r   r   r[   r•   r$   r–   r'   Ú	ParameterÚtorchÚrandnÚscale_shift_tabler�   rš   s   &&&&&&&€r   r   ÚT2IFinalLayer.__init__Ð   s€   ø€ Ü‰ÑÔØ$×.Ñ.¨{ÈuÐZ^ÐfkÐ.Ó{ˆŒØ ×'Ñ'¨À*Õ5LÈ|Õ5[ÐbfÐnsÐ'ó  DˆŒÜ!#§¢¬e¯kªk¸!¸[Ó.IÈKÐ[^ÕL^Õ.^Ó!_ˆÔØ(Ör   c                ó  € V P                   R ,          P                  VP                  VP                  R7      VR,          ,           P	                  ^^R7      w  r4\        V P                  V4      W44      pV P                  V4      pV# )Nr   r¢   ©rf   N)r°   Útor   r   r¤   r   r•   r–   ©r,   r   r   r   r   s   &&&  r   r@   ÚT2IFinalLayer.forward×   sq   € Ø×.Ñ.¨tÕ4×7Ñ7¸a¿g¹gÈaÏhÉhÐ7ÓWÐZ[Ð\cÕZdÕd×kÑkÐlmÐstÐkÓu‰ˆÜ˜Ÿ™¨Ó+¨UÓ:ˆØ�K‰K˜‹NˆØˆr   )r–   r•   r�   r°   r‰   r©   rK   s   @@r   r«   r«   Ì   s   ù‡ € ñ÷)÷ò r   r«   c                   ó@   a a€ ] tR t^Þt oRtRV 3R lltR tRtVtV ;t	# )ÚMaskFinalLayerr�   c                ó8  <€ \         SV `  4        VP                  VR RWVR7      V n        VP	                  WV,          V,          RWVR7      V n        \        P                  ! \        P                  ! 4       VP	                  V^V,          RWVR7      4      V n	        R# r�   r”   )	r,   Úfinal_hidden_sizeÚ
c_emb_sizerœ   r�   r   r   r-   r/   s	   &&&&&&&&€r   r   ÚMaskFinalLayer.__init__â   s’   ø€ Ü‰ÑÔØ$×.Ñ.Ð/@ÐUZÐ`dÐlqÐ.ó  BˆŒØ ×'Ñ'Ð(9È
Õ;RÐUaÕ;aÐhlÐtyÐ'ó  JˆŒÜ "§¢Ü�GŠG‹IØ×Ñ˜j¨!Ð.?Õ*?ÀdÐRWÐÓgó!
ˆÖr   c                ó¦   € V P                  V4      P                  ^^R7      w  r4\        V P                  V4      W44      pV P	                  V4      pV# r    r£   rµ   s   &&&  r   r@   ÚMaskFinalLayer.forwardê   r§   r   r¨   r‰   r©   rK   s   @@r   r¸   r¸   Þ   ó   ù‡ € ñ÷
÷ò r   r¸   c                   ó@   a a€ ] tR t^ñt oRtRV 3R lltR tRtVtV ;t	# )ÚDecoderLayerr�   c                ó  <€ \         SV `  4        VP                  VR RW4R7      V n        VP	                  WRW4R7      V n        \        P                  ! \        P                  ! 4       VP	                  V^V,          RW4R7      4      V n	        R# r�   )
r   r   r[   Únorm_decoderr$   r–   r'   r—   r˜   r™   )r,   r›   Údecoder_hidden_sizer   r   r-   r/   s   &&&&&&€r   r   ÚDecoderLayer.__init__õ   s|   ø€ Ü‰ÑÔØ&×0Ñ0°ÐQVÐ\`ÐhmÐ0Ó}ˆÔØ ×'Ñ'¨ÈtÐ[`Ð'ÓpˆŒÜ "§¢Ü�GŠG‹IØ×Ñ˜k¨1¨{­?ÀÈUÐÓbó!
ˆÖr   c                ó¦   € V P                  V4      P                  ^^R7      w  r4\        V P                  V4      W44      pV P	                  V4      pV# r    )r™   r¤   r   rÃ   r–   rµ   s   &&&  r   r@   ÚDecoderLayer.forwardý   sM   € Ø×,Ñ,¨QÓ/×5Ñ5°a¸QÐ5Ó?‰ˆÜ�T×&Ñ& qÓ)¨5Ó8ˆØ�K‰K˜‹NˆØˆr   )r™   r–   rÃ   r‰   r©   rK   s   @@r   rÁ   rÁ   ñ   r¿   r   rÁ   c                   ó@   a a€ ] tR tRt oRtRV 3R lltR tRtVtV ;t	# )ÚSizeEmbedderi  z6
Embeds scalar timesteps into vector representations.
c                óê   <€ \         SV `  WVR 7       \        P                  ! VP	                  W!RW4R7      \        P
                  ! 4       VP	                  WRW4R7      4      V n        W n        Wn        R# ))r›   Úfrequency_embedding_sizer-   TrP   N)	r   r   r'   r—   r$   r˜   ÚmlprË   Úoutdim)r,   r›   rË   r   r   r-   r/   s   &&&&&&€r   r   ÚSizeEmbedder.__init__  si   ø€ Ü‰Ñ [ÐpzÐÔ{Ü—=’=Ø×ÑÐ6È$ÐV[ÐÓkÜ�GŠG‹IØ×Ñ˜k¸TÈÐÓ^ó
ˆŒð
 )AÔ%Ø!Žr   c                ó(  € VP                   ^8X  d
   VR,          pVP                   ^8X  g   Q hVP                  ^ ,          V8w  dD   VP                  W!P                  ^ ,          ,          ^4      pVP                  ^ ,          V8X  g   Q hVP                  ^ ,          VP                  ^,          rC\        VR4      p\	        WP
                  4      pV P                  VP                  VP                  4      4      p\        VRW4V P                  R7      pV# )r   zb d -> (b d)z(b d) d2 -> b (d d2))ÚbÚdÚd2r³   )
Úndimr5   Úrepeatr   r   rË   rÌ   r´   r   rÍ   )r,   ÚsÚbsrÐ   ÚdimsÚs_freqÚs_embs   &&&    r   r@   ÚSizeEmbedder.forward  sÎ   € Ø�6‰6�QŒ;Ø�'•
ˆAØ�v‰v˜Œ{Ðˆ{Ø�7‰7�1�:˜ÔØ—‘˜ŸW™W Q�Z�¨Ó+ˆAØ—7‘7˜1•: Ô#Ð#Ð#Ø—'‘'˜!•*˜aŸg™g a�jˆ4Ü�a˜Ó(ˆÜ# A×'DÑ'DÓEˆØ—‘˜Ÿ™ 1§7¡7Ó+Ó,ˆÜ˜%Ð!7¸1ÈÏÉÔUˆØˆr   )rË   rÌ   rÍ   )é   NNNr©   rK   s   @@r   rÉ   rÉ     s   ù‡ € ñ÷"÷ò r   rÉ   c                   óN   a a€ ] tR tRt oRtRV 3R lltRR ltRR ltRtVt	V ;t
# )	ÚLabelEmbedderi!  úk
Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.
c                ó†   <€ \         SV `  4        V^ 8„  pVP                  W,           W$VR7      3V n        Wn        W0n        R# )r   r   N)r   r   Ú	EmbeddingÚembedding_tableÚnum_classesÚdropout_prob)	r,   râ   r›   rã   r   r   r-   Úuse_cfg_embeddingr/   s	   &&&&&&& €r   r   ÚLabelEmbedder.__init__%  sG   ø€ Ü‰ÑÔØ(¨1Ñ,ÐØ)×3Ñ3°KÕ4SÐU`Ðv|Ð3Ó}Ð~ˆÔØ&ÔØ(Ör   c                óà   € VfD   \         P                  ! VP                  ^ ,          4      P                  4       V P                  8  pMV^8H  p\         P
                  ! W0P                  V4      pV# )ú2
Drops labels to enable classifier-free guidance.
)r®   Úrandr5   Úcudarã   Úwhererâ   )r,   ÚlabelsÚforce_drop_idsÚdrop_idss   &&& r   Ú
token_dropÚLabelEmbedder.token_drop,  sY   € ð Ò!Ü—z’z &§,¡,¨q¥/Ó2×7Ñ7Ó9¸D×<MÑ<MÑM‰Hà%¨Ñ*ˆHÜ—’˜X×'7Ñ'7¸Ó@ˆØˆr   c                ó�   € V P                   ^ 8„  pV'       d	   V'       g   Ve   V P                  W4      pV P                  V4      pV# )r   )rã   rî   rá   )r,   rë   Útrainrì   Úuse_dropoutÚ
embeddingss   &&&&  r   r@   ÚLabelEmbedder.forward7  s@   € Ø×'Ñ'¨!Ñ+ˆß—k ~Ò'AØ—_‘_ VÓ<ˆFØ×)Ñ)¨&Ó1ˆ
ØÐr   )rã   rá   râ   r‰   rC   )rD   rE   rF   rG   rŠ   r   rî   r@   rH   rI   rJ   rK   s   @@r   rÝ   rÝ   !  s   ù‡ € ñ÷)ô	÷ô r   rÝ   c                   óz   a a€ ] tR tRt oRt]P                  ! RR7      ^xRRR3V 3R lltR
R ltR
R lt	R	t
VtV ;t# )ÚCaptionEmbedderi?  rÞ   Útanh©ÚapproximateNc	           
     óè   <€ \         S	V `  4        \        WW$WgVR 7      V n        V P	                  R\
        P                  ! \        P                  ! WQ4      VR,          ,          4      4       W0n	        R# )©Úin_featuresÚhidden_featuresÚout_featuresÚ	act_layerr   r   r-   Úy_embeddingr|   N)
r   r   r   Úy_projÚregister_bufferr'   r­   r®   r¯   Úuncond_prob©
r,   Úin_channelsr›   r  rÿ   Ú	token_numr   r   r-   r/   s
   &&&&&&&&&€r   r   ÚCaptionEmbedder.__init__C  sZ   ø€ Ü‰ÑÔÜØ#È{Ø°:ô
ˆŒð 	×Ñ˜]¬B¯LªL¼¿ºÀYÓ9\Ð_jÐnqÕ_qÕ9qÓ,rÔsØ&Ör   c                óð   € VfD   \         P                  ! VP                  ^ ,          4      P                  4       V P                  8  pMV^8H  p\         P
                  ! VR,          V P                  V4      pV# )rç   ©rf   NNN)r®   rè   r5   ré   r  rê   r   )r,   Úcaptionrì   rí   s   &&& r   rî   ÚCaptionEmbedder.token_dropL  sb   € ð Ò!Ü—z’z '§-¡-°Õ"2Ó3×8Ñ8Ó:¸T×=MÑ=MÑM‰Hà%¨Ñ*ˆHÜ—+’+˜hÐ':Õ;¸T×=MÑ=MÈwÓWˆØˆr   c                óü   € V'       d/   VP                   R ,          V P                  P                   8X  g   Q hV P                  ^ 8„  pV'       d	   V'       g   Ve   V P                  W4      pV P	                  V4      pV# )ºr¡   NN)r5   r   r  rî   r  )r,   r
  rñ   rì   rò   s   &&&& r   r@   ÚCaptionEmbedder.forwardW  sc   € ßØ—=‘= Õ$¨×(8Ñ(8×(>Ñ(>Ô>Ð>Ð>Ø×&Ñ&¨Ñ*ˆß—k ~Ò'AØ—o‘o gÓ>ˆGØ—+‘+˜gÓ&ˆØˆr   )r  r  rC   ©rD   rE   rF   rG   rŠ   r'   ÚGELUr   rî   r@   rH   rI   rJ   rK   s   @@r   rö   rö   ?  sC   ù‡ € ñð IKÏÊÐ\bÔHcÐorÐz~ð  HLð  Y]÷ 'ô	÷ô r   rö   c                   óz   a a€ ] tR tRt oRt]P                  ! RR7      ^xRRR3V 3R lltR
R ltR
R lt	R	t
VtV ;t# )ÚCaptionEmbedderDoubleBria  rÞ   r÷   rø   Nc	           
     ó2  <€ \         S	V `  4        \        WW$WgVR 7      V n        \        P
                  ! \        P                  ! ^V4      R,          4      V n        \        P
                  ! \        P                  ! WQ4      R,          4      V n	        W0n
        R# )rû   NgS[Ú:XL	@)r   r   r   r*   r'   r­   r®   r¯   Ú	embeddingr   r  r  s
   &&&&&&&&&€r   r   Ú CaptionEmbedderDoubleBr.__init__e  sk   ø€ Ü‰ÑÔÜØ#È{Ø°:ô
ˆŒ	ô Ÿš¤e§k¢k°!°[Ó&AÀIÕ&MÓNˆŒÜŸ<š<¬¯ª°IÓ(KÈiÕ(WÓXˆÔØ&Ör   c                óD  € VfD   \         P                  ! VP                  ^ ,          4      P                  4       V P                  8  pMV^8H  p\         P
                  ! VR,          V P                  V4      p\         P
                  ! VR,          V P                  V4      pW3# )rç   r³   r	  )r®   rè   r5   ré   r  rê   r  r   )r,   Úglobal_captionr
  rì   rí   s   &&&& r   rî   Ú"CaptionEmbedderDoubleBr.token_dropo  sƒ   € ð Ò!Ü—z’z .×"6Ñ"6°qÕ"9Ó:×?Ñ?ÓAÀD×DTÑDTÑT‰Hà%¨Ñ*ˆHÜŸš X¨gÕ%6¸¿¹ÈÓWˆÜ—+’+˜hÐ':Õ;¸T×=MÑ=MÈwÓWˆØÐ&Ð&r   c                ó4  € VP                   R ,          V P                  P                   8X  g   Q hVP                  ^R7      P                  4       pV P                  ^ 8„  pV'       d	   V'       g   Ve   V P                  WAV4      w  rAV P                  V4      pWa3# )r  r¢   )r5   r   ÚmeanÚsqueezer  rî   r*   )r,   r
  rñ   rì   r  rò   Úy_embeds   &&&&   r   r@   ÚCaptionEmbedderDoubleBr.forward{  s‚   € Ø�}‰}˜RÕ! T×%5Ñ%5×%;Ñ%;Ô;Ð;Ð;Ø Ÿ™¨!˜Ó,×4Ñ4Ó6ˆØ×&Ñ&¨Ñ*ˆß—k ~Ò'AØ&*§o¡o°nÈ~Ó&^Ñ#ˆNØ—)‘)˜NÓ+ˆØÐÐr   )r  r*   r  r   rC   r  rK   s   @@r   r  r  a  sC   ù‡ € ñð IKÏÊÐ\bÔHcÐorÐz~ð  HLð  Y]÷ 'ô
'÷ ô  r   r  )r®   Útorch.nnr'   Útorch.nn.functionalÚ
functionalro   Úeinopsr   Ú(comfy.ldm.modules.diffusionmodules.mmditr   r   r   Úcomfy.ldm.modules.attentionr   r   r	   r   r   ÚModuler   rN   rŒ   r«   r¸   rÁ   rÉ   rÝ   rö   r  r   r   r   Ú<module>r%     sÎ   ðó Ý ß Ð Ý ç ^Ñ ^ß eÑ eò=ò#ô7˜bŸi™iô 7ôtc˜"Ÿ)™)ô côL�—‘ô ô&�B—I‘Iô ô$�R—Y‘Yô ô&�2—9‘9ô ô&Ð#ô ô:�B—I‘Iô ô<�b—i‘iô ôD! ˜bŸi™iö ! r   